70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
192 ImmTyMatrixAScaleFmt,
193 ImmTyMatrixBScaleFmt,
226 mutable int MCOpIdx = -1;
229 bool isToken()
const override {
return Kind == Token; }
231 bool isSymbolRefExpr()
const {
235 bool isImm()
const override {
return Kind == Immediate; }
237 bool isInlinableImm(MVT type)
const;
238 bool isLiteralImm(MVT type)
const;
240 bool isRegKind()
const {
return Kind == Register; }
242 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
244 bool isRegOrInline(
unsigned RCID, MVT type)
const {
245 return isRegClass(RCID) || isInlinableImm(type);
248 bool isRegOrInlineTarget(
unsigned TargetRCIdx, MVT type)
const {
249 return isRegClassTarget(TargetRCIdx) || isInlinableImm(type);
253 return isRegOrInline(RCID, type) || isLiteralImm(type);
256 bool isRegOrImmWithInputModsTarget(
unsigned TargetRCIdx, MVT type)
const {
257 return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type);
260 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
262 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
265 bool isRegOrImmWithInt32InputMods()
const {
269 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
270 return isRegOrInline(
271 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
274 bool isRegOrInlineImmWithInt32InputMods()
const {
275 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
278 bool isRegOrImmWithInt64InputMods()
const {
279 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
282 bool isRegOrImmWithFP16InputMods()
const {
286 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
288 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
291 bool isRegOrImmWithFPT16_LO16InputMods()
const {
295 bool isRegOrImmWithFP32InputMods()
const {
299 bool isRegOrImmWithFP64InputMods()
const {
300 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
303 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
304 return isRegOrInline(
305 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
308 bool isRegOrInlineImmWithFP32InputMods()
const {
309 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
312 bool isRegOrInlineImmWithFP64InputMods()
const {
313 return isRegOrInlineTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
316 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
318 bool isVRegWithFP32InputMods()
const {
319 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
322 bool isVRegWithFP64InputMods()
const {
323 return isRegClassTarget(AMDGPU::VReg_64_AlignTarget);
326 bool isPackedFP16InputMods()
const {
330 bool isPackedVGPRFP32InputMods()
const {
334 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
336 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
338 bool isRsrcReg32()
const {
return isRegClass(AMDGPU::RsrcReg32RegClassID); }
340 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
342 bool isAV_LdSt_32_Align2_RegOp()
const {
343 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
344 isRegClass(AMDGPU::AGPR_32RegClassID);
347 bool isVRegWithInputMods()
const;
348 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
349 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
350 bool isT16_LO16VRegWithInputMods()
const;
352 bool isSDWAOperand(MVT type)
const;
353 bool isSDWAFP16Operand()
const;
354 bool isSDWAFP32Operand()
const;
355 bool isSDWAInt16Operand()
const;
356 bool isSDWAInt32Operand()
const;
358 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
360 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
362 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
364 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
366 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
367 bool isDim()
const {
return isImmTy(ImmTyDim); }
368 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
369 bool isOff()
const {
return isImmTy(ImmTyOff); }
370 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
371 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
372 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
373 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
374 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
375 bool isFlatOffset()
const {
376 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
378 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
379 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
380 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
381 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
382 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
383 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
384 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
385 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
386 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
387 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
388 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
389 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
390 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
391 bool isFormat()
const {
return isImmTy(ImmTyFormat) &&
isUInt<7>(
getImm()); }
392 bool isFormatOp()
const {
395 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
396 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
397 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
398 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
399 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
400 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
401 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
402 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
403 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
404 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
405 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
406 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
407 bool isDone()
const {
return isImmTy(ImmTyDone); }
408 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
410 bool isRegOrImm()
const {
return isReg() || isImm(); }
412 bool isRegClass(
unsigned RCID)
const;
415 bool isRegClassTarget(
unsigned TargetRCIdx)
const;
419 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
420 return isRegOrInline(RCID, type) && !hasModifiers();
423 bool isRegOrInlineNoModsTarget(
unsigned TargetRCIdx, MVT type)
const {
424 return isRegOrInlineTarget(TargetRCIdx, type) && !hasModifiers();
427 bool isSCSrcB16()
const {
428 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
431 bool isSCSrc_b32()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
435 bool isSCSrc_b64()
const {
436 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
439 bool isBoolReg()
const;
441 bool isSSrc_b32()
const {
442 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
445 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
447 bool isSSrc_b64()
const {
450 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
451 (((
const MCTargetAsmParser *)AsmParser)
452 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
456 bool isSSrc_f32()
const {
457 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
460 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
462 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
464 bool isSSrc_NoInline_f16()
const {
return isSSrc_f16(); }
466 bool isSSrcOrLds_b32()
const {
467 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
468 isLiteralImm(MVT::i32) || isExpr();
471 bool isVCSrc_b32()
const {
472 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
475 bool isVCSrc_b32_Lo256()
const {
476 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
479 bool isVCSrc_b64_Lo256()
const {
480 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
483 bool isVCSrc_b64()
const {
484 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
487 bool isVCSrcT_b16()
const {
488 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
491 bool isVCSrcTB16_Lo128()
const {
492 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
495 bool isVCSrcFake16B16_Lo128()
const {
496 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
499 bool isVCSrc_b16()
const {
500 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
503 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
505 bool isVCSrc_f32()
const {
506 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
509 bool isVCSrc_f64()
const {
510 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
513 bool isVCSrcTBF16()
const {
514 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
517 bool isVCSrcT_f16()
const {
518 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
521 bool isVCSrcT_bf16()
const {
522 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
525 bool isVCSrcTF16_LO16()
const {
526 return isRegOrInlineNoMods(AMDGPU::VS_16_LO16RegClassID, MVT::f16);
529 bool isVCSrcTBF16_Lo128()
const {
530 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
533 bool isVCSrcTF16_Lo128()
const {
534 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
537 bool isVCSrcFake16BF16_Lo128()
const {
538 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
541 bool isVCSrcFake16F16_Lo128()
const {
542 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
545 bool isVCSrc_bf16()
const {
546 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
549 bool isVCSrc_f16()
const {
550 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
553 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
555 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
557 bool isVSrc_b32()
const {
558 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
561 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
563 bool isVSrc_v2b64()
const {
564 return isRegOrInlineNoModsTarget(AMDGPU::VS_128_AlignTarget, MVT::i64) ||
565 isLiteralImm(MVT::i64);
568 bool isVSrc_v2f64()
const {
569 return isRegOrInlineNoModsTarget(AMDGPU::VS_128_AlignTarget, MVT::f64) ||
570 isLiteralImm(MVT::f64);
573 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
575 bool isVSrcT_b16_Lo128()
const {
576 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
579 bool isVSrcFake16_b16_Lo128()
const {
580 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
583 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
585 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
587 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
589 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
591 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
593 bool isVSrc_f32()
const {
594 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
597 bool isVSrc_f64()
const {
598 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64) ||
599 isLiteralImm(MVT::f64);
602 bool isVSrcT_bf16()
const {
603 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
606 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
608 bool isVSrcT_f16_LO16()
const {
609 return isVCSrcTF16_LO16() || isLiteralImm(MVT::f16);
612 bool isVSrcT_bf16_Lo128()
const {
613 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
616 bool isVSrcT_f16_Lo128()
const {
617 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
620 bool isVSrcFake16_bf16_Lo128()
const {
621 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
624 bool isVSrcFake16_f16_Lo128()
const {
625 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
628 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
630 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
632 bool isVSrc_v2bf16()
const {
633 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
636 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
638 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
640 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
642 bool isVISrc_64_bf16()
const {
643 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::bf16);
646 bool isVISrc_64_f16()
const {
647 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f16);
650 bool isVISrc_64_b32()
const {
651 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::i32);
654 bool isVISrc_64_f64()
const {
655 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f64);
658 bool isVISrc_256_b32()
const {
659 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::i32);
662 bool isVISrc_256_f32()
const {
663 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f32);
666 bool isVISrc_256_f64()
const {
667 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f64);
670 bool isVISrc_512_f64()
const {
671 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f64);
674 bool isVISrc_128_b32()
const {
675 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::i32);
678 bool isVISrc_128_f32()
const {
679 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f32);
682 bool isVISrc_512_b32()
const {
683 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::i32);
686 bool isVISrc_512_f32()
const {
687 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f32);
690 bool isVISrc_1024_b32()
const {
691 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::i32);
694 bool isVISrc_1024_f32()
const {
695 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::f32);
698 bool isAISrc_64_f64()
const {
699 return isRegOrInlineNoModsTarget(AMDGPU::AReg_64_AlignTarget, MVT::f64);
702 bool isAISrc_128_b32()
const {
703 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::i32);
706 bool isAISrc_128_f32()
const {
707 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::f32);
710 bool isVISrc_128_bf16()
const {
711 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::bf16);
714 bool isVISrc_128_f16()
const {
715 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f16);
718 bool isAISrc_256_f64()
const {
719 return isRegOrInlineNoModsTarget(AMDGPU::AReg_256_AlignTarget, MVT::f64);
722 bool isAISrc_512_b32()
const {
723 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::i32);
726 bool isAISrc_512_f32()
const {
727 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::f32);
730 bool isAISrc_1024_b32()
const {
731 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::i32);
734 bool isAISrc_1024_f32()
const {
735 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::f32);
738 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
740 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
742 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
744 bool isMem()
const override {
return false; }
746 bool isExpr()
const {
return Kind == Expression; }
748 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
750 bool isSWaitCnt()
const;
751 bool isDepCtr()
const;
752 bool isSDelayALU()
const;
753 bool isHwreg()
const;
754 bool isSendMsg()
const;
755 bool isWaitEvent()
const;
756 bool isSplitBarrier()
const;
757 bool isSwizzle()
const;
758 bool isSMRDOffset8()
const;
759 bool isSMEMOffset()
const;
760 bool isSMRDLiteralOffset()
const;
762 bool isDPPCtrl()
const;
764 bool isGPRIdxMode()
const;
765 bool isS16Imm()
const;
766 bool isU16Imm()
const;
767 bool isEndpgm()
const;
769 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
770 return [
this,
P]() {
return P(*
this); };
775 return StringRef(Tok.Data, Tok.Length);
783 ImmTy getImmTy()
const {
788 MCRegister
getReg()
const override {
793 SMLoc getStartLoc()
const override {
return StartLoc; }
795 SMLoc getEndLoc()
const override {
return EndLoc; }
797 int getMCOpIdx()
const {
return MCOpIdx; }
799 Modifiers getModifiers()
const {
800 assert(isRegKind() || isImmTy(ImmTyNone));
801 return isRegKind() ?
Reg.Mods :
Imm.Mods;
804 void setModifiers(Modifiers Mods) {
805 assert(isRegKind() || isImmTy(ImmTyNone));
812 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
814 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
816 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
818 bool isForcedLit()
const {
819 return isImmLiteral() && getModifiers().isForcedLit();
822 bool isForcedLit64()
const {
823 return isImmLiteral() && getModifiers().isForcedLit64();
828 void addImmOperands(MCInst &Inst,
unsigned N,
829 bool ApplyModifiers =
true)
const;
831 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
832 bool ApplyModifiers)
const;
834 void addRegOperands(MCInst &Inst,
unsigned N)
const;
836 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
838 addRegOperands(Inst,
N);
840 addImmOperands(Inst,
N);
843 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
844 Modifiers Mods = getModifiers();
847 addRegOperands(Inst,
N);
849 addImmOperands(Inst,
N,
false);
853 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
854 assert(!hasIntModifiers());
855 addRegOrImmWithInputModsOperands(Inst,
N);
858 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
859 Modifiers Mods = getModifiers();
862 addRegOperands(Inst,
N);
865 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
866 assert(!hasIntModifiers());
867 addRegWithInputModsOperands(Inst,
N);
870 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
873 case ImmTyNone: OS <<
"None";
break;
874 case ImmTyGDS: OS <<
"GDS";
break;
875 case ImmTyLDS: OS <<
"LDS";
break;
876 case ImmTyOffen: OS <<
"Offen";
break;
877 case ImmTyIdxen: OS <<
"Idxen";
break;
878 case ImmTyAddr64: OS <<
"Addr64";
break;
879 case ImmTyOffset: OS <<
"Offset";
break;
880 case ImmTyInstOffset: OS <<
"InstOffset";
break;
881 case ImmTyOffset0: OS <<
"Offset0";
break;
882 case ImmTyOffset1: OS <<
"Offset1";
break;
883 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
884 case ImmTyCPol: OS <<
"CPol";
break;
885 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
886 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
887 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
888 case ImmTyTFE: OS <<
"TFE";
break;
889 case ImmTyIsAsync: OS <<
"IsAsync";
break;
890 case ImmTyD16: OS <<
"D16";
break;
891 case ImmTyFormat: OS <<
"Format";
break;
892 case ImmTyFormatOp: OS <<
"FormatOp";
break;
893 case ImmTyClamp: OS <<
"Clamp";
break;
894 case ImmTyOModSI: OS <<
"OModSI";
break;
895 case ImmTyDPP8: OS <<
"DPP8";
break;
896 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
897 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
898 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
899 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
900 case ImmTyDppFI: OS <<
"DppFI";
break;
901 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
902 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
903 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
904 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
905 case ImmTyDMask: OS <<
"DMask";
break;
906 case ImmTyDim: OS <<
"Dim";
break;
907 case ImmTyUNorm: OS <<
"UNorm";
break;
908 case ImmTyDA: OS <<
"DA";
break;
909 case ImmTyR128A16: OS <<
"R128A16";
break;
910 case ImmTyA16: OS <<
"A16";
break;
911 case ImmTyLWE: OS <<
"LWE";
break;
912 case ImmTyOff: OS <<
"Off";
break;
913 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
914 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
915 case ImmTyExpVM: OS <<
"ExpVM";
break;
916 case ImmTyDone: OS <<
"Done";
break;
917 case ImmTyRowEn: OS <<
"RowEn";
break;
918 case ImmTyHwreg: OS <<
"Hwreg";
break;
919 case ImmTySendMsg: OS <<
"SendMsg";
break;
920 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
921 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
922 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
923 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
924 case ImmTyOpSel: OS <<
"OpSel";
break;
925 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
926 case ImmTyNegLo: OS <<
"NegLo";
break;
927 case ImmTyNegHi: OS <<
"NegHi";
break;
928 case ImmTySwizzle: OS <<
"Swizzle";
break;
929 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
930 case ImmTyHigh: OS <<
"High";
break;
931 case ImmTyBLGP: OS <<
"BLGP";
break;
932 case ImmTyCBSZ: OS <<
"CBSZ";
break;
933 case ImmTyABID: OS <<
"ABID";
break;
934 case ImmTyEndpgm: OS <<
"Endpgm";
break;
935 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
936 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
937 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
938 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
939 case ImmTyBitOp3: OS <<
"BitOp3";
break;
940 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
941 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
942 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
943 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
944 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
945 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
946 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
947 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
948 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
949 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
954 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
958 <<
" mods: " <<
Reg.Mods <<
'>';
962 if (getImmTy() != ImmTyNone) {
964 printImmTy(OS, getImmTy());
966 OS <<
" mods: " <<
Imm.Mods <<
'>';
979 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
980 int64_t Val, SMLoc Loc,
981 ImmTy
Type = ImmTyNone,
982 bool IsFPImm =
false) {
983 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
985 Op->Imm.IsFPImm = IsFPImm;
987 Op->Imm.Mods = Modifiers();
993 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
994 StringRef Str, SMLoc Loc,
995 bool HasExplicitEncodingSize =
true) {
996 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
997 Res->Tok.Data = Str.data();
998 Res->Tok.Length = Str.size();
1004 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1005 MCRegister
Reg, SMLoc S, SMLoc
E) {
1006 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1007 Op->Reg.RegNo =
Reg;
1008 Op->Reg.Mods = Modifiers();
1014 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1015 const class MCExpr *Expr, SMLoc S) {
1016 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1025 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1034#define GET_REGISTER_MATCHER
1035#include "AMDGPUGenAsmMatcher.inc"
1036#undef GET_REGISTER_MATCHER
1037#undef GET_SUBTARGET_FEATURE_NAME
1042class KernelScopeInfo {
1043 int SgprIndexUnusedMin = -1;
1044 int VgprIndexUnusedMin = -1;
1045 int AgprIndexUnusedMin = -1;
1049 void usesSgprAt(
int i) {
1050 if (i >= SgprIndexUnusedMin) {
1051 SgprIndexUnusedMin = ++i;
1054 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1060 void usesVgprAt(
int i) {
1061 if (i >= VgprIndexUnusedMin) {
1062 VgprIndexUnusedMin = ++i;
1065 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1068 VgprIndexUnusedMin);
1074 void usesAgprAt(
int i) {
1079 if (i >= AgprIndexUnusedMin) {
1080 AgprIndexUnusedMin = ++i;
1083 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1088 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1091 VgprIndexUnusedMin);
1098 KernelScopeInfo() =
default;
1102 MSTI = Ctx->getSubtargetInfo();
1104 usesSgprAt(SgprIndexUnusedMin = -1);
1105 usesVgprAt(VgprIndexUnusedMin = -1);
1107 usesAgprAt(AgprIndexUnusedMin = -1);
1111 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1112 unsigned RegWidth) {
1115 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1118 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1121 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1130 MCAsmParser &Parser;
1132 unsigned ForcedEncodingSize = 0;
1133 bool ForcedDPP =
false;
1134 bool ForcedSDWA =
false;
1135 KernelScopeInfo KernelScope;
1136 const unsigned HwMode;
1138 const AMDGPU::IsaVersion ISA;
1143#define GET_ASSEMBLER_HEADER
1144#include "AMDGPUGenAsmMatcher.inc"
1149 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1151 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1155 std::optional<AMDGPU::InfoSectionData> InfoData;
1164 SmallVector<unsigned> OpcodeStream;
1166 OpcodeStreamSymbols;
1167 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1170 void checkKernelPrologues();
1173 void createConstantSymbol(StringRef Id, int64_t Val);
1175 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1176 bool OutOfRangeError(SMRange
Range);
1192 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1193 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1194 std::optional<bool> EnableWavefrontSize32,
1195 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1196 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1197 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1198 bool ParseDirectiveAMDGCNTarget();
1199 bool ParseDirectiveAMDHSACodeObjectVersion();
1200 bool ParseDirectiveAMDHSAKernel();
1201 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1202 bool ParseDirectiveAMDKernelCodeT();
1204 bool subtargetHasRegister(
const MCRegisterInfo &
MRI, MCRegister
Reg);
1205 bool ParseDirectiveAMDGPUHsaKernel();
1207 bool ParseDirectiveISAVersion();
1208 bool ParseDirectiveHSAMetadata();
1209 bool ParseDirectivePALMetadataBegin();
1210 bool ParseDirectivePALMetadata();
1211 bool ParseDirectiveAMDGPULDS();
1212 bool ParseDirectiveAMDGPUInfo();
1216 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1217 const char *AssemblerDirectiveEnd,
1218 std::string &CollectString);
1220 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1221 RegisterKind RegKind, MCRegister Reg1,
1222 RegisterKind RegKind1, SMLoc Loc);
1223 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1224 unsigned &RegNum,
unsigned &RegWidth,
1225 bool RestoreOnFailure =
false);
1226 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1227 unsigned &RegNum,
unsigned &RegWidth,
1228 SmallVectorImpl<AsmToken> &Tokens);
1229 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1231 SmallVectorImpl<AsmToken> &Tokens);
1232 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1234 SmallVectorImpl<AsmToken> &Tokens);
1235 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1237 SmallVectorImpl<AsmToken> &Tokens);
1238 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1239 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1240 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1243 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1244 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1245 void initializeGprCountSymbol(RegisterKind RegKind);
1246 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1252 OperandMode_Default,
1256 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1258 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1259 const MCInstrInfo &MII)
1260 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1261 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1266 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1268 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1269 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1270 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1271 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1273 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1274 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1275 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1277 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1278 initializeGprCountSymbol(IS_VGPR);
1279 initializeGprCountSymbol(IS_SGPR);
1284 createConstantSymbol(Symbol, Code);
1286 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1287 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1288 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1328 bool hasBVHRayTracingInsts()
const {
1329 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1332 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1334 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1336 bool hasInv2PiInlineImm()
const {
1337 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1340 bool has64BitLiterals()
const {
1341 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1344 bool hasFlatOffsets()
const {
1345 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1348 bool hasTrue16Insts()
const {
1349 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1353 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1356 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1358 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1360 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1362 bool hasPartialNSAEncoding()
const {
1363 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1366 bool hasGloballyAddressableScratch()
const {
1367 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1370 bool hasScaleOffset()
const {
1371 return getFeatureBits()[AMDGPU::FeatureScaleOffset];
1374 bool hasCacheFillSize()
const {
1375 return getFeatureBits()[AMDGPU::FeatureCacheFillSize];
1388 AMDGPUTargetStreamer &getTargetStreamer() {
1389 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1390 return static_cast<AMDGPUTargetStreamer &
>(TS);
1396 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1399 const MCRegisterInfo *getMRI()
const {
1403 const MCInstrInfo *getMII()
const {
return &MII; }
1407 int16_t getTargetRegClass(
unsigned TargetRCIdx)
const {
1413 const FeatureBitset &getFeatureBits()
const {
1414 return getSTI().getFeatureBits();
1417 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1418 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1419 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1421 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1422 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1423 bool isForcedDPP()
const {
return ForcedDPP; }
1424 bool isForcedSDWA()
const {
return ForcedSDWA; }
1425 ArrayRef<unsigned> getMatchedVariants()
const;
1426 StringRef getMatchedVariantName()
const;
1428 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1429 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1430 bool RestoreOnFailure);
1431 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1432 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1433 SMLoc &EndLoc)
override;
1434 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1435 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1436 unsigned Kind)
override;
1437 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1440 bool MatchingInlineAsm)
override;
1441 bool ParseDirective(AsmToken DirectiveID)
override;
1442 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1443 void onEndOfFile()
override;
1445 OperandMode
Mode = OperandMode_Default);
1446 StringRef parseMnemonicSuffix(StringRef Name);
1447 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1453 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1457 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1458 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1460 ParseStatus parseOperandArrayWithPrefix(
1462 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1463 bool (*ConvertResult)(int64_t &) =
nullptr);
1467 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1468 bool IgnoreNegative =
false);
1469 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1474 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1478 ArrayRef<const char *> Ids,
1482 ArrayRef<const char *> Ids,
1483 AMDGPUOperand::ImmTy
Type);
1486 bool isOperandModifier(
const AsmToken &Token,
1487 const AsmToken &NextToken)
const;
1488 bool isRegOrOperandModifier(
const AsmToken &Token,
1489 const AsmToken &NextToken)
const;
1490 bool isNamedOperandModifier(
const AsmToken &Token,
1491 const AsmToken &NextToken)
const;
1492 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1493 const AsmToken &NextToken)
const;
1494 bool parseSP3NegModifier();
1501 bool AllowImm =
true);
1503 bool AllowImm =
true);
1509 AMDGPUOperand::ImmTy ImmTy);
1514 AMDGPUOperand::ImmTy
Type);
1518 AMDGPUOperand::ImmTy
Type);
1522 AMDGPUOperand::ImmTy
Type);
1526 ParseStatus parseDfmtNfmt(int64_t &
Format);
1527 ParseStatus parseUfmt(int64_t &
Format);
1528 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1530 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1534 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1535 ParseStatus parseNumericFormat(int64_t &
Format);
1539 ParseStatus tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Fmt);
1540 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1545 bool parseCnt(int64_t &IntVal);
1548 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1549 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1552 bool parseDelay(int64_t &Delay);
1558 struct OperandInfoTy {
1561 bool IsSymbolic =
false;
1562 bool IsDefined =
false;
1564 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1567 struct StructuredOpField : OperandInfoTy {
1571 bool IsDefined =
false;
1573 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1574 unsigned Width, int64_t
Default)
1575 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1576 virtual ~StructuredOpField() =
default;
1578 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1579 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1583 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1585 return Error(Parser,
"not supported on this GPU");
1587 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1595 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1596 OperandInfoTy &Stream);
1597 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1598 const OperandInfoTy &Stream);
1600 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1601 OperandInfoTy &Width);
1606 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1613 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1615 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1619 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1625 bool validateBF16InlineConst(
const MCInst &Inst,
1628 bool validateConstantBusLimitations(
const MCInst &Inst,
1630 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1633 bool tryVOPD(
const MCInst &Inst);
1634 bool tryVOPD3(
const MCInst &Inst);
1635 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1637 bool validateIntClampSupported(
const MCInst &Inst);
1638 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1639 bool validateMIMGGatherDMask(
const MCInst &Inst);
1641 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1642 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1643 bool validateMIMGD16(
const MCInst &Inst);
1645 bool validateTensorR128(
const MCInst &Inst);
1646 bool validateMIMGMSAA(
const MCInst &Inst);
1647 bool validateOpSel(
const MCInst &Inst);
1648 bool validateTrue16OpSel(
const MCInst &Inst);
1649 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1651 bool validateVccOperand(MCRegister
Reg)
const;
1656 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1657 bool validateVGPRAlign(
const MCInst &Inst)
const;
1661 bool validateDivScale(
const MCInst &Inst);
1666 const unsigned CPol);
1672 bool validateClusterBarrierIsFirst(
const MCInst &Inst,
1675 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1676 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1677 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1678 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1680 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1681 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1682 ArrayRef<unsigned> Variants);
1683 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1685 bool isId(
const StringRef Id)
const;
1686 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1688 StringRef getId()
const;
1689 bool trySkipId(
const StringRef Id);
1690 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1694 bool parseString(StringRef &Val,
1695 const StringRef ErrMsg =
"expected a string");
1696 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1702 StringRef getTokenStr()
const;
1703 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1705 SMLoc getLoc()
const;
1709 void onBeginOfFile()
override;
1714 void emitTargetDirective();
1715 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1727 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1728 const unsigned MaxVal,
const Twine &ErrMsg,
1730 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1731 const unsigned MinVal,
const unsigned MaxVal,
1732 const StringRef ErrMsg);
1734 bool parseSwizzleOffset(int64_t &
Imm);
1735 bool parseSwizzleMacro(int64_t &
Imm);
1736 bool parseSwizzleQuadPerm(int64_t &
Imm);
1737 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1738 bool parseSwizzleBroadcast(int64_t &
Imm);
1739 bool parseSwizzleSwap(int64_t &
Imm);
1740 bool parseSwizzleReverse(int64_t &
Imm);
1741 bool parseSwizzleFFT(int64_t &
Imm);
1742 bool parseSwizzleRotate(int64_t &
Imm);
1745 int64_t parseGPRIdxMacro();
1748 cvtMubufImpl(Inst,
Operands,
false);
1751 cvtMubufImpl(Inst,
Operands,
true);
1757 OptionalImmIndexMap &OptionalIdx);
1766 OptionalImmIndexMap &OptionalIdx);
1768 OptionalImmIndexMap &OptionalIdx);
1772 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1774 bool parseDimId(
unsigned &Encoding);
1776 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1780 int64_t parseDPPCtrlSel(StringRef Ctrl);
1781 int64_t parseDPPCtrlPerm();
1787 bool IsDPP8 =
false);
1793 AMDGPUOperand::ImmTy
Type);
1801 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1804 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1805 bool SkipSrcVcc =
false);
1915bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
1925 if (!isImmTy(ImmTyNone)) {
1930 if (getModifiers().
Lit != LitModifier::None)
1940 if (type == MVT::f64 || type == MVT::i64) {
1942 AsmParser->hasInv2PiInlineImm());
1945 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
1964 APFloat::rmNearestTiesToEven, &Lost);
1971 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1973 AsmParser->hasInv2PiInlineImm());
1978 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
1979 AsmParser->hasInv2PiInlineImm());
1983 if (type == MVT::f64 || type == MVT::i64) {
1985 AsmParser->hasInv2PiInlineImm());
1994 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
1995 AsmParser->hasInv2PiInlineImm());
1999 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2000 AsmParser->hasInv2PiInlineImm());
2003bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2005 if (!isImmTy(ImmTyNone)) {
2010 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2015 if (type == MVT::f64 && hasFPModifiers()) {
2035 if (type == MVT::f64) {
2040 if (type == MVT::i64) {
2053 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2054 : (type == MVT::v2i16) ? MVT::f32
2055 : (type == MVT::v2f32) ? MVT::f32
2058 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2062bool AMDGPUOperand::isRegClassTarget(
unsigned TargetRCIdx)
const {
2065 int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx);
2066 return RCID >= 0 && isRegClass(RCID);
2069bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2070 return isRegKind() &&
2071 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2074bool AMDGPUOperand::isVRegWithInputMods()
const {
2075 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2077 (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] &&
2078 isRegClassTarget(AMDGPU::VReg_64_AlignTarget));
2081template <
bool IsFake16>
2082bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2083 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2084 : AMDGPU::VGPR_16_Lo128RegClassID);
2087template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2088 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2089 : AMDGPU::VGPR_16RegClassID);
2092bool AMDGPUOperand::isT16_LO16VRegWithInputMods()
const {
2093 return isRegClass(AMDGPU::VGPR_16_LO16RegClassID);
2096bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2097 if (AsmParser->isVI())
2099 if (AsmParser->isGFX9Plus())
2100 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2104bool AMDGPUOperand::isSDWAFP16Operand()
const {
2105 return isSDWAOperand(MVT::f16);
2108bool AMDGPUOperand::isSDWAFP32Operand()
const {
2109 return isSDWAOperand(MVT::f32);
2112bool AMDGPUOperand::isSDWAInt16Operand()
const {
2113 return isSDWAOperand(MVT::i16);
2116bool AMDGPUOperand::isSDWAInt32Operand()
const {
2117 return isSDWAOperand(MVT::i32);
2120bool AMDGPUOperand::isBoolReg()
const {
2121 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2122 (AsmParser->isWave32() && isSCSrc_b32()));
2126 unsigned Size)
const {
2127 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2142void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2143 bool ApplyModifiers)
const {
2153 addLiteralImmOperand(Inst,
Imm.Val,
2154 ApplyModifiers & isImmTy(ImmTyNone) &&
2155 Imm.Mods.hasFPModifiers());
2157 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2162void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2163 bool ApplyModifiers)
const {
2164 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2169 if (ApplyModifiers) {
2171 const unsigned Size =
2173 Val = applyInputFPModifiers(Val,
Size);
2177 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2179 bool CanUse64BitLiterals =
2182 MCContext &Ctx = AsmParser->getContext();
2193 if (
Lit == LitModifier::None &&
2195 AsmParser->hasInv2PiInlineImm())) {
2203 bool HasMandatoryLiteral =
2206 if (
Literal.getLoBits(32) != 0 &&
2207 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2208 !HasMandatoryLiteral) {
2209 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2211 "Can't encode literal as exact 64-bit floating-point operand. "
2212 "Low 32-bits will be set to zero");
2213 Val &= 0xffffffff00000000u;
2219 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2225 Lit = LitModifier::Lit64;
2226 }
else if (
Lit == LitModifier::Lit) {
2240 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2242 Lit = LitModifier::Lit64;
2249 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2250 Literal == 0x3fc45f306725feed) {
2290 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2297 if (
Lit != LitModifier::None) {
2328 if (
Lit == LitModifier::None &&
2338 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2346 if (
Lit == LitModifier::None &&
2354 if (!AsmParser->has64BitLiterals()) {
2355 Val =
static_cast<uint64_t>(Val) << 32;
2362 if (
Lit == LitModifier::Lit ||
2364 Val =
static_cast<uint64_t>(Val) << 32;
2368 if (
Lit == LitModifier::Lit)
2395 if (
Lit != LitModifier::None) {
2403void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2409bool AMDGPUOperand::isInlineValue()
const {
2417void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2428 if (Is == IS_VGPR) {
2433 return AMDGPU::VGPR_32RegClassID;
2435 return AMDGPU::VReg_64RegClassID;
2437 return AMDGPU::VReg_96RegClassID;
2439 return AMDGPU::VReg_128RegClassID;
2441 return AMDGPU::VReg_160RegClassID;
2443 return AMDGPU::VReg_192RegClassID;
2445 return AMDGPU::VReg_224RegClassID;
2447 return AMDGPU::VReg_256RegClassID;
2449 return AMDGPU::VReg_288RegClassID;
2451 return AMDGPU::VReg_320RegClassID;
2453 return AMDGPU::VReg_352RegClassID;
2455 return AMDGPU::VReg_384RegClassID;
2457 return AMDGPU::VReg_512RegClassID;
2459 return AMDGPU::VReg_1024RegClassID;
2461 }
else if (Is == IS_TTMP) {
2466 return AMDGPU::TTMP_32RegClassID;
2468 return AMDGPU::TTMP_64RegClassID;
2470 return AMDGPU::TTMP_128RegClassID;
2472 return AMDGPU::TTMP_256RegClassID;
2474 return AMDGPU::TTMP_512RegClassID;
2476 }
else if (Is == IS_SGPR) {
2481 return AMDGPU::SGPR_32RegClassID;
2483 return AMDGPU::SGPR_64RegClassID;
2485 return AMDGPU::SGPR_96RegClassID;
2487 return AMDGPU::SGPR_128RegClassID;
2489 return AMDGPU::SGPR_160RegClassID;
2491 return AMDGPU::SGPR_192RegClassID;
2493 return AMDGPU::SGPR_224RegClassID;
2495 return AMDGPU::SGPR_256RegClassID;
2497 return AMDGPU::SGPR_288RegClassID;
2499 return AMDGPU::SGPR_320RegClassID;
2501 return AMDGPU::SGPR_352RegClassID;
2503 return AMDGPU::SGPR_384RegClassID;
2505 return AMDGPU::SGPR_512RegClassID;
2507 }
else if (Is == IS_AGPR) {
2512 return AMDGPU::AGPR_32RegClassID;
2514 return AMDGPU::AReg_64RegClassID;
2516 return AMDGPU::AReg_96RegClassID;
2518 return AMDGPU::AReg_128RegClassID;
2520 return AMDGPU::AReg_160RegClassID;
2522 return AMDGPU::AReg_192RegClassID;
2524 return AMDGPU::AReg_224RegClassID;
2526 return AMDGPU::AReg_256RegClassID;
2528 return AMDGPU::AReg_288RegClassID;
2530 return AMDGPU::AReg_320RegClassID;
2532 return AMDGPU::AReg_352RegClassID;
2534 return AMDGPU::AReg_384RegClassID;
2536 return AMDGPU::AReg_512RegClassID;
2538 return AMDGPU::AReg_1024RegClassID;
2546 .
Case(
"exec", AMDGPU::EXEC)
2547 .
Case(
"vcc", AMDGPU::VCC)
2548 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2549 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2550 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2551 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2552 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2553 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2554 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2555 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2556 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2557 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2558 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2559 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2560 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2561 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2562 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2563 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2564 .
Case(
"m0", AMDGPU::M0)
2565 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2566 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2567 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2568 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2569 .
Case(
"scc", AMDGPU::SRC_SCC)
2570 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2571 .
Case(
"tba", AMDGPU::TBA)
2572 .
Case(
"tma", AMDGPU::TMA)
2573 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2574 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2575 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2576 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2577 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2578 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2579 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2580 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2581 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2582 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2583 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2584 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2585 .
Case(
"pc", AMDGPU::PC_REG)
2586 .
Case(
"null", AMDGPU::SGPR_NULL)
2590bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2591 SMLoc &EndLoc,
bool RestoreOnFailure) {
2592 auto R = parseRegister();
2596 RegNo =
R->getReg();
2597 StartLoc =
R->getStartLoc();
2598 EndLoc =
R->getEndLoc();
2602bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2604 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2607ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2609 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2610 bool PendingErrors = getParser().hasPendingError();
2611 getParser().clearPendingErrors();
2619bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2620 RegisterKind RegKind,
2622 RegisterKind RegKind1, SMLoc Loc) {
2624 if (RegKind == IS_SGPR) {
2625 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2626 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2627 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2633 if (RegKind != RegKind1) {
2634 Error(Loc,
"registers in a list must be of the same kind");
2640 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2645 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2646 Reg = AMDGPU::FLAT_SCR;
2650 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2651 Reg = AMDGPU::XNACK_MASK;
2655 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2660 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2665 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2670 Error(Loc,
"register does not fit in the list");
2676 if (Reg1 !=
Reg + RegWidth / 32) {
2677 Error(Loc,
"registers in a list must have consecutive indices");
2693 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2694 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2698 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2704 if (Str.starts_with(
Reg.Name))
2710 return !Str.getAsInteger(10, Num);
2713bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2714 const AsmToken &NextToken)
const {
2729 StringRef RegSuffix = Str.substr(
RegName.size());
2730 if (!RegSuffix.
empty()) {
2747bool AMDGPUAsmParser::isRegister() {
2748 return isRegister(
getToken(), peekToken());
2751MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2752 unsigned SubReg,
unsigned RegWidth,
2756 unsigned AlignSize = 1;
2757 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2763 if (RegNum % AlignSize != 0) {
2764 Error(Loc,
"invalid register alignment");
2765 return MCRegister();
2768 unsigned RegIdx = RegNum / AlignSize;
2771 Error(Loc,
"invalid or unsupported register size");
2772 return MCRegister();
2776 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2777 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2778 Error(Loc,
"register index is out of range");
2779 return AMDGPU::NoRegister;
2782 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2783 Error(Loc,
"register index is out of range");
2784 return MCRegister();
2793 Error(Loc,
"invalid subregister");
2799bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2801 int64_t RegLo, RegHi;
2805 SMLoc FirstIdxLoc = getLoc();
2812 SecondIdxLoc = getLoc();
2823 Error(FirstIdxLoc,
"invalid register index");
2828 Error(SecondIdxLoc,
"invalid register index");
2832 if (RegLo > RegHi) {
2833 Error(FirstIdxLoc,
"first register index should not exceed second index");
2837 if (RegHi == RegLo) {
2838 StringRef RegSuffix = getTokenStr();
2839 if (RegSuffix ==
".l") {
2840 SubReg = AMDGPU::lo16;
2842 }
else if (RegSuffix ==
".h") {
2843 SubReg = AMDGPU::hi16;
2848 Num =
static_cast<unsigned>(RegLo);
2849 RegWidth = 32 * ((RegHi - RegLo) + 1);
2854MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2857 SmallVectorImpl<AsmToken> &Tokens) {
2863 RegKind = IS_SPECIAL;
2870MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2873 SmallVectorImpl<AsmToken> &Tokens) {
2875 StringRef
RegName = getTokenStr();
2876 auto Loc = getLoc();
2880 Error(Loc,
"invalid register name");
2881 return MCRegister();
2889 unsigned SubReg = NoSubRegister;
2890 bool IsRange =
false;
2891 if (!RegSuffix.
empty()) {
2893 SubReg = AMDGPU::lo16;
2895 SubReg = AMDGPU::hi16;
2899 Error(Loc,
"invalid register index");
2900 return MCRegister();
2906 if (!ParseRegRange(RegNum, RegWidth, SubReg))
2907 return MCRegister();
2911 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
2912 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
2913 if (RegKind == IS_SGPR && IsRange
2914 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
2915 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
2916 Error(Loc,
"register index is out of range");
2917 return MCRegister();
2923MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
2924 unsigned &RegNum,
unsigned &RegWidth,
2925 SmallVectorImpl<AsmToken> &Tokens) {
2927 auto ListLoc = getLoc();
2930 "expected a register or a list of registers")) {
2931 return MCRegister();
2936 auto Loc = getLoc();
2937 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
2938 return MCRegister();
2939 if (RegWidth != 32) {
2940 Error(Loc,
"expected a single 32-bit register");
2941 return MCRegister();
2945 RegisterKind NextRegKind;
2947 unsigned NextRegNum, NextRegWidth;
2950 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
2952 return MCRegister();
2954 if (NextRegWidth != 32) {
2955 Error(Loc,
"expected a single 32-bit register");
2956 return MCRegister();
2958 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
2960 return MCRegister();
2964 "expected a comma or a closing square bracket")) {
2965 return MCRegister();
2969 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
2974bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
2975 MCRegister &
Reg,
unsigned &RegNum,
2977 SmallVectorImpl<AsmToken> &Tokens) {
2978 auto Loc = getLoc();
2982 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
2984 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
2986 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
2991 assert(Parser.hasPendingError());
2995 if (!subtargetHasRegister(*
TRI,
Reg)) {
2996 if (
Reg == AMDGPU::SGPR_NULL) {
2997 Error(Loc,
"'null' operand is not supported on this GPU");
3000 " register not available on this GPU");
3008bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3009 MCRegister &
Reg,
unsigned &RegNum,
3011 bool RestoreOnFailure ) {
3015 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3016 if (RestoreOnFailure) {
3017 while (!Tokens.
empty()) {
3026std::optional<StringRef>
3027AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3030 return StringRef(
".amdgcn.next_free_vgpr");
3032 return StringRef(
".amdgcn.next_free_sgpr");
3034 return std::nullopt;
3038void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3039 auto SymbolName = getGprCountSymbolName(RegKind);
3040 assert(SymbolName &&
"initializing invalid register kind");
3046bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3047 unsigned DwordRegIndex,
3048 unsigned RegWidth) {
3053 auto SymbolName = getGprCountSymbolName(RegKind);
3058 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3062 return !
Error(getLoc(),
3063 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3067 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3069 if (OldCount <= NewMax)
3075std::unique_ptr<AMDGPUOperand>
3076AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3078 SMLoc StartLoc = Tok.getLoc();
3079 SMLoc EndLoc = Tok.getEndLoc();
3080 RegisterKind RegKind;
3082 unsigned RegNum, RegWidth;
3084 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3088 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3091 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3092 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3099 if (isRegister() || isModifier())
3102 if (
Lit == LitModifier::None) {
3103 if (trySkipId(
"lit"))
3104 Lit = LitModifier::Lit;
3105 else if (trySkipId(
"lit64"))
3106 Lit = LitModifier::Lit64;
3108 if (
Lit != LitModifier::None) {
3111 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3120 const auto &NextTok = peekToken();
3123 bool Negate =
false;
3131 AMDGPUOperand::Modifiers Mods;
3139 StringRef Num = getTokenStr();
3142 APFloat RealVal(APFloat::IEEEdouble());
3143 auto roundMode = APFloat::rmNearestTiesToEven;
3144 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3147 RealVal.changeSign();
3150 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3151 S, AMDGPUOperand::ImmTyNone,
true));
3152 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3153 Op.setModifiers(Mods);
3162 if (HasSP3AbsModifier) {
3171 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3174 if (Parser.parseExpression(Expr))
3178 if (Expr->evaluateAsAbsolute(IntVal)) {
3180 return Error(S,
"literal value out of range");
3181 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3182 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3183 Op.setModifiers(Mods);
3185 if (
Lit != LitModifier::None)
3187 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3200 if (
auto R = parseRegister()) {
3210 ParseStatus Res = parseReg(
Operands);
3218bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3219 const AsmToken &NextToken)
const {
3222 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3227bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3228 const AsmToken &NextToken)
const {
3232bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3233 const AsmToken &NextToken)
const {
3234 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3237bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3238 const AsmToken &NextToken)
const {
3239 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3256bool AMDGPUAsmParser::isModifier() {
3259 AsmToken NextToken[2];
3260 peekTokens(NextToken);
3264 bool IsOpcodeModifier = isOpcodeModifierWithVal(Tok, NextToken[0]) &&
3267 return isOperandModifier(Tok, NextToken[0]) ||
3269 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3295bool AMDGPUAsmParser::parseSP3NegModifier() {
3297 AsmToken NextToken[2];
3298 peekTokens(NextToken);
3301 (isRegister(NextToken[0], NextToken[1]) ||
3319 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3321 SP3Neg = parseSP3NegModifier();
3324 Neg = trySkipId(
"neg");
3326 return Error(Loc,
"expected register or immediate");
3330 Abs = trySkipId(
"abs");
3335 if (trySkipId(
"lit")) {
3336 Lit = LitModifier::Lit;
3339 }
else if (trySkipId(
"lit64")) {
3340 Lit = LitModifier::Lit64;
3343 if (!has64BitLiterals())
3344 return Error(Loc,
"lit64 is not supported on this GPU");
3350 return Error(Loc,
"expected register or immediate");
3359 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3363 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3364 Error(Loc,
"expected immediate with lit modifier");
3366 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3372 if (
Lit != LitModifier::None &&
3376 AMDGPUOperand::Modifiers Mods;
3377 Mods.Abs = Abs || SP3Abs;
3378 Mods.Neg = Neg || SP3Neg;
3381 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3382 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3384 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3385 Op.setModifiers(Mods);
3393 bool Sext = trySkipId(
"sext");
3394 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3409 AMDGPUOperand::Modifiers Mods;
3412 if (Mods.hasIntModifiers()) {
3413 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3415 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3416 Op.setModifiers(Mods);
3423 return parseRegOrImmWithFPInputMods(
Operands,
false);
3427 return parseRegOrImmWithIntInputMods(
Operands,
false);
3434 if (!trySkipId(
"rsrcidx"))
3440 SMLoc RegLoc = getLoc();
3441 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3449 if (!
Reg->isRsrcReg32())
3450 return Error(RegLoc,
"rsrcidx operand must be a 32-bit SGPR or VGPR");
3460 auto Loc = getLoc();
3461 if (trySkipId(
"off")) {
3463 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3470 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3479unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3484 return Match_InvalidOperand;
3486 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3487 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3490 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3492 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3493 return Match_InvalidOperand;
3501 if (tryAnotherVOPDEncoding(Inst))
3502 return Match_InvalidOperand;
3504 return Match_Success;
3508 static const unsigned Variants[] = {
3517ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3518 if (isForcedDPP() && isForcedVOP3()) {
3522 if (getForcedEncodingSize() == 32) {
3527 if (isForcedVOP3()) {
3532 if (isForcedSDWA()) {
3538 if (isForcedDPP()) {
3546StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3547 if (isForcedDPP() && isForcedVOP3())
3550 if (getForcedEncodingSize() == 32)
3566AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3570 case AMDGPU::FLAT_SCR:
3572 case AMDGPU::VCC_LO:
3573 case AMDGPU::VCC_HI:
3580 return MCRegister();
3587bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3588 unsigned OpIdx)
const {
3596 const MCOperand &MO = Inst.
getOperand(OpIdx);
3646unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3652 case AMDGPU::V_LSHLREV_B64_e64:
3653 case AMDGPU::V_LSHLREV_B64_gfx10:
3654 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3655 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3656 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3657 case AMDGPU::V_LSHRREV_B64_e64:
3658 case AMDGPU::V_LSHRREV_B64_gfx10:
3659 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3660 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3661 case AMDGPU::V_ASHRREV_I64_e64:
3662 case AMDGPU::V_ASHRREV_I64_gfx10:
3663 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3664 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3665 case AMDGPU::V_LSHL_B64_e64:
3666 case AMDGPU::V_LSHR_B64_e64:
3667 case AMDGPU::V_ASHR_I64_e64:
3680 bool AddMandatoryLiterals =
false) {
3683 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3687 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3689 return {getNamedOperandIdx(Opcode, OpName::src0X),
3690 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3691 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3692 getNamedOperandIdx(Opcode, OpName::src0Y),
3693 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3694 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3699 return {getNamedOperandIdx(Opcode, OpName::src0),
3700 getNamedOperandIdx(Opcode, OpName::src1),
3701 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3704bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3705 const MCOperand &MO = Inst.
getOperand(OpIdx);
3707 return !isInlineConstant(Inst, OpIdx);
3714 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3725 const unsigned Opcode = Inst.
getOpcode();
3726 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3729 if (!LaneSelOp.
isReg())
3732 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3735bool AMDGPUAsmParser::validateConstantBusLimitations(
3737 const unsigned Opcode = Inst.
getOpcode();
3738 const MCInstrDesc &
Desc = MII.
get(Opcode);
3739 MCRegister LastSGPR;
3740 unsigned ConstantBusUseCount = 0;
3741 unsigned NumLiterals = 0;
3742 unsigned LiteralSize;
3758 SmallDenseSet<MCRegister> SGPRsUsed;
3759 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3761 SGPRsUsed.
insert(SGPRUsed);
3762 ++ConstantBusUseCount;
3767 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3769 for (
int OpIdx : OpIndices) {
3773 const MCOperand &MO = Inst.
getOperand(OpIdx);
3774 if (usesConstantBus(Inst, OpIdx)) {
3783 if (SGPRsUsed.
insert(LastSGPR).second) {
3784 ++ConstantBusUseCount;
3804 if (NumLiterals == 0) {
3807 }
else if (LiteralSize !=
Size) {
3813 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3815 "invalid operand (violates constant bus restrictions)");
3822std::optional<unsigned>
3823AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3825 const unsigned Opcode = Inst.
getOpcode();
3831 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3832 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3841 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3842 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3843 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3844 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3845 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3846 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3850 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3851 int I = getNamedOperandIdx(Opcode, OpName);
3855 int64_t
Imm =
Op.getImm();
3861 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
3862 OpName::vsrc2Y, OpName::imm}) {
3863 int I = getNamedOperandIdx(Opcode, OpName);
3873 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
3874 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
3876 return InvalidCompOprIdx;
3879bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
3886 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
3887 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
3888 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
3890 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
3894 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
3895 if (!InvalidCompOprIdx.has_value())
3898 auto CompOprIdx = *InvalidCompOprIdx;
3901 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
3902 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
3905 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
3906 if (CompOprIdx == VOPD::Component::DST) {
3908 Error(Loc,
"dst registers must be distinct");
3910 Error(Loc,
"one dst register must be even and the other odd");
3912 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
3913 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
3914 " operands must use different VGPR banks");
3922bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
3924 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
3925 if (!InvalidCompOprIdx.has_value())
3929 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
3930 if (InvalidCompOprIdx.has_value()) {
3935 if (*InvalidCompOprIdx == VOPD::Component::DST)
3948bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
3949 const unsigned Opcode = Inst.
getOpcode();
3964 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
3965 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
3966 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
3967 int I = getNamedOperandIdx(Opcode, OpName);
3974 return !tryVOPD3(Inst);
3979bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
3984 return tryVOPD(Inst);
3985 return tryVOPD3(Inst);
3988bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
3993 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4001bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4009 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4010 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4011 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4016 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4017 hasBVHRayTracingInsts())
4020 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4021 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4026 bool IsPackedD16 =
false;
4029 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4030 IsPackedD16 = D16Idx >= 0;
4032 DataSize = (DataSize + 1) / 2;
4035 if ((VDataSize / 4) == DataSize + TFESize)
4040 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4042 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4044 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4048bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4057 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4059 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4060 AMDGPU::OpName RSrcOpName =
4062 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4063 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4064 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4068 assert(SrsrcIdx > VAddr0Idx);
4071 if (BaseOpcode->
BVH) {
4072 if (IsA16 == BaseOpcode->
A16)
4074 Error(IDLoc,
"image address size does not match a16");
4080 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4081 unsigned ActualAddrSize =
4082 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4084 unsigned ExpectedAddrSize =
4088 if (hasPartialNSAEncoding() &&
4090 int VAddrLastIdx = SrsrcIdx - 1;
4091 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4093 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4096 if (ExpectedAddrSize > 12)
4097 ExpectedAddrSize = 16;
4102 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4106 if (ActualAddrSize == ExpectedAddrSize)
4109 Error(IDLoc,
"image address size does not match dim and a16");
4113bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4120 if (!
Desc.mayLoad() || !
Desc.mayStore())
4123 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4130 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4133bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4140 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4148 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4151bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4165 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4166 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4173bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4180 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4183 if (!BaseOpcode->
MSAA)
4186 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4192 return DimInfo->
MSAA;
4197 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4198 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4199 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4209bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4217 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4228 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4232bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4237 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4240 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4251 "source operand must be either a VGPR or an inline constant");
4258bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4263 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4266 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4270 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4272 "inline constants are not allowed for this operand");
4279bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4287 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4288 if (BlgpIdx != -1) {
4289 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4290 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4300 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4302 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4307 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4309 "wrong register tuple size for blgp value " + Twine(BLGP));
4317 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4327 if (Src2Reg == DstReg)
4332 .getSizeInBits() <= 128)
4335 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4337 "source 2 operand must not partially overlap with dst");
4344bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4348 case V_DIV_SCALE_F32_gfx6_gfx7:
4349 case V_DIV_SCALE_F32_vi:
4350 case V_DIV_SCALE_F32_gfx10:
4351 case V_DIV_SCALE_F64_gfx6_gfx7:
4352 case V_DIV_SCALE_F64_vi:
4353 case V_DIV_SCALE_F64_gfx10:
4360 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4361 AMDGPU::OpName::src2_modifiers}) {
4372bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4379 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4388bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4394 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4401 case AMDGPU::V_SUBREV_F32_e32:
4402 case AMDGPU::V_SUBREV_F32_e64:
4403 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4404 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4405 case AMDGPU::V_SUBREV_F32_e32_vi:
4406 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4407 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4408 case AMDGPU::V_SUBREV_F32_e64_vi:
4410 case AMDGPU::V_SUBREV_CO_U32_e32:
4411 case AMDGPU::V_SUBREV_CO_U32_e64:
4412 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4413 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4415 case AMDGPU::V_SUBBREV_U32_e32:
4416 case AMDGPU::V_SUBBREV_U32_e64:
4417 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4418 case AMDGPU::V_SUBBREV_U32_e32_vi:
4419 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4420 case AMDGPU::V_SUBBREV_U32_e64_vi:
4422 case AMDGPU::V_SUBREV_U32_e32:
4423 case AMDGPU::V_SUBREV_U32_e64:
4424 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4425 case AMDGPU::V_SUBREV_U32_e32_vi:
4426 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4427 case AMDGPU::V_SUBREV_U32_e64_vi:
4429 case AMDGPU::V_SUBREV_F16_e32:
4430 case AMDGPU::V_SUBREV_F16_e64:
4431 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4432 case AMDGPU::V_SUBREV_F16_e32_vi:
4433 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4434 case AMDGPU::V_SUBREV_F16_e64_vi:
4436 case AMDGPU::V_SUBREV_U16_e32:
4437 case AMDGPU::V_SUBREV_U16_e64:
4438 case AMDGPU::V_SUBREV_U16_e32_vi:
4439 case AMDGPU::V_SUBREV_U16_e64_vi:
4441 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4442 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4443 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4445 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4446 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4448 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4449 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4451 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4452 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4454 case AMDGPU::V_LSHRREV_B32_e32:
4455 case AMDGPU::V_LSHRREV_B32_e64:
4456 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4457 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4458 case AMDGPU::V_LSHRREV_B32_e32_vi:
4459 case AMDGPU::V_LSHRREV_B32_e64_vi:
4460 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4461 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4463 case AMDGPU::V_ASHRREV_I32_e32:
4464 case AMDGPU::V_ASHRREV_I32_e64:
4465 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4466 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4467 case AMDGPU::V_ASHRREV_I32_e32_vi:
4468 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4469 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4470 case AMDGPU::V_ASHRREV_I32_e64_vi:
4472 case AMDGPU::V_LSHLREV_B32_e32:
4473 case AMDGPU::V_LSHLREV_B32_e64:
4474 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4475 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4476 case AMDGPU::V_LSHLREV_B32_e32_vi:
4477 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4478 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4479 case AMDGPU::V_LSHLREV_B32_e64_vi:
4481 case AMDGPU::V_LSHLREV_B16_e32:
4482 case AMDGPU::V_LSHLREV_B16_e64:
4483 case AMDGPU::V_LSHLREV_B16_e32_vi:
4484 case AMDGPU::V_LSHLREV_B16_e64_vi:
4485 case AMDGPU::V_LSHLREV_B16_gfx10:
4487 case AMDGPU::V_LSHRREV_B16_e32:
4488 case AMDGPU::V_LSHRREV_B16_e64:
4489 case AMDGPU::V_LSHRREV_B16_e32_vi:
4490 case AMDGPU::V_LSHRREV_B16_e64_vi:
4491 case AMDGPU::V_LSHRREV_B16_gfx10:
4493 case AMDGPU::V_ASHRREV_I16_e32:
4494 case AMDGPU::V_ASHRREV_I16_e64:
4495 case AMDGPU::V_ASHRREV_I16_e32_vi:
4496 case AMDGPU::V_ASHRREV_I16_e64_vi:
4497 case AMDGPU::V_ASHRREV_I16_gfx10:
4499 case AMDGPU::V_LSHLREV_B64_e64:
4500 case AMDGPU::V_LSHLREV_B64_gfx10:
4501 case AMDGPU::V_LSHLREV_B64_vi:
4503 case AMDGPU::V_LSHRREV_B64_e64:
4504 case AMDGPU::V_LSHRREV_B64_gfx10:
4505 case AMDGPU::V_LSHRREV_B64_vi:
4507 case AMDGPU::V_ASHRREV_I64_e64:
4508 case AMDGPU::V_ASHRREV_I64_gfx10:
4509 case AMDGPU::V_ASHRREV_I64_vi:
4511 case AMDGPU::V_PK_LSHLREV_B16:
4512 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4513 case AMDGPU::V_PK_LSHLREV_B16_vi:
4515 case AMDGPU::V_PK_LSHRREV_B16:
4516 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4517 case AMDGPU::V_PK_LSHRREV_B16_vi:
4518 case AMDGPU::V_PK_ASHRREV_I16:
4519 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4520 case AMDGPU::V_PK_ASHRREV_I16_vi:
4527bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4529 const unsigned Opcode = Inst.
getOpcode();
4538 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4539 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4543 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4547 "lds_direct is not supported on this GPU");
4553 "lds_direct cannot be used with this instruction");
4557 if (SrcName != OpName::src0) {
4559 "lds_direct may be used as src0 only");
4569 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4570 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4571 if (
Op.isFlatOffset())
4572 return Op.getStartLoc();
4577bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4580 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4585 return validateFlatOffset(Inst,
Operands);
4588 return validateSMEMOffset(Inst,
Operands);
4593 const unsigned OffsetSize = 24;
4594 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4596 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4597 "-bit unsigned offset for buffer ops");
4601 const unsigned OffsetSize = 16;
4602 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4604 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4611bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4617 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4621 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4623 "flat offset modifier is not supported on this GPU");
4630 bool AllowNegative =
4632 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4634 Twine(
"expected a ") +
4635 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4636 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4645 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4646 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4647 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4648 return Op.getStartLoc();
4653bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4662 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4678 ?
"expected a 23-bit unsigned offset for buffer ops"
4679 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4680 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4681 :
"expected a 21-bit signed offset");
4687bool AMDGPUAsmParser::validateSMEMSData(
const MCInst &Inst,
4693 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::sdata);
4698 Error(getOperandLoc(
Operands, OpNum),
"expected a 5-bit unsigned immediate");
4707bool AMDGPUAsmParser::validateBF16InlineConst(
const MCInst &Inst,
4709 if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32])
4724 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4728 const MCOperandInfo &Src0Info =
Desc.operands()[Src0Idx];
4733 if (!
Src0.isImm() ||
4735 hasInv2PiInlineImm()))
4740 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
4741 if (ModsIdx != -1 &&
4747 "bf16 inline constant is read from the high half of the fp32 inline "
4748 "constant on this GPU; use the e64 encoding with op_sel:[1,0]");
4752bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4755 const MCInstrDesc &
Desc = MII.
get(Opcode);
4759 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4760 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4762 const int OpIndices[] = {Src0Idx, Src1Idx};
4764 unsigned NumExprs = 0;
4765 unsigned NumLiterals = 0;
4768 for (
int OpIdx : OpIndices) {
4772 const MCOperand &MO = Inst.
getOperand(OpIdx);
4776 std::optional<int64_t>
Imm;
4779 }
else if (MO.
isExpr()) {
4788 if (!
Imm.has_value()) {
4790 }
else if (!isInlineConstant(Inst, OpIdx)) {
4794 if (NumLiterals == 0 || LiteralValue !=
Value) {
4802 if (NumLiterals + NumExprs <= 1)
4806 "only one unique literal operand is allowed");
4810bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4813 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4821 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4822 if (OpSelIdx != -1) {
4826 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4827 if (OpSelHiIdx != -1) {
4836 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4843 if (getFeatureBits()[AMDGPU::FeatureGFX950Insts] &&
4844 (
Opc == AMDGPU::V_PK_ADD_F32_vi ||
Opc == AMDGPU::V_PK_MUL_F32_vi ||
4845 Opc == AMDGPU::V_PK_FMA_F32_vi)) {
4846 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4850 if (
Opc == AMDGPU::V_PK_FMA_F32_vi &&
4851 (((OpSel >> 1) & 1) || ((OpSel >> 2) & 1)))
4854 else if ((OpSel & 3) == 2)
4862 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4863 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4864 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4865 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4874 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4876 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4886 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4887 if (Src2Idx != -1) {
4898bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4899 if (!hasTrue16Insts())
4901 const MCRegisterInfo *
MRI = getMRI();
4903 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4909 if (OpSelOpValue == 0)
4911 unsigned OpCount = 0;
4912 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4913 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4914 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4921 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4922 if (OpSelOpIsHi != VGPRSuffixIsHi)
4931bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4932 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
4944 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
4955 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
4956 AMDGPU::OpName::src1_modifiers,
4957 AMDGPU::OpName::src2_modifiers};
4959 for (
unsigned i = 0; i < 3; ++i) {
4969bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
4972 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
4973 if (DppCtrlIdx >= 0) {
4977 getSTI().
hasFeature(AMDGPU::FeatureDPALU_DPP) &&
4981 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
4982 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
4983 :
"DP ALU dpp only supports row_newbcast");
4988 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
4989 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
4992 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4998 "invalid operand for instruction");
5003 "src1 immediate operand invalid for instruction");
5013bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5014 return (
Reg == AMDGPU::VCC && isWave64()) ||
5015 (
Reg == AMDGPU::VCC_LO && isWave32());
5019bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5022 const MCInstrDesc &
Desc = MII.
get(Opcode);
5023 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5030 std::optional<unsigned> LiteralOpIdx;
5033 for (
int OpIdx : OpIndices) {
5037 const MCOperand &MO = Inst.
getOperand(OpIdx);
5043 std::optional<int64_t>
Imm;
5049 bool IsAnotherLiteral =
false;
5050 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5051 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5052 if (!
Imm.has_value()) {
5054 IsAnotherLiteral =
true;
5055 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5060 HasMandatoryLiteral);
5072 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5073 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5075 "invalid operand for instruction");
5080 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5081 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5083 "invalid operand for instruction");
5088 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5089 Value =
static_cast<uint32_t
>(
5097 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5098 !getFeatureBits()[FeatureVOP3Literal]) {
5100 "literal operands are not supported");
5104 if (LiteralOpIdx && IsAnotherLiteral) {
5106 getOperandLoc(
Operands, *LiteralOpIdx)),
5107 "only one unique literal operand is allowed");
5111 if (IsAnotherLiteral)
5112 LiteralOpIdx = OpIdx;
5121 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5135bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5141 ? AMDGPU::OpName::data0
5142 : AMDGPU::OpName::vdata;
5144 const MCRegisterInfo *
MRI = getMRI();
5150 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5154 auto FB = getFeatureBits();
5155 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5156 if (DataAreg < 0 || DstAreg < 0)
5158 return DstAreg == DataAreg;
5161 return DstAreg < 1 && DataAreg < 1;
5164bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5165 auto FB = getFeatureBits();
5166 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5170 const MCRegisterInfo *
MRI = getMRI();
5173 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5176 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5180 case AMDGPU::DS_LOAD_TR6_B96:
5181 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5185 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5186 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5190 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5191 if (VAddrIdx != -1) {
5193 MCRegister
Sub =
MRI->getSubReg(
Op.getReg(), AMDGPU::sub0);
5194 if ((
Sub - AMDGPU::VGPR0) & 1)
5199 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5200 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5205 const MCRegisterClass &VGPR32 =
MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5206 const MCRegisterClass &AGPR32 =
MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5212 MCRegister
Sub =
MRI->getSubReg(
Op.getReg(), AMDGPU::sub0);
5226 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5227 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5229 return Op.getStartLoc();
5234bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5237 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5240 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5243 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5244 auto FB = getFeatureBits();
5245 bool UsesNeg =
false;
5246 if (FB[AMDGPU::FeatureGFX940Insts]) {
5248 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5249 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5250 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5251 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5256 if (IsNeg == UsesNeg)
5259 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5260 :
"invalid modifier: neg is not supported");
5265bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5271 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5272 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5273 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5274 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5277 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5280 if (
Reg == AMDGPU::SGPR_NULL)
5283 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5287bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5292 return validateGWS(Inst,
Operands);
5297 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5302 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5303 Error(S,
"gds modifier is not supported on this GPU");
5311bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5313 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5317 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5318 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5321 const MCRegisterInfo *
MRI = getMRI();
5322 const MCRegisterClass &VGPR32 =
MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5324 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5327 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5329 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5336bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5340 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5347 if (!hasCacheFillSize()) {
5348 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5351 Error(S,
"Cache fill size is not supported on this GPU");
5352 }
else if (!validateCFSBits(Inst,
Operands)) {
5357 if (!hasScaleOffset() && (CPol &
CPol::SCAL)) {
5358 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5361 Error(S,
"scale_offset is not supported on this GPU");
5365 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5368 Error(S,
"nv is not supported on this GPU");
5372 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5375 Error(S,
"scale_offset is not supported for this instruction");
5379 return validateTHAndScopeBits(Inst,
Operands, CPol);
5383 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5384 Error(S,
"cache policy is not supported for SMRD instructions");
5388 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5395 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5399 "scc modifier is not supported for this instruction on this GPU");
5410 :
"instruction must use glc");
5415 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5418 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5420 :
"instruction must not use glc");
5428bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5430 const unsigned CPol) {
5435 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5442 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5447 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5455 return PrintError(
"invalid th value for SMEM instruction");
5462 return PrintError(
"scope and th combination is not valid");
5468 return PrintError(
"invalid th value for atomic instructions");
5471 return PrintError(
"invalid th value for store instructions");
5474 return PrintError(
"invalid th value for load instructions");
5480bool AMDGPUAsmParser::validateCFSBits(
const MCInst &Inst,
5482 const unsigned Opcode = Inst.
getOpcode();
5483 const MCInstrDesc &
Desc = MII.
get(Opcode);
5486 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5494 return PrintError(
"Cache fill size is only supported for VC operations");
5497bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5501 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5503 Error(Loc,
"TFE modifier has no meaning for store instructions");
5511bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5517 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5521 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5524 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5525 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5534 "wrong register tuple size for " +
5539 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5540 !validateFmt(BFmt, AMDGPU::OpName::src1))
5544 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5545 if (AScaleIdx == -1)
5549 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5553 "invalid matrix and scale format combination");
5560bool AMDGPUAsmParser::validateMonitorSleep(
const MCInst &Inst,
5563 if (
Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5564 !getSTI().
hasFeature(AMDGPU::FeatureNoSleepForever))
5567 int ImmIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::simm16);
5570 "sleep forever is unsupported on the target");
5577bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5580 if (
Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5581 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5584 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
5591 "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5595bool AMDGPUAsmParser::validateScaleSel(
const MCInst &Inst,
5598 int ScaleSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::scale_sel);
5599 if (ScaleSelIdx == -1)
5603 case AMDGPU::V_CVT_SCALE_PK16_F16_FP6_e64_gfx1250:
5604 case AMDGPU::V_CVT_SCALE_PK16_BF16_FP6_e64_gfx1250:
5605 case AMDGPU::V_CVT_SCALE_PK16_F16_BF6_e64_gfx1250:
5606 case AMDGPU::V_CVT_SCALE_PK16_BF16_BF6_e64_gfx1250:
5607 case AMDGPU::V_CVT_SCALE_PK16_F32_FP6_e64_gfx1250:
5608 case AMDGPU::V_CVT_SCALE_PK16_F32_BF6_e64_gfx1250:
5609 case AMDGPU::V_CVT_SCALE_PK8_F16_FP4_e64_gfx1250:
5610 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP4_e64_gfx1250:
5611 case AMDGPU::V_CVT_SCALE_PK8_F32_FP4_e64_gfx1250:
5614 case AMDGPU::V_CVT_SCALE_PK8_F16_FP8_e64_gfx1250:
5615 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP8_e64_gfx1250:
5616 case AMDGPU::V_CVT_SCALE_PK8_F16_BF8_e64_gfx1250:
5617 case AMDGPU::V_CVT_SCALE_PK8_BF16_BF8_e64_gfx1250:
5618 case AMDGPU::V_CVT_SCALE_PK8_F32_FP8_e64_gfx1250:
5619 case AMDGPU::V_CVT_SCALE_PK8_F32_BF8_e64_gfx1250:
5626 if (getSTI().
hasFeature(AMDGPU::FeatureBlock16ConversionScaleInsts))
5630 if (ScaleSel < MaxSel)
5634 "scale_sel maximum supported value is " + Twine(MaxSel - 1));
5638bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5640 if (!validateLdsDirect(Inst,
Operands))
5642 if (!validateTrue16OpSel(Inst)) {
5644 "op_sel operand conflicts with 16-bit operand suffix");
5647 if (!validateSOPLiteral(Inst,
Operands))
5649 if (!validateVOPLiteral(Inst,
Operands)) {
5652 if (!validateConstantBusLimitations(Inst,
Operands)) {
5655 if (!validateVOPD(Inst,
Operands)) {
5658 if (!validateIntClampSupported(Inst)) {
5660 "integer clamping is not supported on this GPU");
5663 if (!validateOpSel(Inst)) {
5665 "invalid op_sel operand");
5668 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5670 "invalid neg_lo operand");
5673 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5675 "invalid neg_hi operand");
5678 if (!validateDPP(Inst,
Operands)) {
5682 if (!validateMIMGD16(Inst)) {
5684 "d16 modifier is not supported on this GPU");
5687 if (!validateMIMGDim(Inst,
Operands)) {
5688 Error(IDLoc,
"missing dim operand");
5691 if (!validateTensorR128(Inst)) {
5693 "instruction must set modifier r128=0");
5696 if (!validateMIMGMSAA(Inst)) {
5698 "invalid dim; must be MSAA type");
5701 if (!validateMIMGDataSize(Inst, IDLoc)) {
5704 if (!validateMIMGAddrSize(Inst, IDLoc))
5706 if (!validateMIMGAtomicDMask(Inst)) {
5708 "invalid atomic image dmask");
5711 if (!validateMIMGGatherDMask(Inst)) {
5713 "invalid image_gather dmask: only one bit must be set");
5716 if (!validateMovrels(Inst,
Operands)) {
5719 if (!validateOffset(Inst,
Operands)) {
5722 if (!validateSMEMSData(Inst,
Operands))
5724 if (!validateBF16InlineConst(Inst,
Operands)) {
5727 if (!validateMAIAccWrite(Inst,
Operands)) {
5730 if (!validateMAISrc2(Inst,
Operands)) {
5733 if (!validateMFMA(Inst,
Operands)) {
5736 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5740 if (!validateAGPRLdSt(Inst)) {
5743 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5744 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5745 :
"invalid register class: agpr loads and stores not supported on "
5749 if (!validateVGPRAlign(Inst)) {
5750 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5757 if (!validateBLGP(Inst,
Operands)) {
5761 if (!validateDivScale(Inst)) {
5762 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5765 if (!validateWaitCnt(Inst,
Operands)) {
5768 if (!validateTFE(Inst,
Operands)) {
5771 if (!validateWMMA(Inst,
Operands)) {
5774 if (!validateMonitorSleep(Inst,
Operands)) {
5777 if (!validateClusterBarrierIsFirst(Inst,
Operands)) {
5780 if (!validateScaleSel(Inst,
Operands)) {
5789 unsigned VariantID = 0);
5793 unsigned VariantID);
5795bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5800bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5801 const FeatureBitset &FBS,
5802 ArrayRef<unsigned> Variants) {
5803 for (
auto Variant : Variants) {
5811bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5813 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5816 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5821 getParser().clearPendingErrors();
5825 StringRef VariantName = getMatchedVariantName();
5826 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5827 return Error(IDLoc, Twine(VariantName,
5828 " variant of this instruction is not supported"));
5832 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5833 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5835 FeatureBitset FeaturesWS32 = getFeatureBits();
5836 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5837 .
flip(AMDGPU::FeatureWavefrontSize32);
5838 FeatureBitset AvailableFeaturesWS32 =
5839 ComputeAvailableFeatures(FeaturesWS32);
5841 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5842 return Error(IDLoc,
"instruction requires wavesize=32");
5846 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5847 return Error(IDLoc,
"instruction not supported on this GPU (" +
5848 getSTI().
getCPU() +
")" +
": " + Mnemo);
5853 return Error(IDLoc,
"invalid instruction" + Suggestion);
5859 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5860 if (
Op.isToken() && InvalidOprIdx > 1) {
5861 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5862 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5867bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5871 bool MatchingInlineAsm) {
5874 unsigned Result = Match_Success;
5879 auto atLeastAsSpecific = [](
unsigned New,
unsigned Cur) {
5880 auto rank = [](
unsigned M) {
5881 return M == Match_MnemonicFail ? 1
5882 :
M == Match_InvalidOperand ? 2
5883 :
M == Match_MissingFeature ? 3
5886 return rank(New) >= rank(Cur);
5889 for (
auto Variant : getMatchedVariants()) {
5892 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5893 if (R == Match_Success || atLeastAsSpecific(R, Result)) {
5897 if (R == Match_Success)
5901 if (Result == Match_Success) {
5902 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5905 emitTargetDirective();
5906 Out.emitInstruction(Inst, getSTI());
5913 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5920 case Match_MissingFeature:
5924 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5926 case Match_InvalidOperand: {
5927 SMLoc ErrorLoc = IDLoc;
5928 if (ErrorInfo != ~0ULL) {
5929 if (ErrorInfo >=
Operands.size()) {
5930 return Error(IDLoc,
"too few operands for instruction");
5932 AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*
Operands[ErrorInfo];
5933 ErrorLoc = ErrorOp.getStartLoc();
5934 if (ErrorLoc == SMLoc())
5938 return Error(ErrorLoc,
"invalid VOPDY instruction");
5940 return Error(ErrorLoc,
"invalid operand for instruction");
5943 case Match_MnemonicFail:
5949bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5954 if (getParser().parseAbsoluteExpression(Tmp)) {
5957 Ret =
static_cast<uint32_t
>(Tmp);
5961bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5962 if (!getSTI().getTargetTriple().isAMDGCN())
5963 return TokError(
"directive only supported for amdgcn architecture");
5965 std::string TargetIDDirective;
5966 SMLoc TargetStart = getTok().getLoc();
5967 if (getParser().parseEscapedString(TargetIDDirective))
5970 std::optional<AMDGPU::TargetID> MaybeParsed =
5973 return getParser().Error(TargetStart,
5974 "malformed target id '" + TargetIDDirective +
"'");
5977 const Triple &
TT = getSTI().getTargetTriple();
5983 return getParser().Error(
5984 TargetStart,
"target id '" + TargetIDDirective +
5985 "' specifies a processor that is not valid for "
5987 TT.getArchName() +
"'");
5990 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5991 getTargetStreamer().getTargetID();
5994 const Triple &STITriple = getSTI().getTargetTriple();
5995 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5996 return getParser().Error(
5997 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5998 " is incompatible with " +
5999 Twine(CurrentTargetID->toString()));
6003 StringRef DirectiveProcessor =
6006 if (DirectiveISA != ISA) {
6007 return getParser().Error(TargetStart,
6008 ".amdgcn_target directive processor " +
6009 Twine(DirectiveProcessor) +
6010 " does not match the specified processor " +
6011 Twine(getSTI().
getCPU()));
6017 CurrentTargetID->getXnackSetting())) {
6019 ".amdgcn_target directive has conflicting xnack settings");
6023 CurrentTargetID->getSramEccSetting())) {
6025 ".amdgcn_target directive has conflicting sramecc settings");
6031 getTargetStreamer().getTargetID()->setXnackSetting(
6033 getTargetStreamer().getTargetID()->setSramEccSetting(
6039bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
6043bool AMDGPUAsmParser::calculateGPRBlocks(
6044 const FeatureBitset &Features,
const MCExpr *VCCUsed,
6045 const MCExpr *FlatScrUsed,
bool XNACKUsed,
6046 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
6047 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
6048 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
6053 const MCExpr *
NumSGPRs = NextFreeSGPR;
6054 int64_t EvaluatedSGPRs;
6056 if (
ISA.Major >= 10)
6061 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
6062 !Features.
test(FeatureSGPRInitBug) &&
6063 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6064 return OutOfRangeError(SGPRRange);
6066 const MCExpr *ExtraSGPRs =
6070 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
6071 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6072 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6073 return OutOfRangeError(SGPRRange);
6075 if (Features.
test(FeatureSGPRInitBug))
6082 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6083 unsigned Granule) ->
const MCExpr * {
6087 const MCExpr *AlignToGPR =
6089 const MCExpr *DivGPR =
6095 VGPRBlocks = GetNumGPRBlocks(
6104bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6105 if (!getSTI().getTargetTriple().isAMDGCN())
6106 return TokError(
"directive only supported for amdgcn architecture");
6109 return TokError(
"directive only supported for amdhsa OS");
6111 StringRef KernelName;
6112 if (getParser().parseIdentifier(KernelName))
6119 AMDGPU::MCKernelDescriptor KD =
6129 const MCExpr *NextFreeVGPR = ZeroExpr;
6131 const MCExpr *NamedBarCnt = ZeroExpr;
6136 const MCExpr *NextFreeSGPR = ZeroExpr;
6139 unsigned ImpliedUserSGPRCount = 0;
6143 std::optional<unsigned> ExplicitUserSGPRCount;
6144 const MCExpr *ReserveVCC = OneExpr;
6145 const MCExpr *ReserveFlatScr = OneExpr;
6146 std::optional<bool> EnableWavefrontSize32;
6153 SMRange IDRange = getTok().getLocRange();
6154 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6157 if (ID ==
".end_amdhsa_kernel")
6160 if (!Seen.
insert(ID).second)
6161 return TokError(
".amdhsa_ directives cannot be repeated");
6163 SMLoc ValStart = getLoc();
6164 const MCExpr *ExprVal;
6165 if (getParser().parseExpression(ExprVal))
6167 SMLoc ValEnd = getLoc();
6168 SMRange ValRange = SMRange(ValStart, ValEnd);
6172 bool EvaluatableExpr;
6173 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6175 return OutOfRangeError(ValRange);
6179#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6180 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6181 return OutOfRangeError(RANGE); \
6182 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6187#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6189 return Error(IDRange.Start, "directive should have resolvable expression", \
6192 if (ID ==
".amdhsa_group_segment_fixed_size") {
6195 return OutOfRangeError(ValRange);
6197 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6200 return OutOfRangeError(ValRange);
6202 }
else if (ID ==
".amdhsa_kernarg_size") {
6204 return OutOfRangeError(ValRange);
6206 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6208 ExplicitUserSGPRCount = Val;
6209 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6213 "directive is not supported with architected flat scratch",
6216 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6219 ImpliedUserSGPRCount += 4;
6220 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6223 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6226 return OutOfRangeError(ValRange);
6230 ImpliedUserSGPRCount += Val;
6231 PreloadLength = Val;
6233 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6236 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6239 return OutOfRangeError(ValRange);
6243 PreloadOffset = Val;
6244 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6247 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6250 ImpliedUserSGPRCount += 2;
6251 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6254 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6257 ImpliedUserSGPRCount += 2;
6258 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6261 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6264 ImpliedUserSGPRCount += 2;
6265 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6268 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6271 ImpliedUserSGPRCount += 2;
6272 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6275 "directive is not supported with architected flat scratch",
6279 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6282 ImpliedUserSGPRCount += 2;
6283 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6286 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6289 ImpliedUserSGPRCount += 1;
6290 }
else if (ID ==
".amdhsa_wavefront_size32") {
6293 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6294 EnableWavefrontSize32 = Val;
6296 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6298 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6300 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6302 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6305 "directive is not supported with architected flat scratch",
6308 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6310 }
else if (ID ==
".amdhsa_enable_private_segment") {
6314 "directive is not supported without architected flat scratch",
6317 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6319 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6321 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6323 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6325 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6327 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6329 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6331 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6333 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6335 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6337 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6339 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6340 VGPRRange = ValRange;
6341 NextFreeVGPR = ExprVal;
6342 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6343 SGPRRange = ValRange;
6344 NextFreeSGPR = ExprVal;
6345 }
else if (ID ==
".amdhsa_accum_offset") {
6347 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6348 AccumOffset = ExprVal;
6349 }
else if (ID ==
".amdhsa_named_barrier_count") {
6351 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6352 NamedBarCnt = ExprVal;
6353 }
else if (ID ==
".amdhsa_reserve_vcc") {
6355 return OutOfRangeError(ValRange);
6356 ReserveVCC = ExprVal;
6357 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6359 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6362 "directive is not supported with architected flat scratch",
6365 return OutOfRangeError(ValRange);
6366 ReserveFlatScr = ExprVal;
6367 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6369 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6371 return OutOfRangeError(ValRange);
6372 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny();
6373 if (Val != XnackOn) {
6374 return getParser().Error(
6376 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6378 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6380 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6382 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6384 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6386 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6388 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6390 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6392 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6394 }
else if (ID ==
".amdhsa_dx10_clamp") {
6395 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6396 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6399 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6401 }
else if (ID ==
".amdhsa_ieee_mode") {
6402 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6403 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6406 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6408 }
else if (ID ==
".amdhsa_fp16_overflow") {
6410 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6412 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6414 }
else if (ID ==
".amdhsa_tg_split") {
6416 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6419 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6422 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6424 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6426 }
else if (ID ==
".amdhsa_memory_ordered") {
6428 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6430 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6432 }
else if (ID ==
".amdhsa_forward_progress") {
6434 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6436 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6438 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6440 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6441 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6443 SharedVGPRCount = Val;
6445 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6447 }
else if (ID ==
".amdhsa_inst_pref_size") {
6449 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6450 if (
ISA.Major == 11) {
6452 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6456 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6459 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6462 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6464 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6466 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6468 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6471 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6473 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6475 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6477 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6479 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6481 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6483 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6485 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6487 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6489 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6491 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6493 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6496 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6499#undef PARSE_BITS_ENTRY
6502 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6503 return TokError(
".amdhsa_next_free_vgpr directive is required");
6505 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6506 return TokError(
".amdhsa_next_free_sgpr directive is required");
6508 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6510 return TokError(
"too many user SGPRs enabled, found " +
6511 Twine(UserSGPRCount) +
", but only " +
6517 if (PreloadLength) {
6523 const MCExpr *VGPRBlocks;
6524 const MCExpr *SGPRBlocks;
6525 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6526 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6527 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6528 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6531 int64_t EvaluatedVGPRBlocks;
6532 bool VGPRBlocksEvaluatable =
6533 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6534 if (VGPRBlocksEvaluatable &&
6536 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6537 return OutOfRangeError(VGPRRange);
6541 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6542 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6544 int64_t EvaluatedSGPRBlocks;
6545 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6547 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6548 return OutOfRangeError(SGPRRange);
6551 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6552 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6554 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6555 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6556 "enabled user SGPRs");
6562 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6563 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6568 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6569 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6574 return TokError(
"Kernarg size should be resolvable");
6576 if (PreloadLength && kernarg_size &&
6577 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6578 return TokError(
"Kernarg preload length + offset is larger than the "
6579 "kernarg segment size");
6582 if (!Seen.
contains(
".amdhsa_accum_offset"))
6583 return TokError(
".amdhsa_accum_offset directive is required");
6584 int64_t EvaluatedAccum;
6585 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6586 uint64_t UEvaluatedAccum = EvaluatedAccum;
6587 if (AccumEvaluatable &&
6588 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6589 return TokError(
"accum_offset should be in range [4..256] in "
6592 int64_t EvaluatedNumVGPR;
6593 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6597 return TokError(
"accum_offset exceeds total VGPR allocation");
6603 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6604 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6610 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6611 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6614 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6616 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6617 return TokError(
"shared_vgpr_count directive not valid on "
6618 "wavefront size 32");
6621 if (VGPRBlocksEvaluatable &&
6622 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6624 return TokError(
"shared_vgpr_count*2 + "
6625 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6630 emitTargetDirective();
6631 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6632 NextFreeVGPR, NextFreeSGPR,
6633 ReserveVCC, ReserveFlatScr);
6637bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6639 if (ParseAsAbsoluteExpression(
Version))
6642 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6643 emitTargetDirective();
6647bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6648 AMDGPUMCKernelCodeT &
C) {
6651 if (ID ==
"max_scratch_backing_memory_byte_size") {
6652 Parser.eatToEndOfStatement();
6656 SmallString<40> ErrStr;
6657 raw_svector_ostream Err(ErrStr);
6658 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6659 return TokError(Err.
str());
6663 if (ID ==
"enable_wavefront_size32") {
6666 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6668 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6671 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6675 if (ID ==
"wavefront_size") {
6676 if (
C.wavefront_size == 5) {
6678 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6680 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6681 }
else if (
C.wavefront_size == 6) {
6683 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6690bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6691 AMDGPUMCKernelCodeT KernelCode;
6701 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6704 if (ID ==
".end_amd_kernel_code_t")
6707 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6712 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6717bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6718 StringRef KernelName;
6719 if (!parseId(KernelName,
"expected symbol name"))
6722 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6729bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6730 if (!getSTI().getTargetTriple().isAMDGCN()) {
6731 return Error(getLoc(),
6732 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6736 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6738 std::optional<AMDGPU::TargetID> MaybeParsed =
6741 return Error(getParser().getTok().getLoc(),
6742 "malformed target id '" + TargetIDDirective +
"'");
6745 const Triple &
TT = getSTI().getTargetTriple();
6751 return Error(getParser().getTok().getLoc(),
6752 "target id '" + TargetIDDirective +
6753 "' specifies a processor that is not valid for subarch '" +
6754 TT.getArchName() +
"'");
6757 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6758 getTargetStreamer().getTargetID();
6761 const Triple &STITriple = getSTI().getTargetTriple();
6762 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6763 return Error(getParser().getTok().getLoc(),
6764 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6765 " is incompatible with " +
6766 Twine(CurrentTargetID->toString()));
6770 StringRef DirectiveProcessor =
6773 if (DirectiveISA != ISA) {
6774 return Error(getParser().getTok().getLoc(),
6775 ".amd_amdgpu_isa directive processor " +
6776 Twine(DirectiveProcessor) +
6777 " does not match the specified processor " +
6778 Twine(getSTI().
getCPU()));
6781 getTargetStreamer().EmitISAVersion();
6787bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6790 std::string HSAMetadataString;
6795 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6796 return Error(getLoc(),
"invalid HSA metadata");
6803bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6804 const char *AssemblerDirectiveEnd,
6805 std::string &CollectString) {
6807 raw_string_ostream CollectStream(CollectString);
6809 getLexer().setSkipSpace(
false);
6811 bool FoundEnd =
false;
6814 CollectStream << getTokenStr();
6818 if (trySkipId(AssemblerDirectiveEnd)) {
6823 CollectStream << Parser.parseStringToEndOfStatement()
6824 <<
getContext().getAsmInfo().getSeparatorString();
6826 Parser.eatToEndOfStatement();
6829 getLexer().setSkipSpace(
true);
6832 return TokError(Twine(
"expected directive ") +
6833 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6840bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6846 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6847 if (!PALMetadata->setFromString(
String))
6848 return Error(getLoc(),
"invalid PAL metadata");
6853bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6856 Twine(
" directive is "
6857 "not available on non-amdpal OSes"))
6861 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6862 PALMetadata->setLegacy();
6865 if (ParseAsAbsoluteExpression(
Key)) {
6866 return TokError(Twine(
"invalid value in ") +
6870 return TokError(Twine(
"expected an even number of values in ") +
6873 if (ParseAsAbsoluteExpression(
Value)) {
6874 return TokError(Twine(
"invalid value in ") +
6877 PALMetadata->setRegister(
Key,
Value);
6886bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6887 if (getParser().checkForValidSection())
6891 SMLoc NameLoc = getLoc();
6892 if (getParser().parseIdentifier(Name))
6893 return TokError(
"expected identifier in directive");
6896 if (getParser().parseComma())
6902 SMLoc SizeLoc = getLoc();
6903 if (getParser().parseAbsoluteExpression(
Size))
6906 return Error(SizeLoc,
"size must be non-negative");
6907 if (
Size > LocalMemorySize)
6908 return Error(SizeLoc,
"size is too large");
6912 SMLoc AlignLoc = getLoc();
6913 if (getParser().parseAbsoluteExpression(Alignment))
6916 return Error(AlignLoc,
"alignment must be a power of two");
6921 if (Alignment >= 1u << 31)
6922 return Error(AlignLoc,
"alignment is too large");
6928 Symbol->redefineIfPossible();
6929 if (!
Symbol->isUndefined())
6930 return Error(NameLoc,
"invalid symbol redefinition");
6932 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6936bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6937 if (getParser().checkForValidSection())
6941 if (getParser().parseIdentifier(FuncName))
6942 return TokError(
"expected symbol name after .amdgpu_info");
6945 AMDGPU::InfoSectionData ParsedInfoData;
6946 AMDGPU::FuncInfo FI;
6948 bool HasScalarAttrs =
false;
6955 SMLoc IDLoc = getLoc();
6956 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6959 if (ID ==
".end_amdgpu_info")
6967 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6969 if (Dir ==
"flags") {
6971 if (getParser().parseAbsoluteExpression(Val))
6974 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6976 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6978 HasScalarAttrs =
true;
6979 }
else if (Dir ==
"num_sgpr") {
6981 if (getParser().parseAbsoluteExpression(Val))
6983 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6984 HasScalarAttrs =
true;
6985 }
else if (Dir ==
"num_vgpr") {
6987 if (getParser().parseAbsoluteExpression(Val))
6990 HasScalarAttrs =
true;
6991 }
else if (Dir ==
"num_agpr") {
6993 if (getParser().parseAbsoluteExpression(Val))
6996 HasScalarAttrs =
true;
6997 }
else if (Dir ==
"private_segment_size") {
6999 if (getParser().parseAbsoluteExpression(Val))
7002 HasScalarAttrs =
true;
7003 }
else if (Dir ==
"use") {
7005 if (getParser().parseIdentifier(ResName))
7006 return TokError(
"expected resource symbol for .amdgpu_use");
7007 ParsedInfoData.
Uses.push_back(
7008 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
7009 }
else if (Dir ==
"call") {
7011 if (getParser().parseIdentifier(DstName))
7012 return TokError(
"expected callee symbol for .amdgpu_call");
7013 ParsedInfoData.
Calls.push_back(
7014 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
7015 }
else if (Dir ==
"indirect_call") {
7017 if (getParser().parseEscapedString(TypeId))
7018 return TokError(
"expected type ID string for .amdgpu_indirect_call");
7019 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
7020 }
else if (Dir ==
"typeid") {
7022 if (getParser().parseEscapedString(TypeId))
7023 return TokError(
"expected type ID string for .amdgpu_typeid");
7024 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
7026 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
7031 ParsedInfoData.
Funcs.push_back(std::move(FI));
7033 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
7034 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
7035 Data.Funcs.push_back(std::move(Func));
7036 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
7037 Data.Uses.push_back(Use);
7038 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
7040 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
7043 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
7044 Data.TypeIds.push_back(std::move(TypeId));
7049void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7056void AMDGPUAsmParser::checkKernelPrologues() {
7057 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7058 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7059 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7060 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
7061 if (!AMDHSAKernelSymbols.
contains(Sym))
7063 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
7064 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
7068 "' does not begin with the required prologue "
7069 "sequence: s_mov_b64 followed by v_nop and "
7070 "global_prefetch_b8");
7074 OpcodeStream.
clear();
7075 OpcodeStreamSymbols.clear();
7076 AMDHSAKernelSymbols.
clear();
7079void AMDGPUAsmParser::onEndOfFile() {
7080 emitTargetDirective();
7081 checkKernelPrologues();
7083 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7086bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7087 StringRef IDVal = DirectiveID.
getString();
7090 if (IDVal ==
".amdhsa_kernel")
7091 return ParseDirectiveAMDHSAKernel();
7093 if (IDVal ==
".amdhsa_code_object_version")
7094 return ParseDirectiveAMDHSACodeObjectVersion();
7098 return ParseDirectiveHSAMetadata();
7100 if (IDVal ==
".amd_kernel_code_t")
7101 return ParseDirectiveAMDKernelCodeT();
7103 if (IDVal ==
".amdgpu_hsa_kernel")
7104 return ParseDirectiveAMDGPUHsaKernel();
7106 if (IDVal ==
".amd_amdgpu_isa")
7107 return ParseDirectiveISAVersion();
7111 Twine(
" directive is "
7112 "not available on non-amdhsa OSes"))
7117 if (IDVal ==
".amdgcn_target")
7118 return ParseDirectiveAMDGCNTarget();
7120 if (IDVal ==
".amdgpu_lds")
7121 return ParseDirectiveAMDGPULDS();
7123 if (IDVal ==
".amdgpu_info")
7124 return ParseDirectiveAMDGPUInfo();
7127 return ParseDirectivePALMetadataBegin();
7130 return ParseDirectivePALMetadata();
7135bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &
MRI,
7137 if (
MRI.regsOverlap(TTMP12_TTMP13_TTMP14_TTMP15,
Reg))
7141 if (
MRI.regsOverlap(SGPR104_SGPR105,
Reg))
7142 return hasSGPR104_SGPR105();
7145 case SRC_SHARED_BASE_LO:
7146 case SRC_SHARED_BASE:
7147 case SRC_SHARED_LIMIT_LO:
7148 case SRC_SHARED_LIMIT:
7150 case SRC_PRIVATE_BASE_LO:
7151 case SRC_PRIVATE_BASE:
7152 case SRC_PRIVATE_LIMIT_LO:
7153 case SRC_PRIVATE_LIMIT:
7155 case SRC_FLAT_SCRATCH_BASE_LO:
7156 case SRC_FLAT_SCRATCH_BASE_HI:
7157 return hasGloballyAddressableScratch();
7158 case SRC_POPS_EXITING_WAVE_ID:
7171 getTargetStreamer().getTargetID()->isXnackSupported();
7200 if (
MRI.regsOverlap(SGPR102_SGPR103,
Reg))
7201 return hasSGPR102_SGPR103();
7209 ParseStatus Res = parseVOPD(
Operands);
7214 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7226 SMLoc LBraceLoc = getLoc();
7231 auto Loc = getLoc();
7234 Error(Loc,
"expected a register");
7238 RBraceLoc = getLoc();
7243 "expected a comma or a closing square bracket"))
7247 if (
Operands.size() - Prefix > 1) {
7249 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7250 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7259StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7261 setForcedEncodingSize(0);
7262 setForcedDPP(
false);
7263 setForcedSDWA(
false);
7265 if (
Name.consume_back(
"_e64_dpp")) {
7267 setForcedEncodingSize(64);
7270 if (
Name.consume_back(
"_e64")) {
7271 setForcedEncodingSize(64);
7274 if (
Name.consume_back(
"_e32")) {
7275 setForcedEncodingSize(32);
7278 if (
Name.consume_back(
"_dpp")) {
7282 if (
Name.consume_back(
"_sdwa")) {
7283 setForcedSDWA(
true);
7291 unsigned VariantID);
7297 Name = parseMnemonicSuffix(Name);
7303 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7305 bool IsMIMG = Name.starts_with(
"image_");
7308 OperandMode
Mode = OperandMode_Default;
7310 Mode = OperandMode_NSA;
7314 checkUnsupportedInstruction(Name, NameLoc);
7315 if (!Parser.hasPendingError()) {
7318 :
"not a valid operand.";
7338ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7341 if (!trySkipId(Name))
7344 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7348ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7357ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7359 std::function<
bool(int64_t &)> ConvertResult) {
7363 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7367 if (ConvertResult && !ConvertResult(
Value)) {
7368 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7371 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7375ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7377 bool (*ConvertResult)(int64_t &)) {
7386 const unsigned MaxSize = 4;
7390 for (
int I = 0;; ++
I) {
7392 SMLoc Loc = getLoc();
7396 if (
Op != 0 &&
Op != 1)
7397 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7404 if (
I + 1 == MaxSize)
7405 return Error(getLoc(),
"expected a closing square bracket");
7411 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7415ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7417 AMDGPUOperand::ImmTy ImmTy,
7418 bool IgnoreNegative) {
7422 if (trySkipId(Name)) {
7424 }
else if (trySkipId(
"no", Name)) {
7433 return Error(S,
"r128 modifier is not supported on this GPU");
7434 if (Name ==
"a16" && !
hasA16())
7435 return Error(S,
"a16 modifier is not supported on this GPU");
7437 if (Bit == 0 && Name ==
"gds") {
7440 return Error(S,
"nogds is not allowed");
7443 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7444 ImmTy = AMDGPUOperand::ImmTyR128A16;
7446 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7450unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7451 bool &Disabling)
const {
7452 Disabling =
Id.consume_front(
"no");
7455 return StringSwitch<unsigned>(Id)
7462 return StringSwitch<unsigned>(Id)
7472 SMLoc StringLoc = getLoc();
7474 int64_t CPolVal = 0;
7495 ResScope = parseScope(
Operands, Scope);
7508 if (trySkipId(
"nv")) {
7512 }
else if (trySkipId(
"no",
"nv")) {
7519 if (trySkipId(
"scale_offset")) {
7523 }
else if (trySkipId(
"no",
"scale_offset")) {
7547 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7548 AMDGPUOperand::ImmTyCPol));
7553 SMLoc OpLoc = getLoc();
7554 unsigned Enabled = 0, Seen = 0;
7558 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7565 return Error(S,
"dlc modifier is not supported on this GPU");
7568 return Error(S,
"scc modifier is not supported on this GPU");
7571 return Error(S,
"duplicate cache policy modifier");
7583 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7592 ParseStatus Res = parseStringOrIntWithPrefix(
7593 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7607 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7611 if (
Value ==
"TH_DEFAULT")
7613 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7614 Value ==
"TH_LOAD_NT_WB") {
7615 return Error(StringLoc,
"invalid th value");
7616 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7618 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7620 }
else if (
Value.consume_front(
"TH_STORE_")) {
7623 return Error(StringLoc,
"invalid th value");
7626 if (
Value ==
"BYPASS")
7631 TH |= StringSwitch<int64_t>(
Value)
7641 .Default(0xffffffff);
7643 TH |= StringSwitch<int64_t>(
Value)
7654 .Default(0xffffffff);
7657 if (TH == 0xffffffff)
7658 return Error(StringLoc,
"invalid th value");
7667 ParseStatus Res = parseStringOrIntWithPrefix(
7668 Operands,
"cfs", {
"CFS_256B",
"CFS_128B",
"CFS_64B",
"CFS_32B"},
CFS);
7678 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7679 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7680 std::optional<unsigned> InsertAt = std::nullopt) {
7681 auto i = OptionalIdx.find(ImmT);
7682 if (i != OptionalIdx.end()) {
7683 unsigned Idx = i->second;
7684 const AMDGPUOperand &
Op =
7685 static_cast<const AMDGPUOperand &
>(*
Operands[
Idx]);
7689 Op.addImmOperands(Inst, 1);
7691 if (InsertAt.has_value())
7698ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7704 StringLoc = getLoc();
7709ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7715 SMLoc StringLoc = getLoc();
7719 Value = getTokenStr();
7723 if (
Value == Ids[IntVal])
7728 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7729 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7734ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7736 AMDGPUOperand::ImmTy
Type) {
7740 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7742 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7751ParseStatus AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7754 SMLoc Loc = getLoc();
7756 ParseStatus Res = parseIntWithPrefix(Pref, Val);
7760 if (Val < 0 || Val > MaxVal)
7761 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7768 AMDGPUOperand::ImmTy ImmTy) {
7769 const char *Pref =
"index_key";
7771 SMLoc Loc = getLoc();
7772 auto Res = parseIntWithPrefix(Pref, ImmVal);
7776 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7777 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7778 (ImmVal < 0 || ImmVal > 1))
7779 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7781 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7782 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7784 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7789 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7793 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7797 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7802 AMDGPUOperand::ImmTy
Type) {
7808 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7809 AMDGPUOperand::ImmTyMatrixAFMT);
7813 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7814 AMDGPUOperand::ImmTyMatrixBFMT);
7819 AMDGPUOperand::ImmTy
Type) {
7825 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7826 AMDGPUOperand::ImmTyMatrixAScale);
7830 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7831 AMDGPUOperand::ImmTyMatrixBScale);
7836 AMDGPUOperand::ImmTy
Type) {
7842 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7843 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7847 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7848 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7853ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7854 using namespace llvm::AMDGPU::MTBUFFormat;
7859 auto TryAfterComma = [&](
const char *Pref, int64_t MaxVal,
7860 int64_t &Fmt) -> ParseStatus {
7862 return Error(getLoc(),
"expected a comma");
7863 AsmToken NextTokens[2];
7864 peekTokens(NextTokens);
7869 return tryParseFmt(Pref, MaxVal, Fmt);
7874 ParseStatus Res = tryParseFmt(
"dfmt", DFMT_MAX, Dfmt);
7876 Res = TryAfterComma(
"nfmt", NFMT_MAX, Nfmt);
7878 Res = tryParseFmt(
"nfmt", NFMT_MAX, Nfmt);
7880 Res = TryAfterComma(
"dfmt", DFMT_MAX, Dfmt);
7889ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7893bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7894 StringRef FormatStr, SMLoc Loc) {
7895 using namespace llvm::AMDGPU::MTBUFFormat;
7899 if (
Format != DFMT_UNDEF) {
7905 if (
Format != NFMT_UNDEF) {
7910 Error(Loc,
"unsupported format");
7914ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7917 using namespace llvm::AMDGPU::MTBUFFormat;
7921 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7926 SMLoc Loc = getLoc();
7927 if (!parseId(Str,
"expected a format string") ||
7928 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7930 if (Dfmt == DFMT_UNDEF)
7931 return Error(Loc,
"duplicate numeric format");
7932 if (Nfmt == NFMT_UNDEF)
7933 return Error(Loc,
"duplicate data format");
7936 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7937 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7941 if (Ufmt == UFMT_UNDEF)
7942 return Error(FormatLoc,
"unsupported format");
7951ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7954 using namespace llvm::AMDGPU::MTBUFFormat;
7957 if (Id == UFMT_UNDEF)
7961 return Error(Loc,
"unified format is not supported on this GPU");
7967ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7968 using namespace llvm::AMDGPU::MTBUFFormat;
7969 SMLoc Loc = getLoc();
7974 return Error(Loc,
"out of range format");
7979ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7980 using namespace llvm::AMDGPU::MTBUFFormat;
7986 StringRef FormatStr;
7987 SMLoc Loc = getLoc();
7988 if (!parseId(FormatStr,
"expected a format string"))
7991 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7993 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
8003 return parseNumericFormat(
Format);
8007 SMLoc Loc = getLoc();
8009 ParseStatus Res = parseSymbolicOrNumericFormat(
Format);
8014 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFormat));
8019 SMLoc Loc = getLoc();
8026 AMDGPUOperand::ImmTyFormatOp));
8032 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
8034 Res = parseIntWithPrefix(
"inst_offset",
Operands,
8035 AMDGPUOperand::ImmTyInstOffset);
8042 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
8044 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
8050 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
8053 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8063 OptionalImmIndexMap OptionalIdx;
8065 unsigned OperandIdx[4];
8066 unsigned EnMask = 0;
8069 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8070 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8075 OperandIdx[SrcIdx] = Inst.
size();
8076 Op.addRegOperands(Inst, 1);
8083 OperandIdx[SrcIdx] = Inst.
size();
8089 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8090 Op.addImmOperands(Inst, 1);
8094 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8098 OptionalIdx[
Op.getImmTy()] = i;
8104 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8111 for (
auto i = 0; i < SrcIdx; ++i) {
8113 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8119 AMDGPUOperand::ImmTyExpCompr);
8129 int64_t CntVal,
bool Saturate,
8135 IntVal =
encode(ISA, IntVal, CntVal);
8136 if (CntVal !=
decode(ISA, IntVal)) {
8138 IntVal =
encode(ISA, IntVal, -1);
8146bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8148 SMLoc CntLoc = getLoc();
8149 StringRef CntName = getTokenStr();
8156 SMLoc ValLoc = getLoc();
8163 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8165 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8167 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8170 Error(CntLoc,
"invalid counter name " + CntName);
8175 Error(ValLoc,
"too large value for " + CntName);
8184 Error(getLoc(),
"expected a counter name");
8198 if (!parseCnt(Waitcnt))
8206 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8210bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8211 SMLoc FieldLoc = getLoc();
8212 StringRef FieldName = getTokenStr();
8217 SMLoc ValueLoc = getLoc();
8224 if (FieldName ==
"instid0") {
8226 }
else if (FieldName ==
"instskip") {
8228 }
else if (FieldName ==
"instid1") {
8231 Error(FieldLoc,
"invalid field name " + FieldName);
8250 .Case(
"VALU_DEP_1", 1)
8251 .Case(
"VALU_DEP_2", 2)
8252 .Case(
"VALU_DEP_3", 3)
8253 .Case(
"VALU_DEP_4", 4)
8254 .Case(
"TRANS32_DEP_1", 5)
8255 .Case(
"TRANS32_DEP_2", 6)
8256 .Case(
"TRANS32_DEP_3", 7)
8257 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8258 .Case(
"SALU_CYCLE_1", 9)
8259 .Case(
"SALU_CYCLE_2", 10)
8260 .Case(
"SALU_CYCLE_3", 11)
8268 Delay |=
Value << Shift;
8278 if (!parseDelay(Delay))
8286 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8290bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8292bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8298void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8299 StringRef DepCtrName) {
8302 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8305 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8308 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8311 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8318bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8320 using namespace llvm::AMDGPU::DepCtr;
8322 SMLoc DepCtrLoc = getLoc();
8323 StringRef DepCtrName = getTokenStr();
8333 unsigned PrevOprMask = UsedOprMask;
8334 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8337 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8346 Error(getLoc(),
"expected a counter name");
8351 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8352 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8357 using namespace llvm::AMDGPU::DepCtr;
8360 SMLoc Loc = getLoc();
8363 unsigned UsedOprMask = 0;
8365 if (!parseDepCtr(DepCtr, UsedOprMask))
8373 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8377bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8383ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8385 OperandInfoTy &Width) {
8386 using namespace llvm::AMDGPU::Hwreg;
8392 HwReg.Loc = getLoc();
8395 HwReg.IsSymbolic =
true;
8397 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8405 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8415 Width.Loc = getLoc();
8424 using namespace llvm::AMDGPU::Hwreg;
8427 SMLoc Loc = getLoc();
8429 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8431 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8432 HwregOffset::Default);
8433 struct : StructuredOpField {
8434 using StructuredOpField::StructuredOpField;
8435 bool validate(AMDGPUAsmParser &Parser)
const override {
8437 return Error(Parser,
"only values from 1 to 32 are legal");
8440 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8441 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8444 Res = parseHwregFunc(HwReg,
Offset, Width);
8447 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8449 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8453 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8460 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8462 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8466bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8472bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8473 OperandInfoTy &Stream) {
8474 using namespace llvm::AMDGPU::SendMsg;
8479 Msg.IsSymbolic =
true;
8486 Op.IsDefined =
true;
8492 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8497 Stream.IsDefined =
true;
8498 Stream.Loc = getLoc();
8507bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8508 const OperandInfoTy &
Op,
8509 const OperandInfoTy &Stream) {
8510 using namespace llvm::AMDGPU::SendMsg;
8519 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8524 Error(
Msg.Loc,
"invalid message id");
8530 Error(
Op.Loc,
"message does not support operations");
8532 Error(
Msg.Loc,
"missing message operation");
8538 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8540 Error(
Op.Loc,
"invalid operation id");
8545 Error(Stream.Loc,
"message operation does not support streams");
8549 Error(Stream.Loc,
"invalid message stream id");
8556 using namespace llvm::AMDGPU::SendMsg;
8559 SMLoc Loc = getLoc();
8563 OperandInfoTy
Op(OP_NONE_);
8564 OperandInfoTy Stream(STREAM_ID_NONE_);
8565 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8570 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8572 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8578 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8582bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8585 using namespace llvm::AMDGPU::WaitEvent;
8587 SMLoc Loc = getLoc();
8590 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8592 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8594 StructuredOpField *TargetBitfield =
8595 isGFX11() ? &DontWaitExportReady : &ExportReady;
8597 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8601 if (!validateStructuredOpFields({TargetBitfield}))
8603 ImmVal = TargetBitfield->Val;
8610 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8612 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8613 AMDGPUOperand::ImmTyWaitEvent));
8617bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8630 int Slot = StringSwitch<int>(Str)
8637 return Error(S,
"invalid interpolation slot");
8640 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8651 if (!Str.starts_with(
"attr"))
8652 return Error(S,
"invalid interpolation attribute");
8654 StringRef Chan = Str.take_back(2);
8655 int AttrChan = StringSwitch<int>(Chan)
8662 return Error(S,
"invalid or missing interpolation attribute channel");
8664 Str = Str.drop_back(2).drop_front(4);
8667 if (Str.getAsInteger(10, Attr))
8668 return Error(S,
"invalid or missing interpolation attribute number");
8671 return Error(S,
"out of bounds interpolation attribute number");
8676 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8677 Operands.push_back(AMDGPUOperand::CreateImm(
8678 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8687 using namespace llvm::AMDGPU::Exp;
8697 return Error(S, (Id == ET_INVALID)
8698 ?
"invalid exp target"
8699 :
"exp target is not supported on this GPU");
8702 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8710bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8714bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8719 return getTokenKind() ==
Kind;
8722StringRef AMDGPUAsmParser::getId()
const {
8726bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8734bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8736 StringRef Tok = getTokenStr();
8745bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8747 if (isId(Id) && peekToken().is(Kind)) {
8756 if (isToken(Kind)) {
8764 const StringRef ErrMsg) {
8765 if (!trySkipToken(Kind)) {
8766 Error(getLoc(), ErrMsg);
8772bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8776 if (Parser.parseExpression(Expr))
8779 if (Expr->evaluateAsAbsolute(
Imm))
8782 if (Expected.empty()) {
8783 Error(S,
"expected absolute expression");
8786 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8795 if (Parser.parseExpression(Expr))
8799 if (Expr->evaluateAsAbsolute(IntVal)) {
8800 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8802 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8807bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8809 Val =
getToken().getStringContents();
8813 Error(getLoc(), ErrMsg);
8817bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8819 Val = getTokenStr();
8823 if (!ErrMsg.
empty())
8824 Error(getLoc(), ErrMsg);
8828AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8830AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8833 : getLexer().peekTok(ShouldSkipSpace);
8837 auto TokCount = getLexer().peekTokens(Tokens);
8844 return getLexer().getKind();
8847SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8849StringRef AMDGPUAsmParser::getTokenStr()
const {
8853void AMDGPUAsmParser::lex() { Parser.Lex(); }
8855const AMDGPUOperand &
8857 int MCOpIdx)
const {
8859 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8860 if (TargetOp.getMCOpIdx() == MCOpIdx)
8867 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8871SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8876 int MCOpIdx)
const {
8877 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8880SMLoc AMDGPUAsmParser::getOperandLoc(
8881 std::function<
bool(
const AMDGPUOperand &)>
Test,
8883 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8884 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8886 return Op.getStartLoc();
8891SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8893 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8908 StringRef
Id = getTokenStr();
8909 SMLoc IdLoc = getLoc();
8915 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8916 if (
I == Fields.
end())
8917 return Error(IdLoc,
"unknown field");
8918 if ((*I)->IsDefined)
8919 return Error(IdLoc,
"duplicate field");
8922 (*I)->Loc = getLoc();
8925 (*I)->IsDefined =
true;
8932bool AMDGPUAsmParser::validateStructuredOpFields(
8934 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8935 return F->validate(*
this);
8945 const unsigned XorMask) {
8952bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8953 const unsigned MaxVal,
8954 const Twine &ErrMsg, SMLoc &Loc) {
8970bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8971 const unsigned MinVal,
8972 const unsigned MaxVal,
8973 const StringRef ErrMsg) {
8975 for (
unsigned i = 0; i < OpNum; ++i) {
8976 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8983bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8984 using namespace llvm::AMDGPU::Swizzle;
8987 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8988 "expected a 2-bit lane id")) {
8998bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8999 using namespace llvm::AMDGPU::Swizzle;
9005 if (!parseSwizzleOperand(GroupSize, 2, 32,
9006 "group size must be in the interval [2,32]", Loc)) {
9010 Error(Loc,
"group size must be a power of two");
9013 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
9014 "lane id must be in the interval [0,group size - 1]",
9022bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
9023 using namespace llvm::AMDGPU::Swizzle;
9028 if (!parseSwizzleOperand(GroupSize, 2, 32,
9029 "group size must be in the interval [2,32]", Loc)) {
9033 Error(Loc,
"group size must be a power of two");
9041bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
9042 using namespace llvm::AMDGPU::Swizzle;
9047 if (!parseSwizzleOperand(GroupSize, 1, 16,
9048 "group size must be in the interval [1,16]", Loc)) {
9052 Error(Loc,
"group size must be a power of two");
9060bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9061 using namespace llvm::AMDGPU::Swizzle;
9068 SMLoc StrLoc = getLoc();
9069 if (!parseString(Ctl)) {
9072 if (Ctl.
size() != BITMASK_WIDTH) {
9073 Error(StrLoc,
"expected a 5-character mask");
9077 unsigned AndMask = 0;
9078 unsigned OrMask = 0;
9079 unsigned XorMask = 0;
9081 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9085 Error(StrLoc,
"invalid mask");
9106bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9107 using namespace llvm::AMDGPU::Swizzle;
9110 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9116 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9117 "FFT swizzle must be in the interval [0," +
9118 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9126bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9127 using namespace llvm::AMDGPU::Swizzle;
9130 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9137 if (!parseSwizzleOperand(
Direction, 0, 1,
9138 "direction must be 0 (left) or 1 (right)", Loc))
9142 if (!parseSwizzleOperand(
9143 RotateSize, 0, ROTATE_MAX_SIZE,
9144 "number of threads to rotate must be in the interval [0," +
9145 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9150 (RotateSize << ROTATE_SIZE_SHIFT);
9154bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9156 SMLoc OffsetLoc = getLoc();
9162 Error(OffsetLoc,
"expected a 16-bit offset");
9168bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9169 using namespace llvm::AMDGPU::Swizzle;
9173 SMLoc ModeLoc = getLoc();
9176 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9177 Ok = parseSwizzleQuadPerm(
Imm);
9178 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9179 Ok = parseSwizzleBitmaskPerm(
Imm);
9180 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9181 Ok = parseSwizzleBroadcast(
Imm);
9182 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9183 Ok = parseSwizzleSwap(
Imm);
9184 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9185 Ok = parseSwizzleReverse(
Imm);
9186 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9187 Ok = parseSwizzleFFT(
Imm);
9188 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9189 Ok = parseSwizzleRotate(
Imm);
9191 Error(ModeLoc,
"expected a swizzle mode");
9194 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9204 if (trySkipId(
"offset")) {
9208 if (trySkipId(
"swizzle")) {
9209 Ok = parseSwizzleMacro(
Imm);
9211 Ok = parseSwizzleOffset(
Imm);
9216 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9223bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9229int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9231 using namespace llvm::AMDGPU::VGPRIndexMode;
9243 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9244 if (trySkipId(IdSymbolic[ModeId])) {
9252 ?
"expected a VGPR index mode or a closing parenthesis"
9253 :
"expected a VGPR index mode");
9258 Error(S,
"duplicate VGPR index mode");
9266 "expected a comma or a closing parenthesis"))
9275 using namespace llvm::AMDGPU::VGPRIndexMode;
9281 Imm = parseGPRIdxMacro();
9285 if (getParser().parseAbsoluteExpression(
Imm))
9288 return Error(S,
"invalid immediate: only 4-bit values are legal");
9292 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9296bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9307 if (isRegister() || isModifier())
9314 assert(Opr.isImm() || Opr.isExpr());
9315 SMLoc Loc = Opr.getStartLoc();
9319 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9320 Error(Loc,
"expected an absolute expression or a label");
9321 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9322 Error(Loc,
"expected a 16-bit signed jump offset");
9342 OptionalImmIndexMap OptionalIdx;
9343 unsigned FirstOperandIdx = 1;
9344 bool IsAtomicReturn =
false;
9350 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9351 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9355 Op.addRegOperands(Inst, 1);
9359 if (IsAtomicReturn && i == FirstOperandIdx)
9360 Op.addRegOperands(Inst, 1);
9365 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9366 Op.addImmOperands(Inst, 1);
9378 OptionalIdx[
Op.getImmTy()] = i;
9382 AMDGPUOperand::ImmTyOffset);
9398bool AMDGPUOperand::isSMRDOffset8()
const {
9402bool AMDGPUOperand::isSMEMOffset()
const {
9404 return isImmLiteral();
9407bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9442bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9443 if (BoundCtrl == 0 || BoundCtrl == 1) {
9451void AMDGPUAsmParser::onBeginOfFile() {
9452 if (!getParser().getStreamer().getTargetStreamer())
9455 if (!getTargetStreamer().getTargetID())
9456 getTargetStreamer().initializeTargetID(getSTI(),
9460void AMDGPUAsmParser::emitTargetDirective() {
9461 if (!getParser().getStreamer().getTargetStreamer() ||
9466 getTargetStreamer().emitDirectiveAMDGCNTargetOnce();
9475bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9479 StringRef TokenId = getTokenStr();
9480 AGVK VK = StringSwitch<AGVK>(TokenId)
9481 .Case(
"max", AGVK::AGVK_Max)
9482 .Case(
"min", AGVK::AGVK_Min)
9483 .Case(
"or", AGVK::AGVK_Or)
9484 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9485 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9486 .Case(
"alignto", AGVK::AGVK_AlignTo)
9487 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9488 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9489 .Default(AGVK::AGVK_None);
9498 if (Exprs.
empty()) {
9500 "empty " + Twine(TokenId) +
" expression");
9503 if (CommaCount + 1 != Exprs.
size()) {
9505 "mismatch of commas in " + Twine(TokenId) +
" expression");
9509 Expected && Exprs.
size() != Expected) {
9510 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9511 Twine(Expected) +
" operands");
9518 if (getParser().parseExpression(Expr, EndLoc))
9522 if (LastTokenWasComma)
9526 "unexpected token in " + Twine(TokenId) +
" expression");
9532 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9536 StringRef
Name = getTokenStr();
9537 if (Name ==
"mul") {
9538 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9542 if (Name ==
"div") {
9543 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9554 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9559 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9560 AMDGPU::OpName::src2};
9568 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9573 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9575 if (
DstOp.isReg() &&
9576 MRI.getRegClass(AMDGPU::VGPR_16RegClassID).contains(
DstOp.
getReg())) {
9580 if ((OpSel & (1 << SrcNum)) != 0)
9586void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9593 OptionalImmIndexMap &OptionalIdx) {
9594 cvtVOP3P(Inst,
Operands, OptionalIdx);
9603 &&
Desc.NumOperands > (OpNum + 1)
9605 &&
Desc.operands()[OpNum + 1].RegClass != -1
9607 &&
Desc.getOperandConstraint(OpNum + 1,
9611void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9613 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9614 AMDGPU::OpName::src2};
9615 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9616 AMDGPU::OpName::src1_modifiers,
9617 AMDGPU::OpName::src2_modifiers};
9618 for (
int J = 0; J < 3; ++J) {
9619 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9625 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9628 if ((OpSel & (1 << J)) != 0)
9631 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9638void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9640 OptionalImmIndexMap OptionalIdx;
9645 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9646 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9650 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9652 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9653 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9654 Op.isInterpAttrChan()) {
9656 }
else if (
Op.isImmModifier()) {
9657 OptionalIdx[
Op.getImmTy()] =
I;
9665 AMDGPUOperand::ImmTyHigh);
9669 AMDGPUOperand::ImmTyClamp);
9673 AMDGPUOperand::ImmTyOModSI);
9678 AMDGPUOperand::ImmTyOpSel);
9679 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9682 cvtOpSelHelper(Inst, OpSel);
9687 OptionalImmIndexMap OptionalIdx;
9692 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9693 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9697 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9699 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9700 }
else if (
Op.isImmModifier()) {
9701 OptionalIdx[
Op.getImmTy()] =
I;
9709 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9712 AMDGPUOperand::ImmTyOpSel);
9715 AMDGPUOperand::ImmTyWaitEXP);
9721 cvtOpSelHelper(Inst, OpSel);
9724void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9726 OptionalImmIndexMap OptionalIdx;
9729 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9733 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9734 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9737 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9742 if (NumOperands == CbszOpIdx) {
9747 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9748 }
else if (
Op.isImmModifier()) {
9749 OptionalIdx[
Op.getImmTy()] =
I;
9751 Op.addRegOrImmOperands(Inst, 1);
9756 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9757 if (CbszIdx != OptionalIdx.end()) {
9758 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9762 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9763 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9764 if (BlgpIdx != OptionalIdx.end()) {
9765 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9776 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9777 if (OpselIdx != OptionalIdx.end()) {
9778 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9782 unsigned OpSelHi = 0;
9783 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9784 if (OpselHiIdx != OptionalIdx.end()) {
9785 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9788 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9789 AMDGPU::OpName::src1_modifiers};
9791 for (
unsigned J = 0; J < 2; ++J) {
9792 unsigned ModVal = 0;
9793 if (OpSel & (1 << J))
9795 if (OpSelHi & (1 << J))
9798 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9804 OptionalImmIndexMap &OptionalIdx) {
9809 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9810 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9814 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9816 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9817 }
else if (
Op.isImmModifier()) {
9818 OptionalIdx[
Op.getImmTy()] =
I;
9820 Op.addRegOrImmOperands(Inst, 1);
9826 AMDGPUOperand::ImmTyScaleSel);
9830 AMDGPUOperand::ImmTyClamp);
9836 AMDGPUOperand::ImmTyByteSel);
9841 AMDGPUOperand::ImmTyOModSI);
9848 auto *it = Inst.
begin();
9850 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9859 OptionalImmIndexMap OptionalIdx;
9860 cvtVOP3(Inst,
Operands, OptionalIdx);
9864 OptionalImmIndexMap &OptIdx) {
9869 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9870 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9871 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9872 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9873 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9874 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9875 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9876 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9877 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9878 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9887 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9888 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9891 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9892 if (BitOp3Idx != -1) {
9899 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9900 if (OpSelIdx != -1) {
9904 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9905 if (OpSelHiIdx != -1) {
9906 int DefaultVal =
IsPacked ? -1 : 0;
9912 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9913 if (MatrixAFMTIdx != -1) {
9915 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9919 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9920 if (MatrixBFMTIdx != -1) {
9922 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9925 int MatrixAScaleIdx =
9926 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9927 if (MatrixAScaleIdx != -1) {
9929 AMDGPUOperand::ImmTyMatrixAScale, 0);
9932 int MatrixBScaleIdx =
9933 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9934 if (MatrixBScaleIdx != -1) {
9936 AMDGPUOperand::ImmTyMatrixBScale, 0);
9939 int MatrixAScaleFmtIdx =
9940 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9941 if (MatrixAScaleFmtIdx != -1) {
9943 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9946 int MatrixBScaleFmtIdx =
9947 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9948 if (MatrixBScaleFmtIdx != -1) {
9950 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9955 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9959 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9961 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9965 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9969 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9970 AMDGPU::OpName::src2};
9971 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9972 AMDGPU::OpName::src1_modifiers,
9973 AMDGPU::OpName::src2_modifiers};
9976 unsigned OpSelHi = 0;
9983 if (OpSelHiIdx != -1)
9992 for (
int J = 0; J < 3; ++J) {
9993 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9997 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
10007 uint32_t ModVal = 0;
10009 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
10010 if (SrcOp.
isReg() && getMRI()
10014 if (VGPRSuffixIsHi)
10017 if ((OpSel & (1 << J)) != 0)
10021 if ((OpSelHi & (1 << J)) != 0)
10024 if ((NegLo & (1 << J)) != 0)
10027 if ((NegHi & (1 << J)) != 0)
10035 OptionalImmIndexMap OptIdx;
10041 unsigned i,
unsigned Opc,
10042 AMDGPU::OpName
OpName) {
10043 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
10044 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
10046 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
10052 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10055 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10056 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10058 OptionalImmIndexMap OptIdx;
10059 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10060 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10061 OptIdx[
Op.getImmTy()] = i;
10066 AMDGPUOperand::ImmTyIndexKey8bit);
10070 AMDGPUOperand::ImmTyIndexKey16bit);
10074 AMDGPUOperand::ImmTyIndexKey32bit);
10091 SMLoc S = getLoc();
10094 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10095 SMLoc OpYLoc = getLoc();
10098 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10101 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10110 auto addOp = [&](uint16_t ParsedOprIdx) {
10111 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10113 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10117 Op.addRegOperands(Inst, 1);
10120 if (
Op.isImm() ||
Op.isExpr()) {
10121 Op.addImmOperands(Inst, 1);
10133 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10137 const auto &CInfo = InstInfo[CompIdx];
10138 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10139 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10140 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10141 if (CInfo.hasSrc2Acc())
10142 addOp(CInfo.getIndexOfDstInParsedOperands());
10146 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10147 if (BitOp3Idx != -1) {
10148 OptionalImmIndexMap OptIdx;
10149 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10151 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10161bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10163bool AMDGPUOperand::isDPPCtrl()
const {
10164 using namespace AMDGPU::DPP;
10166 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10169 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10170 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10171 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10172 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10173 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10174 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10175 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10176 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10177 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10178 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10179 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10180 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10189bool AMDGPUOperand::isBLGP()
const {
10193bool AMDGPUOperand::isS16Imm()
const {
10197bool AMDGPUOperand::isU16Imm()
const {
10205bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10210 SMLoc Loc =
getToken().getEndLoc();
10211 Token = std::string(getTokenStr());
10213 if (getLoc() != Loc)
10218 if (!parseId(Suffix))
10222 StringRef DimId = Token;
10237 SMLoc S = getLoc();
10243 SMLoc Loc = getLoc();
10244 if (!parseDimId(Encoding))
10245 return Error(Loc,
"invalid dim value");
10248 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10257 SMLoc S = getLoc();
10266 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10269 for (
size_t i = 0; i < 8; ++i) {
10273 SMLoc Loc = getLoc();
10274 if (getParser().parseAbsoluteExpression(Sels[i]))
10276 if (0 > Sels[i] || 7 < Sels[i])
10277 return Error(Loc,
"expected a 3-bit value");
10280 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10284 for (
size_t i = 0; i < 8; ++i)
10285 DPP8 |= (Sels[i] << (i * 3));
10288 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10292bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10294 if (Ctrl ==
"row_newbcast")
10297 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10300 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10301 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10304 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10305 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10309int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10312 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10316 for (
int i = 0; i < 4; ++i) {
10321 SMLoc Loc = getLoc();
10322 if (getParser().parseAbsoluteExpression(Temp))
10324 if (Temp < 0 || Temp > 3) {
10325 Error(Loc,
"expected a 2-bit value");
10329 Val += (Temp << i * 2);
10332 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10338int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10339 using namespace AMDGPU::DPP;
10344 SMLoc Loc = getLoc();
10346 if (getParser().parseAbsoluteExpression(Val))
10349 struct DppCtrlCheck {
10355 DppCtrlCheck
Check =
10356 StringSwitch<DppCtrlCheck>(Ctrl)
10357 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10358 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10359 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10360 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10361 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10362 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10363 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10364 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10365 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10366 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10370 if (
Check.Ctrl == -1) {
10371 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10379 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10387 using namespace AMDGPU::DPP;
10390 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10393 SMLoc S = getLoc();
10399 if (Ctrl ==
"row_mirror") {
10400 Val = DppCtrl::ROW_MIRROR;
10401 }
else if (Ctrl ==
"row_half_mirror") {
10402 Val = DppCtrl::ROW_HALF_MIRROR;
10405 if (Ctrl ==
"quad_perm") {
10406 Val = parseDPPCtrlPerm();
10408 Val = parseDPPCtrlSel(Ctrl);
10417 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10423 OptionalImmIndexMap OptionalIdx;
10430 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10432 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10433 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10437 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10438 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10442 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10443 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10444 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10445 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10446 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10447 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10448 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10449 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10450 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10456 if (OldIdx == NumOperands) {
10458 constexpr int DST_IDX = 0;
10460 }
else if (Src2ModIdx == NumOperands) {
10470 if (IsVOP3CvtSrDpp) {
10479 if (TiedTo != -1) {
10484 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10486 if (IsDPP8 &&
Op.isDppFI()) {
10489 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10490 }
else if (
Op.isReg()) {
10491 Op.addRegOperands(Inst, 1);
10492 }
else if (
Op.isImm() &&
10494 Op.addImmOperands(Inst, 1);
10495 }
else if (
Op.isImm()) {
10496 OptionalIdx[
Op.getImmTy()] =
I;
10504 AMDGPUOperand::ImmTyClamp);
10510 AMDGPUOperand::ImmTyByteSel);
10515 AMDGPUOperand::ImmTyOModSI);
10518 cvtVOP3P(Inst,
Operands, OptionalIdx);
10520 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10523 AMDGPUOperand::ImmTyOpSel);
10528 AMDGPUOperand::ImmTyDPP8);
10529 using namespace llvm::AMDGPU::DPP;
10533 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10535 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10537 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10539 AMDGPUOperand::ImmTyDppBoundCtrl);
10543 AMDGPUOperand::ImmTyDppFI);
10549 OptionalImmIndexMap OptionalIdx;
10553 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10554 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10561 if (TiedTo != -1) {
10566 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10568 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10576 Op.addImmOperands(Inst, 1);
10578 Op.addRegWithFPInputModsOperands(Inst, 2);
10579 }
else if (
Op.isDppFI()) {
10581 }
else if (
Op.isReg()) {
10582 Op.addRegOperands(Inst, 1);
10588 Op.addRegWithFPInputModsOperands(Inst, 2);
10589 }
else if (
Op.isReg()) {
10590 Op.addRegOperands(Inst, 1);
10591 }
else if (
Op.isDPPCtrl()) {
10592 Op.addImmOperands(Inst, 1);
10593 }
else if (
Op.isImm()) {
10595 OptionalIdx[
Op.getImmTy()] =
I;
10603 using namespace llvm::AMDGPU::DPP;
10607 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10609 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10611 AMDGPUOperand::ImmTyDppBoundCtrl);
10614 AMDGPUOperand::ImmTyDppFI);
10625 AMDGPUOperand::ImmTy
Type) {
10626 return parseStringOrIntWithPrefix(
10628 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10633 return parseStringOrIntWithPrefix(
10634 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10635 AMDGPUOperand::ImmTySDWADstUnused);
10639 cvtSDWA(Inst,
Operands, SDWAInstType::VOP1);
10643 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2);
10646void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10648 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
true,
true);
10651void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10653 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
false,
true);
10661 SDWAInstType BasicInstType,
bool SkipDstVcc,
10663 using namespace llvm::AMDGPU::SDWA;
10665 OptionalImmIndexMap OptionalIdx;
10666 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10667 bool SkippedVcc =
false;
10671 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10672 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10676 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10677 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10678 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10684 if (BasicInstType == SDWAInstType::VOP2 &&
10690 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10696 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10697 }
else if (
Op.isImm()) {
10699 OptionalIdx[
Op.getImmTy()] =
I;
10703 SkippedVcc =
false;
10707 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10708 Opc != AMDGPU::V_NOP_sdwa_vi) {
10710 switch (BasicInstType) {
10711 case SDWAInstType::VOP1:
10714 AMDGPUOperand::ImmTyClamp, 0);
10718 AMDGPUOperand::ImmTyOModSI, 0);
10722 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10726 AMDGPUOperand::ImmTySDWADstUnused,
10727 DstUnused::UNUSED_PRESERVE);
10730 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10733 case SDWAInstType::VOP2:
10735 AMDGPUOperand::ImmTyClamp, 0);
10739 AMDGPUOperand::ImmTyOModSI, 0);
10742 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10744 AMDGPUOperand::ImmTySDWADstUnused,
10745 DstUnused::UNUSED_PRESERVE);
10747 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10749 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10752 case SDWAInstType::VOPC:
10755 AMDGPUOperand::ImmTyClamp, 0);
10757 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10759 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10766 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10767 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10768 auto *it = Inst.
begin();
10770 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10783#define GET_MATCHER_IMPLEMENTATION
10784#define GET_MNEMONIC_SPELL_CHECKER
10785#define GET_MNEMONIC_CHECKER
10786#include "AMDGPUGenAsmMatcher.inc"
10792 return parseTokenOp(
"addr64",
Operands);
10794 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10796 return parseTokenOp(
"idxen",
Operands);
10798 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10801 return parseTokenOp(
"offen",
Operands);
10803 return parseTokenOp(
"off",
Operands);
10804 case MCK_row_95_en:
10805 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10807 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10809 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10811 return tryCustomParseOperand(
Operands, MCK);
10816unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10822 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10825 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10827 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10829 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10831 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10833 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10835 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10837 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10838 case MCK_row_95_en:
10839 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10847 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10849 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10850 case MCK_SOPPBrTarget:
10851 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10852 case MCK_VReg32OrOff:
10853 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10854 case MCK_InterpSlot:
10855 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10856 case MCK_InterpAttr:
10857 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10858 case MCK_InterpAttrChan:
10859 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10861 case MCK_SReg_64_XEXEC:
10871 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10873 return Match_InvalidOperand;
10882 SMLoc S = getLoc();
10891 return Error(S,
"expected a 16-bit value");
10894 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10898bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10904bool AMDGPUOperand::isSplitBarrier()
const {
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static Triple::ObjectFormatType parseFormat(StringRef EnvironmentName)
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
const int16_t * getRegClassByHwModeTable(unsigned ModeId) const
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
LLVM_ABI unsigned getTotalNumVGPRs(GPUKind AK, bool IsWave32)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
GPUKind
GPU kinds supported by the AMDGPU target.
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_IMM_NOINLINE_FP16
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
constexpr bool isBF16SrcOperand(const MCOperandInfo &OpInfo)
Is this a scalar (i.e. not packed) bf16 source operand?
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
EnumSet< Modifier > Modifiers
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size