71 enum KindTy { Token, Immediate, Register, Expression } Kind;
73 SMLoc StartLoc, EndLoc;
74 const AMDGPUAsmParser *AsmParser;
77 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
78 : Kind(Kind_), AsmParser(AsmParser_) {}
80 using Ptr = std::unique_ptr<AMDGPUOperand>;
88 bool hasFPModifiers()
const {
return Abs || Neg; }
89 bool hasIntModifiers()
const {
return Sext; }
90 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
91 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
92 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
94 int64_t getFPModifiersOperand()
const {
101 int64_t getIntModifiersOperand()
const {
107 int64_t getModifiersOperand()
const {
108 assert(!(hasFPModifiers() && hasIntModifiers()) &&
109 "fp and int modifiers should not be used simultaneously");
110 if (hasFPModifiers())
111 return getFPModifiersOperand();
112 if (hasIntModifiers())
113 return getIntModifiersOperand();
117 friend raw_ostream &
operator<<(raw_ostream &OS,
118 AMDGPUOperand::Modifiers Mods);
192 ImmTyMatrixAScaleFmt,
193 ImmTyMatrixBScaleFmt,
226 mutable int MCOpIdx = -1;
229 bool isToken()
const override {
return Kind == Token; }
231 bool isSymbolRefExpr()
const {
235 bool isImm()
const override {
return Kind == Immediate; }
237 bool isInlinableImm(MVT type)
const;
238 bool isLiteralImm(MVT type)
const;
240 bool isRegKind()
const {
return Kind == Register; }
242 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
244 bool isRegOrInline(
unsigned RCID, MVT type)
const {
245 return isRegClass(RCID) || isInlinableImm(type);
249 return isRegOrInline(RCID, type) || isLiteralImm(type);
252 bool isRegOrImmWithInt16InputMods()
const {
256 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
258 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
261 bool isRegOrImmWithInt32InputMods()
const {
265 bool isRegOrInlineImmWithInt16InputMods()
const {
266 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
269 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
270 return isRegOrInline(
271 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
274 bool isRegOrInlineImmWithInt32InputMods()
const {
275 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
278 bool isRegOrImmWithInt64InputMods()
const {
282 bool isRegOrImmWithFP16InputMods()
const {
286 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
288 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
291 bool isRegOrImmWithFP32InputMods()
const {
295 bool isRegOrImmWithFP64InputMods()
const {
299 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
300 return isRegOrInline(
301 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
304 bool isRegOrInlineImmWithFP32InputMods()
const {
305 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
308 bool isRegOrInlineImmWithFP64InputMods()
const {
309 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
312 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
314 bool isVRegWithFP32InputMods()
const {
315 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
318 bool isVRegWithFP64InputMods()
const {
319 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
322 bool isPackedFP16InputMods()
const {
326 bool isPackedVGPRFP32InputMods()
const {
330 bool isVReg()
const {
331 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
332 isRegClass(AMDGPU::VReg_64RegClassID) ||
333 isRegClass(AMDGPU::VReg_96RegClassID) ||
334 isRegClass(AMDGPU::VReg_128RegClassID) ||
335 isRegClass(AMDGPU::VReg_160RegClassID) ||
336 isRegClass(AMDGPU::VReg_192RegClassID) ||
337 isRegClass(AMDGPU::VReg_256RegClassID) ||
338 isRegClass(AMDGPU::VReg_512RegClassID) ||
339 isRegClass(AMDGPU::VReg_1024RegClassID);
342 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
344 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
346 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
348 bool isAV_LdSt_32_Align2_RegOp()
const {
349 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
350 isRegClass(AMDGPU::AGPR_32RegClassID);
353 bool isVRegWithInputMods()
const;
354 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
355 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
357 bool isSDWAOperand(MVT type)
const;
358 bool isSDWAFP16Operand()
const;
359 bool isSDWAFP32Operand()
const;
360 bool isSDWAInt16Operand()
const;
361 bool isSDWAInt32Operand()
const;
363 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
365 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
367 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
369 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
371 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
372 bool isDim()
const {
return isImmTy(ImmTyDim); }
373 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
374 bool isOff()
const {
return isImmTy(ImmTyOff); }
375 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
376 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
377 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
378 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
379 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
380 bool isFlatOffset()
const {
381 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
383 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
384 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
385 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
386 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
387 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
388 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
389 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
390 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
391 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
392 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
393 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
394 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
395 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
396 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
397 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
398 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
399 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
400 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
401 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
402 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
403 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
404 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
405 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
406 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
407 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
408 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
409 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
410 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
411 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
412 bool isDone()
const {
return isImmTy(ImmTyDone); }
413 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
415 bool isRegOrImm()
const {
return isReg() || isImm(); }
417 bool isRegClass(
unsigned RCID)
const;
421 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
422 return isRegOrInline(RCID, type) && !hasModifiers();
425 bool isSCSrcB16()
const {
426 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
429 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
431 bool isSCSrc_b32()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
435 bool isSCSrc_b64()
const {
436 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
439 bool isBoolReg()
const;
441 bool isSCSrcF16()
const {
442 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
445 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
447 bool isSCSrcF32()
const {
448 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
451 bool isSCSrcF64()
const {
452 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
455 bool isSSrc_b32()
const {
456 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
459 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
461 bool isSSrcV2B16()
const {
466 bool isSSrc_b64()
const {
469 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
470 (((
const MCTargetAsmParser *)AsmParser)
471 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
475 bool isSSrc_f32()
const {
476 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
479 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
481 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
483 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
485 bool isSSrcV2F16()
const {
490 bool isSSrcV2FP32()
const {
495 bool isSCSrcV2FP32()
const {
500 bool isSSrcV2INT32()
const {
505 bool isSCSrcV2INT32()
const {
507 return isSCSrc_b32();
510 bool isSSrcOrLds_b32()
const {
511 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
512 isLiteralImm(MVT::i32) || isExpr();
515 bool isVCSrc_b32()
const {
516 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
519 bool isVCSrc_b32_Lo256()
const {
520 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
523 bool isVCSrc_b64_Lo256()
const {
524 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
527 bool isVCSrc_b64()
const {
528 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
531 bool isVCSrcT_b16()
const {
532 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
535 bool isVCSrcTB16_Lo128()
const {
536 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
539 bool isVCSrcFake16B16_Lo128()
const {
540 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
543 bool isVCSrc_b16()
const {
544 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
547 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
549 bool isVCSrc_f32()
const {
550 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
553 bool isVCSrc_f64()
const {
554 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
557 bool isVCSrcTBF16()
const {
558 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
561 bool isVCSrcT_f16()
const {
562 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
565 bool isVCSrcT_bf16()
const {
566 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
569 bool isVCSrcTBF16_Lo128()
const {
570 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
573 bool isVCSrcTF16_Lo128()
const {
574 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
577 bool isVCSrcFake16BF16_Lo128()
const {
578 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
581 bool isVCSrcFake16F16_Lo128()
const {
582 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
585 bool isVCSrc_bf16()
const {
586 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
589 bool isVCSrc_f16()
const {
590 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
593 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
595 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
597 bool isVSrc_b32()
const {
598 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
601 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
603 bool isVSrc_v2b64()
const {
604 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
605 isLiteralImm(MVT::i64);
608 bool isVSrc_v2f64()
const {
609 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
610 isLiteralImm(MVT::f64);
613 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
615 bool isVSrcT_b16_Lo128()
const {
616 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
619 bool isVSrcFake16_b16_Lo128()
const {
620 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
623 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
625 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
627 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
629 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
631 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
633 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
635 bool isVSrc_f32()
const {
636 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
639 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
641 bool isVSrcT_bf16()
const {
642 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
645 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
647 bool isVSrcT_bf16_Lo128()
const {
648 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
651 bool isVSrcT_f16_Lo128()
const {
652 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
655 bool isVSrcFake16_bf16_Lo128()
const {
656 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
659 bool isVSrcFake16_f16_Lo128()
const {
660 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
663 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
665 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
667 bool isVSrc_v2bf16()
const {
668 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
671 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
673 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
675 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
677 bool isVISrcB32()
const {
678 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
681 bool isVISrcB16()
const {
682 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
685 bool isVISrcV2B16()
const {
return isVISrcB16(); }
687 bool isVISrcF32()
const {
688 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
691 bool isVISrcF16()
const {
692 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
695 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
697 bool isVISrc_64_bf16()
const {
698 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
701 bool isVISrc_64_f16()
const {
702 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
705 bool isVISrc_64_b32()
const {
706 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
709 bool isVISrc_64B64()
const {
710 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
713 bool isVISrc_64_f64()
const {
714 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
717 bool isVISrc_64V2FP32()
const {
718 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
721 bool isVISrc_64V2INT32()
const {
722 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
725 bool isVISrc_256_b32()
const {
726 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
729 bool isVISrc_256_f32()
const {
730 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
733 bool isVISrc_256B64()
const {
734 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
737 bool isVISrc_256_f64()
const {
738 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
741 bool isVISrc_512_f64()
const {
742 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
745 bool isVISrc_128B16()
const {
746 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
749 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
751 bool isVISrc_128_b32()
const {
752 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
755 bool isVISrc_128_f32()
const {
756 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
759 bool isVISrc_256V2FP32()
const {
760 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
763 bool isVISrc_256V2INT32()
const {
764 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
767 bool isVISrc_512_b32()
const {
768 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
771 bool isVISrc_512B16()
const {
772 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
775 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
777 bool isVISrc_512_f32()
const {
778 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
781 bool isVISrc_512F16()
const {
782 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
785 bool isVISrc_512V2F16()
const {
786 return isVISrc_512F16() || isVISrc_512_b32();
789 bool isVISrc_1024_b32()
const {
790 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
793 bool isVISrc_1024B16()
const {
794 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
797 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
799 bool isVISrc_1024_f32()
const {
800 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
803 bool isVISrc_1024F16()
const {
804 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
807 bool isVISrc_1024V2F16()
const {
808 return isVISrc_1024F16() || isVISrc_1024_b32();
811 bool isAISrcB32()
const {
812 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
815 bool isAISrcB16()
const {
816 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
819 bool isAISrcV2B16()
const {
return isAISrcB16(); }
821 bool isAISrcF32()
const {
822 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
825 bool isAISrcF16()
const {
826 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
829 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
831 bool isAISrc_64B64()
const {
832 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
835 bool isAISrc_64_f64()
const {
836 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
839 bool isAISrc_128_b32()
const {
840 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
843 bool isAISrc_128B16()
const {
844 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
847 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
849 bool isAISrc_128_f32()
const {
850 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
853 bool isAISrc_128F16()
const {
854 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
857 bool isAISrc_128V2F16()
const {
858 return isAISrc_128F16() || isAISrc_128_b32();
861 bool isVISrc_128_bf16()
const {
862 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
865 bool isVISrc_128_f16()
const {
866 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
869 bool isVISrc_128V2F16()
const {
870 return isVISrc_128_f16() || isVISrc_128_b32();
873 bool isAISrc_256B64()
const {
874 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
877 bool isAISrc_256_f64()
const {
878 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
881 bool isAISrc_512_b32()
const {
882 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
885 bool isAISrc_512B16()
const {
886 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
889 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
891 bool isAISrc_512_f32()
const {
892 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
895 bool isAISrc_512F16()
const {
896 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
899 bool isAISrc_512V2F16()
const {
900 return isAISrc_512F16() || isAISrc_512_b32();
903 bool isAISrc_1024_b32()
const {
904 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
907 bool isAISrc_1024B16()
const {
908 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
911 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
913 bool isAISrc_1024_f32()
const {
914 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
917 bool isAISrc_1024F16()
const {
918 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
921 bool isAISrc_1024V2F16()
const {
922 return isAISrc_1024F16() || isAISrc_1024_b32();
925 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
927 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
929 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
931 bool isMem()
const override {
return false; }
933 bool isExpr()
const {
return Kind == Expression; }
935 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
937 bool isSWaitCnt()
const;
938 bool isDepCtr()
const;
939 bool isSDelayALU()
const;
940 bool isHwreg()
const;
941 bool isSendMsg()
const;
942 bool isWaitEvent()
const;
943 bool isSplitBarrier()
const;
944 bool isSwizzle()
const;
945 bool isSMRDOffset8()
const;
946 bool isSMEMOffset()
const;
947 bool isSMRDLiteralOffset()
const;
949 bool isDPPCtrl()
const;
951 bool isGPRIdxMode()
const;
952 bool isS16Imm()
const;
953 bool isU16Imm()
const;
954 bool isEndpgm()
const;
956 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
957 return [
this,
P]() {
return P(*
this); };
962 return StringRef(Tok.Data, Tok.Length);
970 void setImm(int64_t Val) {
975 ImmTy getImmTy()
const {
980 MCRegister
getReg()
const override {
985 SMLoc getStartLoc()
const override {
return StartLoc; }
987 SMLoc getEndLoc()
const override {
return EndLoc; }
989 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
991 int getMCOpIdx()
const {
return MCOpIdx; }
993 Modifiers getModifiers()
const {
994 assert(isRegKind() || isImmTy(ImmTyNone));
995 return isRegKind() ?
Reg.Mods :
Imm.Mods;
998 void setModifiers(Modifiers Mods) {
999 assert(isRegKind() || isImmTy(ImmTyNone));
1006 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1008 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1010 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1012 bool isForcedLit()
const {
1013 return isImmLiteral() && getModifiers().isForcedLit();
1016 bool isForcedLit64()
const {
1017 return isImmLiteral() && getModifiers().isForcedLit64();
1020 uint64_t applyInputFPModifiers(uint64_t Val,
unsigned Size)
const;
1022 void addImmOperands(MCInst &Inst,
unsigned N,
1023 bool ApplyModifiers =
true)
const;
1025 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1026 bool ApplyModifiers)
const;
1028 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1030 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1032 addRegOperands(Inst,
N);
1034 addImmOperands(Inst,
N);
1037 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1038 Modifiers Mods = getModifiers();
1041 addRegOperands(Inst,
N);
1043 addImmOperands(Inst,
N,
false);
1047 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1048 assert(!hasIntModifiers());
1049 addRegOrImmWithInputModsOperands(Inst,
N);
1052 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1053 assert(!hasFPModifiers());
1054 addRegOrImmWithInputModsOperands(Inst,
N);
1057 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1058 Modifiers Mods = getModifiers();
1061 addRegOperands(Inst,
N);
1064 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1065 assert(!hasIntModifiers());
1066 addRegWithInputModsOperands(Inst,
N);
1069 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1070 assert(!hasFPModifiers());
1071 addRegWithInputModsOperands(Inst,
N);
1074 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1077 case ImmTyNone: OS <<
"None";
break;
1078 case ImmTyGDS: OS <<
"GDS";
break;
1079 case ImmTyLDS: OS <<
"LDS";
break;
1080 case ImmTyOffen: OS <<
"Offen";
break;
1081 case ImmTyIdxen: OS <<
"Idxen";
break;
1082 case ImmTyAddr64: OS <<
"Addr64";
break;
1083 case ImmTyOffset: OS <<
"Offset";
break;
1084 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1085 case ImmTyOffset0: OS <<
"Offset0";
break;
1086 case ImmTyOffset1: OS <<
"Offset1";
break;
1087 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1088 case ImmTyCPol: OS <<
"CPol";
break;
1089 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1090 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1091 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1092 case ImmTyTFE: OS <<
"TFE";
break;
1093 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1094 case ImmTyD16: OS <<
"D16";
break;
1095 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1096 case ImmTyClamp: OS <<
"Clamp";
break;
1097 case ImmTyOModSI: OS <<
"OModSI";
break;
1098 case ImmTyDPP8: OS <<
"DPP8";
break;
1099 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1100 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1101 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1102 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1103 case ImmTyDppFI: OS <<
"DppFI";
break;
1104 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1105 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1106 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1107 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1108 case ImmTyDMask: OS <<
"DMask";
break;
1109 case ImmTyDim: OS <<
"Dim";
break;
1110 case ImmTyUNorm: OS <<
"UNorm";
break;
1111 case ImmTyDA: OS <<
"DA";
break;
1112 case ImmTyR128A16: OS <<
"R128A16";
break;
1113 case ImmTyA16: OS <<
"A16";
break;
1114 case ImmTyLWE: OS <<
"LWE";
break;
1115 case ImmTyOff: OS <<
"Off";
break;
1116 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1117 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1118 case ImmTyExpVM: OS <<
"ExpVM";
break;
1119 case ImmTyDone: OS <<
"Done";
break;
1120 case ImmTyRowEn: OS <<
"RowEn";
break;
1121 case ImmTyHwreg: OS <<
"Hwreg";
break;
1122 case ImmTySendMsg: OS <<
"SendMsg";
break;
1123 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1124 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1125 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1126 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1127 case ImmTyOpSel: OS <<
"OpSel";
break;
1128 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1129 case ImmTyNegLo: OS <<
"NegLo";
break;
1130 case ImmTyNegHi: OS <<
"NegHi";
break;
1131 case ImmTySwizzle: OS <<
"Swizzle";
break;
1132 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1133 case ImmTyHigh: OS <<
"High";
break;
1134 case ImmTyBLGP: OS <<
"BLGP";
break;
1135 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1136 case ImmTyABID: OS <<
"ABID";
break;
1137 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1138 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1139 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1140 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1141 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1142 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1143 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1144 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1145 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1146 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1147 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1148 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1149 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1150 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1151 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1152 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1157 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1161 <<
" mods: " <<
Reg.Mods <<
'>';
1165 if (getImmTy() != ImmTyNone) {
1167 printImmTy(OS, getImmTy());
1169 OS <<
" mods: " <<
Imm.Mods <<
'>';
1182 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1183 int64_t Val, SMLoc Loc,
1184 ImmTy
Type = ImmTyNone,
1185 bool IsFPImm =
false) {
1186 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1188 Op->Imm.IsFPImm = IsFPImm;
1190 Op->Imm.Mods = Modifiers();
1196 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1197 StringRef Str, SMLoc Loc,
1198 bool HasExplicitEncodingSize =
true) {
1199 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1200 Res->Tok.Data = Str.data();
1201 Res->Tok.Length = Str.size();
1202 Res->StartLoc = Loc;
1207 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1208 MCRegister
Reg, SMLoc S, SMLoc
E) {
1209 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1210 Op->Reg.RegNo =
Reg;
1211 Op->Reg.Mods = Modifiers();
1217 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1218 const class MCExpr *Expr, SMLoc S) {
1219 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1228 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1237#define GET_REGISTER_MATCHER
1238#include "AMDGPUGenAsmMatcher.inc"
1239#undef GET_REGISTER_MATCHER
1240#undef GET_SUBTARGET_FEATURE_NAME
1245class KernelScopeInfo {
1246 int SgprIndexUnusedMin = -1;
1247 int VgprIndexUnusedMin = -1;
1248 int AgprIndexUnusedMin = -1;
1252 void usesSgprAt(
int i) {
1253 if (i >= SgprIndexUnusedMin) {
1254 SgprIndexUnusedMin = ++i;
1257 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1263 void usesVgprAt(
int i) {
1264 if (i >= VgprIndexUnusedMin) {
1265 VgprIndexUnusedMin = ++i;
1268 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1270 VgprIndexUnusedMin);
1276 void usesAgprAt(
int i) {
1281 if (i >= AgprIndexUnusedMin) {
1282 AgprIndexUnusedMin = ++i;
1285 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1290 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1292 VgprIndexUnusedMin);
1299 KernelScopeInfo() =
default;
1303 MSTI = Ctx->getSubtargetInfo();
1305 usesSgprAt(SgprIndexUnusedMin = -1);
1306 usesVgprAt(VgprIndexUnusedMin = -1);
1308 usesAgprAt(AgprIndexUnusedMin = -1);
1312 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1313 unsigned RegWidth) {
1316 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1319 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1322 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1331 MCAsmParser &Parser;
1333 unsigned ForcedEncodingSize = 0;
1334 bool ForcedDPP =
false;
1335 bool ForcedSDWA =
false;
1336 KernelScopeInfo KernelScope;
1337 const unsigned HwMode;
1339 const AMDGPU::IsaVersion ISA;
1344#define GET_ASSEMBLER_HEADER
1345#include "AMDGPUGenAsmMatcher.inc"
1350 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1352 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1356 std::optional<AMDGPU::InfoSectionData> InfoData;
1363 bool TargetDirectiveEmitted =
false;
1372 SmallVector<unsigned> OpcodeStream;
1374 OpcodeStreamSymbols;
1375 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1378 void checkKernelPrologues();
1381 void createConstantSymbol(StringRef Id, int64_t Val);
1383 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1384 bool OutOfRangeError(SMRange
Range);
1400 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1401 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1402 std::optional<bool> EnableWavefrontSize32,
1403 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1404 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1405 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1406 bool ParseDirectiveAMDGCNTarget();
1407 bool ParseDirectiveAMDHSACodeObjectVersion();
1408 bool ParseDirectiveAMDHSAKernel();
1409 bool ParseAMDKernelCodeTValue(StringRef
ID, AMDGPUMCKernelCodeT &Header);
1410 bool ParseDirectiveAMDKernelCodeT();
1412 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1413 bool ParseDirectiveAMDGPUHsaKernel();
1415 bool ParseDirectiveISAVersion();
1416 bool ParseDirectiveHSAMetadata();
1417 bool ParseDirectivePALMetadataBegin();
1418 bool ParseDirectivePALMetadata();
1419 bool ParseDirectiveAMDGPULDS();
1420 bool ParseDirectiveAMDGPUInfo();
1424 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1425 const char *AssemblerDirectiveEnd,
1426 std::string &CollectString);
1428 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1429 RegisterKind RegKind, MCRegister Reg1,
1430 RegisterKind RegKind1, SMLoc Loc);
1431 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1432 unsigned &RegNum,
unsigned &RegWidth,
1433 bool RestoreOnFailure =
false);
1434 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1435 unsigned &RegNum,
unsigned &RegWidth,
1436 SmallVectorImpl<AsmToken> &Tokens);
1437 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1439 SmallVectorImpl<AsmToken> &Tokens);
1440 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1442 SmallVectorImpl<AsmToken> &Tokens);
1443 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1445 SmallVectorImpl<AsmToken> &Tokens);
1446 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1447 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1448 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1451 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1452 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1453 void initializeGprCountSymbol(RegisterKind RegKind);
1454 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1456 void cvtMubufImpl(MCInst &Inst,
const OperandVector &Operands,
bool IsAtomic);
1460 OperandMode_Default,
1464 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1466 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1467 const MCInstrInfo &MII)
1468 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1469 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1474 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1476 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1477 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1478 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1479 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1481 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1482 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1483 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1485 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1486 initializeGprCountSymbol(IS_VGPR);
1487 initializeGprCountSymbol(IS_SGPR);
1492 createConstantSymbol(Symbol, Code);
1494 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1495 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1496 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1544 bool hasBVHRayTracingInsts()
const {
1545 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1550 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1552 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1554 bool hasInv2PiInlineImm()
const {
1555 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1558 bool has64BitLiterals()
const {
1559 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1562 bool hasFlatOffsets()
const {
1563 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1566 bool hasTrue16Insts()
const {
1567 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1571 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1574 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1576 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1578 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1580 bool hasPartialNSAEncoding()
const {
1581 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1584 bool hasGloballyAddressableScratch()
const {
1585 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1598 AMDGPUTargetStreamer &getTargetStreamer() {
1599 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1600 return static_cast<AMDGPUTargetStreamer &
>(TS);
1606 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1609 const MCRegisterInfo *getMRI()
const {
1613 const MCInstrInfo *getMII()
const {
return &MII; }
1617 const FeatureBitset &getFeatureBits()
const {
1618 return getSTI().getFeatureBits();
1621 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1622 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1623 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1625 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1626 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1627 bool isForcedDPP()
const {
return ForcedDPP; }
1628 bool isForcedSDWA()
const {
return ForcedSDWA; }
1629 ArrayRef<unsigned> getMatchedVariants()
const;
1630 StringRef getMatchedVariantName()
const;
1632 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1633 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1634 bool RestoreOnFailure);
1635 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1636 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1637 SMLoc &EndLoc)
override;
1638 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1639 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1640 unsigned Kind)
override;
1641 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1643 uint64_t &ErrorInfo,
1644 bool MatchingInlineAsm)
override;
1645 bool ParseDirective(AsmToken DirectiveID)
override;
1646 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1647 void onEndOfFile()
override;
1648 ParseStatus parseOperand(
OperandVector &Operands, StringRef Mnemonic,
1649 OperandMode
Mode = OperandMode_Default);
1650 StringRef parseMnemonicSuffix(StringRef Name);
1651 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1655 ParseStatus parseTokenOp(StringRef Name,
OperandVector &Operands);
1657 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1660 parseIntWithPrefix(
const char *Prefix,
OperandVector &Operands,
1661 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1662 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1664 ParseStatus parseOperandArrayWithPrefix(
1666 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1667 bool (*ConvertResult)(int64_t &) =
nullptr);
1671 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1672 bool IgnoreNegative =
false);
1673 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1675 ParseStatus parseScope(
OperandVector &Operands, int64_t &Scope);
1677 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1679 ParseStatus parseStringOrIntWithPrefix(
OperandVector &Operands,
1681 ArrayRef<const char *> Ids,
1683 ParseStatus parseStringOrIntWithPrefix(
OperandVector &Operands,
1685 ArrayRef<const char *> Ids,
1686 AMDGPUOperand::ImmTy
Type);
1689 bool isOperandModifier(
const AsmToken &Token,
1690 const AsmToken &NextToken)
const;
1691 bool isRegOrOperandModifier(
const AsmToken &Token,
1692 const AsmToken &NextToken)
const;
1693 bool isNamedOperandModifier(
const AsmToken &Token,
1694 const AsmToken &NextToken)
const;
1695 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1696 const AsmToken &NextToken)
const;
1697 bool parseSP3NegModifier();
1698 ParseStatus parseImm(
OperandVector &Operands,
bool HasSP3AbsModifier =
false,
1701 ParseStatus parseRegOrImm(
OperandVector &Operands,
bool HasSP3AbsMod =
false,
1703 ParseStatus parseRegOrImmWithFPInputMods(
OperandVector &Operands,
1704 bool AllowImm =
true);
1705 ParseStatus parseRegOrImmWithIntInputMods(
OperandVector &Operands,
1706 bool AllowImm =
true);
1707 ParseStatus parseRegWithFPInputMods(
OperandVector &Operands);
1708 ParseStatus parseRegWithIntInputMods(
OperandVector &Operands);
1711 AMDGPUOperand::ImmTy ImmTy);
1715 ParseStatus tryParseMatrixFMT(
OperandVector &Operands, StringRef Name,
1716 AMDGPUOperand::ImmTy
Type);
1719 ParseStatus tryParseMatrixScale(
OperandVector &Operands, StringRef Name,
1720 AMDGPUOperand::ImmTy
Type);
1723 ParseStatus tryParseMatrixScaleFmt(
OperandVector &Operands, StringRef Name,
1724 AMDGPUOperand::ImmTy
Type);
1728 ParseStatus parseDfmtNfmt(int64_t &
Format);
1729 ParseStatus parseUfmt(int64_t &
Format);
1730 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1732 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1735 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1736 ParseStatus parseNumericFormat(int64_t &
Format);
1740 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1741 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1746 bool parseCnt(int64_t &IntVal);
1749 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1750 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1753 bool parseDelay(int64_t &Delay);
1759 struct OperandInfoTy {
1762 bool IsSymbolic =
false;
1763 bool IsDefined =
false;
1765 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1768 struct StructuredOpField : OperandInfoTy {
1772 bool IsDefined =
false;
1774 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1775 unsigned Width, int64_t
Default)
1776 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1777 virtual ~StructuredOpField() =
default;
1779 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1780 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1784 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1786 return Error(Parser,
"not supported on this GPU");
1788 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1796 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1797 OperandInfoTy &Stream);
1798 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1799 const OperandInfoTy &Stream);
1801 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1802 OperandInfoTy &Width);
1804 const AMDGPUOperand &findMCOperand(
const OperandVector &Operands,
1807 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1809 SMLoc getFlatOffsetLoc(
const OperandVector &Operands)
const;
1810 SMLoc getSMEMOffsetLoc(
const OperandVector &Operands)
const;
1813 SMLoc getOperandLoc(
const OperandVector &Operands,
int MCOpIdx)
const;
1814 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1816 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1820 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1822 bool validateOffset(
const MCInst &Inst,
const OperandVector &Operands);
1823 bool validateFlatOffset(
const MCInst &Inst,
const OperandVector &Operands);
1824 bool validateSMEMOffset(
const MCInst &Inst,
const OperandVector &Operands);
1825 bool validateSOPLiteral(
const MCInst &Inst,
const OperandVector &Operands);
1826 bool validateConstantBusLimitations(
const MCInst &Inst,
1828 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1830 bool validateVOPD(
const MCInst &Inst,
const OperandVector &Operands);
1831 bool tryVOPD(
const MCInst &Inst);
1832 bool tryVOPD3(
const MCInst &Inst);
1833 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1835 bool validateIntClampSupported(
const MCInst &Inst);
1836 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1837 bool validateMIMGGatherDMask(
const MCInst &Inst);
1838 bool validateMovrels(
const MCInst &Inst,
const OperandVector &Operands);
1839 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1840 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1841 bool validateMIMGD16(
const MCInst &Inst);
1842 bool validateMIMGDim(
const MCInst &Inst,
const OperandVector &Operands);
1843 bool validateTensorR128(
const MCInst &Inst);
1844 bool validateMIMGMSAA(
const MCInst &Inst);
1845 bool validateOpSel(
const MCInst &Inst);
1846 bool validateTrue16OpSel(
const MCInst &Inst);
1847 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1848 bool validateDPP(
const MCInst &Inst,
const OperandVector &Operands);
1849 bool validateVccOperand(MCRegister
Reg)
const;
1850 bool validateVOPLiteral(
const MCInst &Inst,
const OperandVector &Operands);
1851 bool validateMAIAccWrite(
const MCInst &Inst,
const OperandVector &Operands);
1852 bool validateMAISrc2(
const MCInst &Inst,
const OperandVector &Operands);
1853 bool validateMFMA(
const MCInst &Inst,
const OperandVector &Operands);
1854 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1855 bool validateVGPRAlign(
const MCInst &Inst)
const;
1856 bool validateBLGP(
const MCInst &Inst,
const OperandVector &Operands);
1857 bool validateDS(
const MCInst &Inst,
const OperandVector &Operands);
1858 bool validateGWS(
const MCInst &Inst,
const OperandVector &Operands);
1859 bool validateDivScale(
const MCInst &Inst);
1860 bool validateWaitCnt(
const MCInst &Inst,
const OperandVector &Operands);
1861 bool validateCoherencyBits(
const MCInst &Inst,
const OperandVector &Operands,
1863 bool validateTHAndScopeBits(
const MCInst &Inst,
const OperandVector &Operands,
1864 const unsigned CPol);
1865 bool validateTFE(
const MCInst &Inst,
const OperandVector &Operands);
1866 bool validateLdsDirect(
const MCInst &Inst,
const OperandVector &Operands);
1867 bool validateWMMA(
const MCInst &Inst,
const OperandVector &Operands);
1868 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1869 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1870 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1871 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1873 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1874 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1875 ArrayRef<unsigned> Variants);
1876 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1878 bool isId(
const StringRef Id)
const;
1879 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1881 StringRef getId()
const;
1882 bool trySkipId(
const StringRef Id);
1883 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1887 bool parseString(StringRef &Val,
1888 const StringRef ErrMsg =
"expected a string");
1889 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1895 StringRef getTokenStr()
const;
1896 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1898 SMLoc getLoc()
const;
1902 void onBeginOfFile()
override;
1906 void emitTargetDirective();
1907 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1909 ParseStatus parseCustomOperand(
OperandVector &Operands,
unsigned MCK);
1919 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1920 const unsigned MaxVal,
const Twine &ErrMsg,
1922 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1923 const unsigned MinVal,
const unsigned MaxVal,
1924 const StringRef ErrMsg);
1926 bool parseSwizzleOffset(int64_t &
Imm);
1927 bool parseSwizzleMacro(int64_t &
Imm);
1928 bool parseSwizzleQuadPerm(int64_t &
Imm);
1929 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1930 bool parseSwizzleBroadcast(int64_t &
Imm);
1931 bool parseSwizzleSwap(int64_t &
Imm);
1932 bool parseSwizzleReverse(int64_t &
Imm);
1933 bool parseSwizzleFFT(int64_t &
Imm);
1934 bool parseSwizzleRotate(int64_t &
Imm);
1937 int64_t parseGPRIdxMacro();
1939 void cvtMubuf(MCInst &Inst,
const OperandVector &Operands) {
1940 cvtMubufImpl(Inst, Operands,
false);
1942 void cvtMubufAtomic(MCInst &Inst,
const OperandVector &Operands) {
1943 cvtMubufImpl(Inst, Operands,
true);
1949 OptionalImmIndexMap &OptionalIdx);
1950 void cvtScaledMFMA(MCInst &Inst,
const OperandVector &Operands);
1951 void cvtVOP3OpSel(MCInst &Inst,
const OperandVector &Operands);
1954 void cvtSWMMAC(MCInst &Inst,
const OperandVector &Operands);
1957 void cvtVOP3OpSel(MCInst &Inst,
const OperandVector &Operands,
1958 OptionalImmIndexMap &OptionalIdx);
1960 OptionalImmIndexMap &OptionalIdx);
1962 void cvtVOP3Interp(MCInst &Inst,
const OperandVector &Operands);
1963 void cvtVINTERP(MCInst &Inst,
const OperandVector &Operands);
1964 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1966 bool parseDimId(
unsigned &Encoding);
1968 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1971 bool isSupportedDPPCtrl(StringRef Ctrl,
const OperandVector &Operands);
1972 int64_t parseDPPCtrlSel(StringRef Ctrl);
1973 int64_t parseDPPCtrlPerm();
1974 void cvtDPP(MCInst &Inst,
const OperandVector &Operands,
bool IsDPP8 =
false);
1976 cvtDPP(Inst, Operands,
true);
1978 void cvtVOP3DPP(MCInst &Inst,
const OperandVector &Operands,
1979 bool IsDPP8 =
false);
1980 void cvtVOP3DPP8(MCInst &Inst,
const OperandVector &Operands) {
1981 cvtVOP3DPP(Inst, Operands,
true);
1984 ParseStatus parseSDWASel(
OperandVector &Operands, StringRef Prefix,
1985 AMDGPUOperand::ImmTy
Type);
1987 void cvtSdwaVOP1(MCInst &Inst,
const OperandVector &Operands);
1988 void cvtSdwaVOP2(MCInst &Inst,
const OperandVector &Operands);
1989 void cvtSdwaVOP2b(MCInst &Inst,
const OperandVector &Operands);
1990 void cvtSdwaVOP2e(MCInst &Inst,
const OperandVector &Operands);
1991 void cvtSdwaVOPC(MCInst &Inst,
const OperandVector &Operands);
1993 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1996 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1997 bool SkipSrcVcc =
false);
2106bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2116 if (!isImmTy(ImmTyNone)) {
2121 if (getModifiers().
Lit != LitModifier::None)
2131 if (type == MVT::f64 || type == MVT::i64) {
2133 AsmParser->hasInv2PiInlineImm());
2136 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2155 APFloat::rmNearestTiesToEven, &Lost);
2162 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2164 AsmParser->hasInv2PiInlineImm());
2169 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2170 AsmParser->hasInv2PiInlineImm());
2174 if (type == MVT::f64 || type == MVT::i64) {
2176 AsmParser->hasInv2PiInlineImm());
2185 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2186 AsmParser->hasInv2PiInlineImm());
2190 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2191 AsmParser->hasInv2PiInlineImm());
2194bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2196 if (!isImmTy(ImmTyNone)) {
2201 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2206 if (type == MVT::f64 && hasFPModifiers()) {
2226 if (type == MVT::f64) {
2231 if (type == MVT::i64) {
2244 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2245 : (type == MVT::v2i16) ? MVT::f32
2246 : (type == MVT::v2f32) ? MVT::f32
2249 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2253bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2254 return isRegKind() &&
2255 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2258bool AMDGPUOperand::isVRegWithInputMods()
const {
2259 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2261 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2262 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2265template <
bool IsFake16>
2266bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2267 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2268 : AMDGPU::VGPR_16_Lo128RegClassID);
2271template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2272 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2273 : AMDGPU::VGPR_16RegClassID);
2276bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2277 if (AsmParser->isVI())
2279 if (AsmParser->isGFX9Plus())
2280 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2284bool AMDGPUOperand::isSDWAFP16Operand()
const {
2285 return isSDWAOperand(MVT::f16);
2288bool AMDGPUOperand::isSDWAFP32Operand()
const {
2289 return isSDWAOperand(MVT::f32);
2292bool AMDGPUOperand::isSDWAInt16Operand()
const {
2293 return isSDWAOperand(MVT::i16);
2296bool AMDGPUOperand::isSDWAInt32Operand()
const {
2297 return isSDWAOperand(MVT::i32);
2300bool AMDGPUOperand::isBoolReg()
const {
2301 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2302 (AsmParser->isWave32() && isSCSrc_b32()));
2305uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val,
2306 unsigned Size)
const {
2307 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2310 const uint64_t FpSignMask = (1ULL << (
Size * 8 - 1));
2322void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2323 bool ApplyModifiers)
const {
2333 addLiteralImmOperand(Inst,
Imm.Val,
2334 ApplyModifiers & isImmTy(ImmTyNone) &&
2335 Imm.Mods.hasFPModifiers());
2337 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2342void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2343 bool ApplyModifiers)
const {
2344 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2349 if (ApplyModifiers) {
2351 const unsigned Size =
2353 Val = applyInputFPModifiers(Val,
Size);
2357 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2359 bool CanUse64BitLiterals =
2362 MCContext &Ctx = AsmParser->getContext();
2373 if (
Lit == LitModifier::None &&
2375 AsmParser->hasInv2PiInlineImm())) {
2383 bool HasMandatoryLiteral =
2386 if (
Literal.getLoBits(32) != 0 &&
2387 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2388 !HasMandatoryLiteral) {
2389 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2391 "Can't encode literal as exact 64-bit floating-point operand. "
2392 "Low 32-bits will be set to zero");
2393 Val &= 0xffffffff00000000u;
2399 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2405 Lit = LitModifier::Lit64;
2406 }
else if (
Lit == LitModifier::Lit) {
2420 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2422 Lit = LitModifier::Lit64;
2429 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2430 Literal == 0x3fc45f306725feed) {
2469 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2476 if (
Lit != LitModifier::None) {
2507 if (
Lit == LitModifier::None &&
2517 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2525 if (
Lit == LitModifier::None &&
2533 if (!AsmParser->has64BitLiterals()) {
2534 Val =
static_cast<uint64_t
>(Val) << 32;
2541 if (
Lit == LitModifier::Lit ||
2543 Val =
static_cast<uint64_t
>(Val) << 32;
2547 if (
Lit == LitModifier::Lit)
2573 if (
Lit != LitModifier::None) {
2581void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2587bool AMDGPUOperand::isInlineValue()
const {
2595void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2606 if (Is == IS_VGPR) {
2611 return AMDGPU::VGPR_32RegClassID;
2613 return AMDGPU::VReg_64RegClassID;
2615 return AMDGPU::VReg_96RegClassID;
2617 return AMDGPU::VReg_128RegClassID;
2619 return AMDGPU::VReg_160RegClassID;
2621 return AMDGPU::VReg_192RegClassID;
2623 return AMDGPU::VReg_224RegClassID;
2625 return AMDGPU::VReg_256RegClassID;
2627 return AMDGPU::VReg_288RegClassID;
2629 return AMDGPU::VReg_320RegClassID;
2631 return AMDGPU::VReg_352RegClassID;
2633 return AMDGPU::VReg_384RegClassID;
2635 return AMDGPU::VReg_512RegClassID;
2637 return AMDGPU::VReg_1024RegClassID;
2639 }
else if (Is == IS_TTMP) {
2644 return AMDGPU::TTMP_32RegClassID;
2646 return AMDGPU::TTMP_64RegClassID;
2648 return AMDGPU::TTMP_128RegClassID;
2650 return AMDGPU::TTMP_256RegClassID;
2652 return AMDGPU::TTMP_512RegClassID;
2654 }
else if (Is == IS_SGPR) {
2659 return AMDGPU::SGPR_32RegClassID;
2661 return AMDGPU::SGPR_64RegClassID;
2663 return AMDGPU::SGPR_96RegClassID;
2665 return AMDGPU::SGPR_128RegClassID;
2667 return AMDGPU::SGPR_160RegClassID;
2669 return AMDGPU::SGPR_192RegClassID;
2671 return AMDGPU::SGPR_224RegClassID;
2673 return AMDGPU::SGPR_256RegClassID;
2675 return AMDGPU::SGPR_288RegClassID;
2677 return AMDGPU::SGPR_320RegClassID;
2679 return AMDGPU::SGPR_352RegClassID;
2681 return AMDGPU::SGPR_384RegClassID;
2683 return AMDGPU::SGPR_512RegClassID;
2685 }
else if (Is == IS_AGPR) {
2690 return AMDGPU::AGPR_32RegClassID;
2692 return AMDGPU::AReg_64RegClassID;
2694 return AMDGPU::AReg_96RegClassID;
2696 return AMDGPU::AReg_128RegClassID;
2698 return AMDGPU::AReg_160RegClassID;
2700 return AMDGPU::AReg_192RegClassID;
2702 return AMDGPU::AReg_224RegClassID;
2704 return AMDGPU::AReg_256RegClassID;
2706 return AMDGPU::AReg_288RegClassID;
2708 return AMDGPU::AReg_320RegClassID;
2710 return AMDGPU::AReg_352RegClassID;
2712 return AMDGPU::AReg_384RegClassID;
2714 return AMDGPU::AReg_512RegClassID;
2716 return AMDGPU::AReg_1024RegClassID;
2724 .
Case(
"exec", AMDGPU::EXEC)
2725 .
Case(
"vcc", AMDGPU::VCC)
2726 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2727 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2728 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2729 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2730 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2731 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2732 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2733 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2734 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2735 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2736 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2737 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2738 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2739 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2740 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2741 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2742 .
Case(
"m0", AMDGPU::M0)
2743 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2744 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2745 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2746 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2747 .
Case(
"scc", AMDGPU::SRC_SCC)
2748 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2749 .
Case(
"tba", AMDGPU::TBA)
2750 .
Case(
"tma", AMDGPU::TMA)
2751 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2752 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2753 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2754 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2755 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2756 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2757 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2758 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2759 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2760 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2761 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2762 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2763 .
Case(
"pc", AMDGPU::PC_REG)
2764 .
Case(
"null", AMDGPU::SGPR_NULL)
2768bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2769 SMLoc &EndLoc,
bool RestoreOnFailure) {
2770 auto R = parseRegister();
2774 RegNo =
R->getReg();
2775 StartLoc =
R->getStartLoc();
2776 EndLoc =
R->getEndLoc();
2780bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2782 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2785ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2787 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2788 bool PendingErrors = getParser().hasPendingError();
2789 getParser().clearPendingErrors();
2797bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2798 RegisterKind RegKind,
2800 RegisterKind RegKind1, SMLoc Loc) {
2802 if (RegKind == IS_SGPR) {
2803 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2804 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2805 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2811 if (RegKind != RegKind1) {
2812 Error(Loc,
"registers in a list must be of the same kind");
2813 return MCRegister();
2818 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2823 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2824 Reg = AMDGPU::FLAT_SCR;
2828 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2829 Reg = AMDGPU::XNACK_MASK;
2833 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2838 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2843 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2848 Error(Loc,
"register does not fit in the list");
2854 if (Reg1 !=
Reg + RegWidth / 32) {
2855 Error(Loc,
"registers in a list must have consecutive indices");
2871 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2872 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2876 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2882 if (Str.starts_with(
Reg.Name))
2888 return !Str.getAsInteger(10, Num);
2891bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2892 const AsmToken &NextToken)
const {
2907 StringRef RegSuffix = Str.substr(
RegName.size());
2908 if (!RegSuffix.
empty()) {
2925bool AMDGPUAsmParser::isRegister() {
2926 return isRegister(
getToken(), peekToken());
2929MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2930 unsigned SubReg,
unsigned RegWidth,
2934 unsigned AlignSize = 1;
2935 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2941 if (RegNum % AlignSize != 0) {
2942 Error(Loc,
"invalid register alignment");
2943 return MCRegister();
2946 unsigned RegIdx = RegNum / AlignSize;
2949 Error(Loc,
"invalid or unsupported register size");
2950 return MCRegister();
2954 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2955 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2956 Error(Loc,
"register index is out of range");
2957 return AMDGPU::NoRegister;
2960 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2961 Error(Loc,
"register index is out of range");
2962 return MCRegister();
2978bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2980 int64_t RegLo, RegHi;
2984 SMLoc FirstIdxLoc = getLoc();
2991 SecondIdxLoc = getLoc();
3002 Error(FirstIdxLoc,
"invalid register index");
3007 Error(SecondIdxLoc,
"invalid register index");
3011 if (RegLo > RegHi) {
3012 Error(FirstIdxLoc,
"first register index should not exceed second index");
3016 if (RegHi == RegLo) {
3017 StringRef RegSuffix = getTokenStr();
3018 if (RegSuffix ==
".l") {
3019 SubReg = AMDGPU::lo16;
3021 }
else if (RegSuffix ==
".h") {
3022 SubReg = AMDGPU::hi16;
3027 Num =
static_cast<unsigned>(RegLo);
3028 RegWidth = 32 * ((RegHi - RegLo) + 1);
3033MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3036 SmallVectorImpl<AsmToken> &Tokens) {
3042 RegKind = IS_SPECIAL;
3049MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3052 SmallVectorImpl<AsmToken> &Tokens) {
3054 StringRef
RegName = getTokenStr();
3055 auto Loc = getLoc();
3059 Error(Loc,
"invalid register name");
3060 return MCRegister();
3068 unsigned SubReg = NoSubRegister;
3069 bool IsRange =
false;
3070 if (!RegSuffix.
empty()) {
3072 SubReg = AMDGPU::lo16;
3074 SubReg = AMDGPU::hi16;
3078 Error(Loc,
"invalid register index");
3079 return MCRegister();
3085 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3086 return MCRegister();
3090 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3091 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3092 if (RegKind == IS_SGPR && IsRange
3093 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3094 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3095 Error(Loc,
"register index is out of range");
3096 return MCRegister();
3102MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3103 unsigned &RegNum,
unsigned &RegWidth,
3104 SmallVectorImpl<AsmToken> &Tokens) {
3106 auto ListLoc = getLoc();
3109 "expected a register or a list of registers")) {
3110 return MCRegister();
3115 auto Loc = getLoc();
3116 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3117 return MCRegister();
3118 if (RegWidth != 32) {
3119 Error(Loc,
"expected a single 32-bit register");
3120 return MCRegister();
3124 RegisterKind NextRegKind;
3126 unsigned NextRegNum, NextRegWidth;
3129 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3131 return MCRegister();
3133 if (NextRegWidth != 32) {
3134 Error(Loc,
"expected a single 32-bit register");
3135 return MCRegister();
3137 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3139 return MCRegister();
3143 "expected a comma or a closing square bracket")) {
3144 return MCRegister();
3148 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3153bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3154 MCRegister &
Reg,
unsigned &RegNum,
3156 SmallVectorImpl<AsmToken> &Tokens) {
3157 auto Loc = getLoc();
3161 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3163 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3165 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3170 assert(Parser.hasPendingError());
3174 if (!subtargetHasRegister(*
TRI,
Reg)) {
3175 if (
Reg == AMDGPU::SGPR_NULL) {
3176 Error(Loc,
"'null' operand is not supported on this GPU");
3179 " register not available on this GPU");
3187bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3188 MCRegister &
Reg,
unsigned &RegNum,
3190 bool RestoreOnFailure ) {
3194 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3195 if (RestoreOnFailure) {
3196 while (!Tokens.
empty()) {
3205std::optional<StringRef>
3206AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3209 return StringRef(
".amdgcn.next_free_vgpr");
3211 return StringRef(
".amdgcn.next_free_sgpr");
3213 return std::nullopt;
3217void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3218 auto SymbolName = getGprCountSymbolName(RegKind);
3219 assert(SymbolName &&
"initializing invalid register kind");
3225bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3226 unsigned DwordRegIndex,
3227 unsigned RegWidth) {
3232 auto SymbolName = getGprCountSymbolName(RegKind);
3237 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3241 return !
Error(getLoc(),
3242 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3246 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3248 if (OldCount <= NewMax)
3254std::unique_ptr<AMDGPUOperand>
3255AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3257 SMLoc StartLoc = Tok.getLoc();
3258 SMLoc EndLoc = Tok.getEndLoc();
3259 RegisterKind RegKind;
3261 unsigned RegNum, RegWidth;
3263 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3267 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3270 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3271 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3274ParseStatus AMDGPUAsmParser::parseImm(
OperandVector &Operands,
3278 if (isRegister() || isModifier())
3281 if (
Lit == LitModifier::None) {
3282 if (trySkipId(
"lit"))
3283 Lit = LitModifier::Lit;
3284 else if (trySkipId(
"lit64"))
3285 Lit = LitModifier::Lit64;
3287 if (
Lit != LitModifier::None) {
3290 ParseStatus S = parseImm(Operands, HasSP3AbsModifier,
Lit);
3299 const auto &NextTok = peekToken();
3302 bool Negate =
false;
3310 AMDGPUOperand::Modifiers Mods;
3318 StringRef Num = getTokenStr();
3321 APFloat RealVal(APFloat::IEEEdouble());
3322 auto roundMode = APFloat::rmNearestTiesToEven;
3323 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3326 RealVal.changeSign();
3329 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3330 S, AMDGPUOperand::ImmTyNone,
true));
3331 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*Operands.
back());
3332 Op.setModifiers(Mods);
3341 if (HasSP3AbsModifier) {
3350 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3353 if (Parser.parseExpression(Expr))
3357 if (Expr->evaluateAsAbsolute(IntVal)) {
3359 return Error(S,
"literal value out of range");
3360 Operands.
push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3361 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*Operands.
back());
3362 Op.setModifiers(Mods);
3364 if (
Lit != LitModifier::None)
3366 Operands.
push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3375ParseStatus AMDGPUAsmParser::parseReg(
OperandVector &Operands) {
3379 if (
auto R = parseRegister()) {
3387ParseStatus AMDGPUAsmParser::parseRegOrImm(
OperandVector &Operands,
3389 ParseStatus Res = parseReg(Operands);
3394 return parseImm(Operands, HasSP3AbsMod,
Lit);
3397bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3398 const AsmToken &NextToken)
const {
3401 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3406bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3407 const AsmToken &NextToken)
const {
3411bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3412 const AsmToken &NextToken)
const {
3413 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3416bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3417 const AsmToken &NextToken)
const {
3418 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3434bool AMDGPUAsmParser::isModifier() {
3437 AsmToken NextToken[2];
3438 peekTokens(NextToken);
3440 return isOperandModifier(Tok, NextToken[0]) ||
3442 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3443 isOpcodeModifierWithVal(Tok, NextToken[0]);
3468bool AMDGPUAsmParser::parseSP3NegModifier() {
3470 AsmToken NextToken[2];
3471 peekTokens(NextToken);
3474 (isRegister(NextToken[0], NextToken[1]) ||
3484AMDGPUAsmParser::parseRegOrImmWithFPInputMods(
OperandVector &Operands,
3492 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3494 SP3Neg = parseSP3NegModifier();
3497 Neg = trySkipId(
"neg");
3499 return Error(Loc,
"expected register or immediate");
3503 Abs = trySkipId(
"abs");
3508 if (trySkipId(
"lit")) {
3509 Lit = LitModifier::Lit;
3512 }
else if (trySkipId(
"lit64")) {
3513 Lit = LitModifier::Lit64;
3516 if (!has64BitLiterals())
3517 return Error(Loc,
"lit64 is not supported on this GPU");
3523 return Error(Loc,
"expected register or immediate");
3527 Res = parseRegOrImm(Operands, SP3Abs,
Lit);
3529 Res = parseReg(Operands);
3532 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3536 if (
Lit != LitModifier::None && !Operands.
back()->isImm())
3537 Error(Loc,
"expected immediate with lit modifier");
3539 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3545 if (
Lit != LitModifier::None &&
3549 AMDGPUOperand::Modifiers Mods;
3550 Mods.Abs = Abs || SP3Abs;
3551 Mods.Neg = Neg || SP3Neg;
3554 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3555 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*Operands.
back());
3557 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3558 Op.setModifiers(Mods);
3564AMDGPUAsmParser::parseRegOrImmWithIntInputMods(
OperandVector &Operands,
3566 bool Sext = trySkipId(
"sext");
3567 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3572 Res = parseRegOrImm(Operands);
3574 Res = parseReg(Operands);
3582 AMDGPUOperand::Modifiers Mods;
3585 if (Mods.hasIntModifiers()) {
3586 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*Operands.
back());
3588 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3589 Op.setModifiers(Mods);
3595ParseStatus AMDGPUAsmParser::parseRegWithFPInputMods(
OperandVector &Operands) {
3596 return parseRegOrImmWithFPInputMods(Operands,
false);
3599ParseStatus AMDGPUAsmParser::parseRegWithIntInputMods(
OperandVector &Operands) {
3600 return parseRegOrImmWithIntInputMods(Operands,
false);
3603ParseStatus AMDGPUAsmParser::parseVReg32OrOff(
OperandVector &Operands) {
3604 auto Loc = getLoc();
3605 if (trySkipId(
"off")) {
3607 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3614 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3623unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3628 return Match_InvalidOperand;
3630 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3631 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3634 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3636 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3637 return Match_InvalidOperand;
3645 if (tryAnotherVOPDEncoding(Inst))
3646 return Match_InvalidOperand;
3648 return Match_Success;
3652 static const unsigned Variants[] = {
3661ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3662 if (isForcedDPP() && isForcedVOP3()) {
3666 if (getForcedEncodingSize() == 32) {
3671 if (isForcedVOP3()) {
3676 if (isForcedSDWA()) {
3682 if (isForcedDPP()) {
3690StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3691 if (isForcedDPP() && isForcedVOP3())
3694 if (getForcedEncodingSize() == 32)
3710AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3714 case AMDGPU::FLAT_SCR:
3716 case AMDGPU::VCC_LO:
3717 case AMDGPU::VCC_HI:
3724 return MCRegister();
3731bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3732 unsigned OpIdx)
const {
3789unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3795 case AMDGPU::V_LSHLREV_B64_e64:
3796 case AMDGPU::V_LSHLREV_B64_gfx10:
3797 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3798 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3799 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3800 case AMDGPU::V_LSHRREV_B64_e64:
3801 case AMDGPU::V_LSHRREV_B64_gfx10:
3802 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3803 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3804 case AMDGPU::V_ASHRREV_I64_e64:
3805 case AMDGPU::V_ASHRREV_I64_gfx10:
3806 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3807 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3808 case AMDGPU::V_LSHL_B64_e64:
3809 case AMDGPU::V_LSHR_B64_e64:
3810 case AMDGPU::V_ASHR_I64_e64:
3823 bool AddMandatoryLiterals =
false) {
3826 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3830 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3832 return {getNamedOperandIdx(Opcode, OpName::src0X),
3833 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3834 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3835 getNamedOperandIdx(Opcode, OpName::src0Y),
3836 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3837 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3842 return {getNamedOperandIdx(Opcode, OpName::src0),
3843 getNamedOperandIdx(Opcode, OpName::src1),
3844 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3847bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3850 return !isInlineConstant(Inst,
OpIdx);
3857 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3868 const unsigned Opcode = Inst.
getOpcode();
3869 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3872 if (!LaneSelOp.
isReg())
3875 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3878bool AMDGPUAsmParser::validateConstantBusLimitations(
3880 const unsigned Opcode = Inst.
getOpcode();
3881 const MCInstrDesc &
Desc = MII.
get(Opcode);
3882 MCRegister LastSGPR;
3883 unsigned ConstantBusUseCount = 0;
3884 unsigned NumLiterals = 0;
3885 unsigned LiteralSize;
3901 SmallDenseSet<MCRegister> SGPRsUsed;
3902 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3904 SGPRsUsed.
insert(SGPRUsed);
3905 ++ConstantBusUseCount;
3910 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3912 for (
int OpIdx : OpIndices) {
3917 if (usesConstantBus(Inst,
OpIdx)) {
3926 if (SGPRsUsed.
insert(LastSGPR).second) {
3927 ++ConstantBusUseCount;
3947 if (NumLiterals == 0) {
3950 }
else if (LiteralSize !=
Size) {
3956 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3958 "invalid operand (violates constant bus restrictions)");
3965std::optional<unsigned>
3966AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3968 const unsigned Opcode = Inst.
getOpcode();
3974 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3975 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3984 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3985 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3986 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3987 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3988 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3989 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3993 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3994 int I = getNamedOperandIdx(Opcode, OpName);
3998 int64_t
Imm =
Op.getImm();
4004 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4005 OpName::vsrc2Y, OpName::imm}) {
4006 int I = getNamedOperandIdx(Opcode, OpName);
4016 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4017 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4019 return InvalidCompOprIdx;
4022bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4029 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand : Operands) {
4030 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4031 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4033 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4037 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4038 if (!InvalidCompOprIdx.has_value())
4041 auto CompOprIdx = *InvalidCompOprIdx;
4044 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4045 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4046 assert(ParsedIdx > 0 && ParsedIdx < Operands.size());
4048 auto Loc = ((AMDGPUOperand &)*Operands[ParsedIdx]).getStartLoc();
4049 if (CompOprIdx == VOPD::Component::DST) {
4051 Error(Loc,
"dst registers must be distinct");
4053 Error(Loc,
"one dst register must be even and the other odd");
4055 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4056 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4057 " operands must use different VGPR banks");
4065bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4067 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4068 if (!InvalidCompOprIdx.has_value())
4072 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4073 if (InvalidCompOprIdx.has_value()) {
4078 if (*InvalidCompOprIdx == VOPD::Component::DST)
4091bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4092 const unsigned Opcode = Inst.
getOpcode();
4107 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4108 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4109 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4110 int I = getNamedOperandIdx(Opcode, OpName);
4117 return !tryVOPD3(Inst);
4122bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4127 return tryVOPD(Inst);
4128 return tryVOPD3(Inst);
4131bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4136 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4144bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4152 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4153 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4154 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4159 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4160 hasBVHRayTracingInsts())
4163 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4164 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4169 bool IsPackedD16 =
false;
4172 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4173 IsPackedD16 = D16Idx >= 0;
4175 DataSize = (DataSize + 1) / 2;
4178 if ((VDataSize / 4) == DataSize + TFESize)
4181 StringRef Modifiers;
4183 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4185 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4187 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4191bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4200 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4202 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4203 AMDGPU::OpName RSrcOpName =
4205 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4206 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4207 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4211 assert(SrsrcIdx > VAddr0Idx);
4214 if (BaseOpcode->
BVH) {
4215 if (IsA16 == BaseOpcode->
A16)
4217 Error(IDLoc,
"image address size does not match a16");
4223 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4224 unsigned ActualAddrSize =
4225 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4227 unsigned ExpectedAddrSize =
4231 if (hasPartialNSAEncoding() &&
4233 int VAddrLastIdx = SrsrcIdx - 1;
4234 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4236 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4239 if (ExpectedAddrSize > 12)
4240 ExpectedAddrSize = 16;
4245 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4249 if (ActualAddrSize == ExpectedAddrSize)
4252 Error(IDLoc,
"image address size does not match dim and a16");
4256bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4263 if (!
Desc.mayLoad() || !
Desc.mayStore())
4266 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4273 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4276bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4283 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4291 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4294bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4308 for (
unsigned i = 1, e = Operands.
size(); i != e; ++i) {
4309 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
4316bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4323 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4326 if (!BaseOpcode->
MSAA)
4329 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4335 return DimInfo->
MSAA;
4340 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4341 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4342 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4352bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4360 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4363 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4371 Error(getOperandLoc(Operands, Src0Idx),
"source operand must be a VGPR");
4375bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4380 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4383 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4386 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4393 Error(getOperandLoc(Operands, Src0Idx),
4394 "source operand must be either a VGPR or an inline constant");
4401bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4406 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4409 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4413 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4414 Error(getOperandLoc(Operands, Src2Idx),
4415 "inline constants are not allowed for this operand");
4422bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4430 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4431 if (BlgpIdx != -1) {
4432 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4433 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4443 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4444 Error(getOperandLoc(Operands, Src0Idx),
4445 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4450 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4451 Error(getOperandLoc(Operands, Src1Idx),
4452 "wrong register tuple size for blgp value " + Twine(BLGP));
4460 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4464 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4468 MCRegister Src2Reg = Src2.
getReg();
4470 if (Src2Reg == DstReg)
4475 .getSizeInBits() <= 128)
4478 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4479 Error(getOperandLoc(Operands, Src2Idx),
4480 "source 2 operand must not partially overlap with dst");
4487bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4491 case V_DIV_SCALE_F32_gfx6_gfx7:
4492 case V_DIV_SCALE_F32_vi:
4493 case V_DIV_SCALE_F32_gfx10:
4494 case V_DIV_SCALE_F64_gfx6_gfx7:
4495 case V_DIV_SCALE_F64_vi:
4496 case V_DIV_SCALE_F64_gfx10:
4503 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4504 AMDGPU::OpName::src2_modifiers}) {
4515bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4522 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4531bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4537 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4544 case AMDGPU::V_SUBREV_F32_e32:
4545 case AMDGPU::V_SUBREV_F32_e64:
4546 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4547 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4548 case AMDGPU::V_SUBREV_F32_e32_vi:
4549 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4550 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4551 case AMDGPU::V_SUBREV_F32_e64_vi:
4553 case AMDGPU::V_SUBREV_CO_U32_e32:
4554 case AMDGPU::V_SUBREV_CO_U32_e64:
4555 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4556 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4558 case AMDGPU::V_SUBBREV_U32_e32:
4559 case AMDGPU::V_SUBBREV_U32_e64:
4560 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4561 case AMDGPU::V_SUBBREV_U32_e32_vi:
4562 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4563 case AMDGPU::V_SUBBREV_U32_e64_vi:
4565 case AMDGPU::V_SUBREV_U32_e32:
4566 case AMDGPU::V_SUBREV_U32_e64:
4567 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4568 case AMDGPU::V_SUBREV_U32_e32_vi:
4569 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4570 case AMDGPU::V_SUBREV_U32_e64_vi:
4572 case AMDGPU::V_SUBREV_F16_e32:
4573 case AMDGPU::V_SUBREV_F16_e64:
4574 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4575 case AMDGPU::V_SUBREV_F16_e32_vi:
4576 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4577 case AMDGPU::V_SUBREV_F16_e64_vi:
4579 case AMDGPU::V_SUBREV_U16_e32:
4580 case AMDGPU::V_SUBREV_U16_e64:
4581 case AMDGPU::V_SUBREV_U16_e32_vi:
4582 case AMDGPU::V_SUBREV_U16_e64_vi:
4584 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4585 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4586 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4588 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4589 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4591 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4592 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4594 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4595 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4597 case AMDGPU::V_LSHRREV_B32_e32:
4598 case AMDGPU::V_LSHRREV_B32_e64:
4599 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4600 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4601 case AMDGPU::V_LSHRREV_B32_e32_vi:
4602 case AMDGPU::V_LSHRREV_B32_e64_vi:
4603 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4604 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4606 case AMDGPU::V_ASHRREV_I32_e32:
4607 case AMDGPU::V_ASHRREV_I32_e64:
4608 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4609 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4610 case AMDGPU::V_ASHRREV_I32_e32_vi:
4611 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4612 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4613 case AMDGPU::V_ASHRREV_I32_e64_vi:
4615 case AMDGPU::V_LSHLREV_B32_e32:
4616 case AMDGPU::V_LSHLREV_B32_e64:
4617 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4618 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4619 case AMDGPU::V_LSHLREV_B32_e32_vi:
4620 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4621 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4622 case AMDGPU::V_LSHLREV_B32_e64_vi:
4624 case AMDGPU::V_LSHLREV_B16_e32:
4625 case AMDGPU::V_LSHLREV_B16_e64:
4626 case AMDGPU::V_LSHLREV_B16_e32_vi:
4627 case AMDGPU::V_LSHLREV_B16_e64_vi:
4628 case AMDGPU::V_LSHLREV_B16_gfx10:
4630 case AMDGPU::V_LSHRREV_B16_e32:
4631 case AMDGPU::V_LSHRREV_B16_e64:
4632 case AMDGPU::V_LSHRREV_B16_e32_vi:
4633 case AMDGPU::V_LSHRREV_B16_e64_vi:
4634 case AMDGPU::V_LSHRREV_B16_gfx10:
4636 case AMDGPU::V_ASHRREV_I16_e32:
4637 case AMDGPU::V_ASHRREV_I16_e64:
4638 case AMDGPU::V_ASHRREV_I16_e32_vi:
4639 case AMDGPU::V_ASHRREV_I16_e64_vi:
4640 case AMDGPU::V_ASHRREV_I16_gfx10:
4642 case AMDGPU::V_LSHLREV_B64_e64:
4643 case AMDGPU::V_LSHLREV_B64_gfx10:
4644 case AMDGPU::V_LSHLREV_B64_vi:
4646 case AMDGPU::V_LSHRREV_B64_e64:
4647 case AMDGPU::V_LSHRREV_B64_gfx10:
4648 case AMDGPU::V_LSHRREV_B64_vi:
4650 case AMDGPU::V_ASHRREV_I64_e64:
4651 case AMDGPU::V_ASHRREV_I64_gfx10:
4652 case AMDGPU::V_ASHRREV_I64_vi:
4654 case AMDGPU::V_PK_LSHLREV_B16:
4655 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4656 case AMDGPU::V_PK_LSHLREV_B16_vi:
4658 case AMDGPU::V_PK_LSHRREV_B16:
4659 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4660 case AMDGPU::V_PK_LSHRREV_B16_vi:
4661 case AMDGPU::V_PK_ASHRREV_I16:
4662 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4663 case AMDGPU::V_PK_ASHRREV_I16_vi:
4670bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4672 const unsigned Opcode = Inst.
getOpcode();
4681 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4682 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4686 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4689 Error(getOperandLoc(Operands, SrcIdx),
4690 "lds_direct is not supported on this GPU");
4695 Error(getOperandLoc(Operands, SrcIdx),
4696 "lds_direct cannot be used with this instruction");
4700 if (SrcName != OpName::src0) {
4701 Error(getOperandLoc(Operands, SrcIdx),
4702 "lds_direct may be used as src0 only");
4711SMLoc AMDGPUAsmParser::getFlatOffsetLoc(
const OperandVector &Operands)
const {
4712 for (
unsigned i = 1, e = Operands.
size(); i != e; ++i) {
4713 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
4714 if (
Op.isFlatOffset())
4715 return Op.getStartLoc();
4720bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4723 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4728 return validateFlatOffset(Inst, Operands);
4731 return validateSMEMOffset(Inst, Operands);
4736 const unsigned OffsetSize = 24;
4737 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4738 Error(getFlatOffsetLoc(Operands),
4739 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4740 "-bit unsigned offset for buffer ops");
4744 const unsigned OffsetSize = 16;
4745 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4746 Error(getFlatOffsetLoc(Operands),
4747 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4754bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4760 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4764 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4765 Error(getFlatOffsetLoc(Operands),
4766 "flat offset modifier is not supported on this GPU");
4773 bool AllowNegative =
4775 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4776 Error(getFlatOffsetLoc(Operands),
4777 Twine(
"expected a ") +
4778 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4779 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4786SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(
const OperandVector &Operands)
const {
4788 for (
unsigned i = 2, e = Operands.
size(); i != e; ++i) {
4789 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
4790 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4791 return Op.getStartLoc();
4796bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4805 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4819 Error(getSMEMOffsetLoc(Operands),
4821 ?
"expected a 23-bit unsigned offset for buffer ops"
4822 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4823 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4824 :
"expected a 21-bit signed offset");
4829bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4832 const MCInstrDesc &
Desc = MII.
get(Opcode);
4836 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4837 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4839 const int OpIndices[] = {Src0Idx, Src1Idx};
4841 unsigned NumExprs = 0;
4842 unsigned NumLiterals = 0;
4845 for (
int OpIdx : OpIndices) {
4853 std::optional<int64_t>
Imm;
4856 }
else if (MO.
isExpr()) {
4865 if (!
Imm.has_value()) {
4867 }
else if (!isInlineConstant(Inst,
OpIdx)) {
4871 if (NumLiterals == 0 || LiteralValue !=
Value) {
4879 if (NumLiterals + NumExprs <= 1)
4882 Error(getOperandLoc(Operands, Src1Idx),
4883 "only one unique literal operand is allowed");
4887bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4890 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4898 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4899 if (OpSelIdx != -1) {
4903 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4904 if (OpSelHiIdx != -1) {
4913 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4923 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4924 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4925 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4926 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4928 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4929 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4935 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4937 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4947 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4948 if (Src2Idx != -1) {
4949 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4959bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4960 if (!hasTrue16Insts())
4962 const MCRegisterInfo *MRI = getMRI();
4964 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4970 if (OpSelOpValue == 0)
4972 unsigned OpCount = 0;
4973 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4974 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4975 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4982 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4983 if (OpSelOpIsHi != VGPRSuffixIsHi)
4992bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4993 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5005 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5016 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5017 AMDGPU::OpName::src1_modifiers,
5018 AMDGPU::OpName::src2_modifiers};
5020 for (
unsigned i = 0; i < 3; ++i) {
5030bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5033 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5034 if (DppCtrlIdx >= 0) {
5041 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl, Operands);
5042 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5043 :
"DP ALU dpp only supports row_newbcast");
5048 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5049 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5052 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5054 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5057 Error(getOperandLoc(Operands, Src1Idx),
5058 "invalid operand for instruction");
5062 Error(getInstLoc(Operands),
5063 "src1 immediate operand invalid for instruction");
5073bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5074 return (
Reg == AMDGPU::VCC && isWave64()) ||
5075 (
Reg == AMDGPU::VCC_LO && isWave32());
5079bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5082 const MCInstrDesc &
Desc = MII.
get(Opcode);
5083 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5090 std::optional<unsigned> LiteralOpIdx;
5093 for (
int OpIdx : OpIndices) {
5103 std::optional<int64_t>
Imm;
5109 bool IsAnotherLiteral =
false;
5110 bool IsForcedLit = findMCOperand(Operands,
OpIdx).isForcedLit();
5111 bool IsForcedLit64 = findMCOperand(Operands,
OpIdx).isForcedLit64();
5112 if (!
Imm.has_value()) {
5114 IsAnotherLiteral =
true;
5115 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst,
OpIdx)) {
5120 HasMandatoryLiteral);
5121 unsigned OpTy =
Desc.operands()[
OpIdx].OperandType;
5131 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5132 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5134 "invalid operand for instruction");
5139 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5140 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5142 "invalid operand for instruction");
5146 if (IsFP64 && IsValid32Op && !IsForcedFP64)
5153 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5154 !getFeatureBits()[FeatureVOP3Literal]) {
5156 "literal operands are not supported");
5160 if (LiteralOpIdx && IsAnotherLiteral) {
5161 Error(getLaterLoc(getOperandLoc(Operands,
OpIdx),
5162 getOperandLoc(Operands, *LiteralOpIdx)),
5163 "only one unique literal operand is allowed");
5167 if (IsAnotherLiteral)
5168 LiteralOpIdx =
OpIdx;
5191bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5197 ? AMDGPU::OpName::data0
5198 : AMDGPU::OpName::vdata;
5200 const MCRegisterInfo *MRI = getMRI();
5201 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5205 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5206 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5210 auto FB = getFeatureBits();
5211 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5212 if (DataAreg < 0 || DstAreg < 0)
5214 return DstAreg == DataAreg;
5217 return DstAreg < 1 && DataAreg < 1;
5220bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5221 auto FB = getFeatureBits();
5222 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5226 const MCRegisterInfo *MRI = getMRI();
5229 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5232 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5236 case AMDGPU::DS_LOAD_TR6_B96:
5237 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5241 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5242 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5246 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5247 if (VAddrIdx != -1) {
5250 if ((
Sub - AMDGPU::VGPR0) & 1)
5255 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5256 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5261 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5262 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5281SMLoc AMDGPUAsmParser::getBLGPLoc(
const OperandVector &Operands)
const {
5282 for (
unsigned i = 1, e = Operands.
size(); i != e; ++i) {
5283 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
5285 return Op.getStartLoc();
5290bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5293 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5296 SMLoc BLGPLoc = getBLGPLoc(Operands);
5299 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5300 auto FB = getFeatureBits();
5301 bool UsesNeg =
false;
5302 if (FB[AMDGPU::FeatureGFX940Insts]) {
5304 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5305 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5306 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5307 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5312 if (IsNeg == UsesNeg)
5315 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5316 :
"invalid modifier: neg is not supported");
5321bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5327 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5328 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5329 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5330 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5333 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5336 if (
Reg == AMDGPU::SGPR_NULL)
5339 Error(getOperandLoc(Operands, Src0Idx),
"src0 must be null");
5343bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5348 return validateGWS(Inst, Operands);
5353 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5358 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS, Operands);
5359 Error(S,
"gds modifier is not supported on this GPU");
5367bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5369 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5373 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5374 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5377 const MCRegisterInfo *MRI = getMRI();
5378 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5380 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5383 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5385 Error(getOperandLoc(Operands, Data0Pos),
"vgpr must be even aligned");
5392bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5396 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5404 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5407 Error(S,
"scale_offset is not supported on this GPU");
5410 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5413 Error(S,
"nv is not supported on this GPU");
5418 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5421 Error(S,
"scale_offset is not supported for this instruction");
5425 return validateTHAndScopeBits(Inst, Operands, CPol);
5429 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5430 Error(S,
"cache policy is not supported for SMRD instructions");
5434 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5441 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5445 "scc modifier is not supported for this instruction on this GPU");
5456 :
"instruction must use glc");
5461 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5464 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5466 :
"instruction must not use glc");
5474bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5476 const unsigned CPol) {
5481 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
5488 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5493 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5501 return PrintError(
"invalid th value for SMEM instruction");
5508 return PrintError(
"scope and th combination is not valid");
5514 return PrintError(
"invalid th value for atomic instructions");
5517 return PrintError(
"invalid th value for store instructions");
5520 return PrintError(
"invalid th value for load instructions");
5526bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5530 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE, Operands);
5531 if (Loc != getInstLoc(Operands)) {
5532 Error(Loc,
"TFE modifier has no meaning for store instructions");
5540bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5546 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5550 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5553 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5554 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5562 Error(getOperandLoc(Operands, SrcIdx),
5563 "wrong register tuple size for " +
5568 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5569 !validateFmt(BFmt, AMDGPU::OpName::src1))
5573 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5574 if (AScaleIdx == -1)
5578 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5581 Error(getImmLoc(AMDGPUOperand::ImmTyMatrixAFMT, Operands),
5582 "invalid matrix and scale format combination");
5589bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5591 if (!validateLdsDirect(Inst, Operands))
5593 if (!validateTrue16OpSel(Inst)) {
5594 Error(getImmLoc(AMDGPUOperand::ImmTyOpSel, Operands),
5595 "op_sel operand conflicts with 16-bit operand suffix");
5598 if (!validateSOPLiteral(Inst, Operands))
5600 if (!validateVOPLiteral(Inst, Operands)) {
5603 if (!validateConstantBusLimitations(Inst, Operands)) {
5606 if (!validateVOPD(Inst, Operands)) {
5609 if (!validateIntClampSupported(Inst)) {
5610 Error(getImmLoc(AMDGPUOperand::ImmTyClamp, Operands),
5611 "integer clamping is not supported on this GPU");
5614 if (!validateOpSel(Inst)) {
5615 Error(getImmLoc(AMDGPUOperand::ImmTyOpSel, Operands),
5616 "invalid op_sel operand");
5619 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5620 Error(getImmLoc(AMDGPUOperand::ImmTyNegLo, Operands),
5621 "invalid neg_lo operand");
5624 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5625 Error(getImmLoc(AMDGPUOperand::ImmTyNegHi, Operands),
5626 "invalid neg_hi operand");
5629 if (!validateDPP(Inst, Operands)) {
5633 if (!validateMIMGD16(Inst)) {
5634 Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
5635 "d16 modifier is not supported on this GPU");
5638 if (!validateMIMGDim(Inst, Operands)) {
5639 Error(IDLoc,
"missing dim operand");
5642 if (!validateTensorR128(Inst)) {
5643 Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
5644 "instruction must set modifier r128=0");
5647 if (!validateMIMGMSAA(Inst)) {
5648 Error(getImmLoc(AMDGPUOperand::ImmTyDim, Operands),
5649 "invalid dim; must be MSAA type");
5652 if (!validateMIMGDataSize(Inst, IDLoc)) {
5655 if (!validateMIMGAddrSize(Inst, IDLoc))
5657 if (!validateMIMGAtomicDMask(Inst)) {
5658 Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
5659 "invalid atomic image dmask");
5662 if (!validateMIMGGatherDMask(Inst)) {
5663 Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
5664 "invalid image_gather dmask: only one bit must be set");
5667 if (!validateMovrels(Inst, Operands)) {
5670 if (!validateOffset(Inst, Operands)) {
5673 if (!validateMAIAccWrite(Inst, Operands)) {
5676 if (!validateMAISrc2(Inst, Operands)) {
5679 if (!validateMFMA(Inst, Operands)) {
5682 if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
5686 if (!validateAGPRLdSt(Inst)) {
5689 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5690 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5691 :
"invalid register class: agpr loads and stores not supported on "
5695 if (!validateVGPRAlign(Inst)) {
5696 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5699 if (!validateDS(Inst, Operands)) {
5703 if (!validateBLGP(Inst, Operands)) {
5707 if (!validateDivScale(Inst)) {
5708 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5711 if (!validateWaitCnt(Inst, Operands)) {
5714 if (!validateTFE(Inst, Operands)) {
5717 if (!validateWMMA(Inst, Operands)) {
5726 unsigned VariantID = 0);
5730 unsigned VariantID);
5732bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5737bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5738 const FeatureBitset &FBS,
5739 ArrayRef<unsigned> Variants) {
5740 for (
auto Variant : Variants) {
5748bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5750 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5753 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5758 getParser().clearPendingErrors();
5762 StringRef VariantName = getMatchedVariantName();
5763 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5764 return Error(IDLoc, Twine(VariantName,
5765 " variant of this instruction is not supported"));
5769 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5770 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5772 FeatureBitset FeaturesWS32 = getFeatureBits();
5773 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5774 .
flip(AMDGPU::FeatureWavefrontSize32);
5775 FeatureBitset AvailableFeaturesWS32 =
5776 ComputeAvailableFeatures(FeaturesWS32);
5778 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5779 return Error(IDLoc,
"instruction requires wavesize=32");
5783 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5784 return Error(IDLoc,
"instruction not supported on this GPU (" +
5785 getSTI().
getCPU() +
")" +
": " + Mnemo);
5790 return Error(IDLoc,
"invalid instruction" + Suggestion);
5796 const auto &
Op = ((AMDGPUOperand &)*Operands[InvalidOprIdx]);
5797 if (
Op.isToken() && InvalidOprIdx > 1) {
5798 const auto &PrevOp = ((AMDGPUOperand &)*Operands[InvalidOprIdx - 1]);
5799 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5804bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5807 uint64_t &ErrorInfo,
5808 bool MatchingInlineAsm) {
5811 unsigned Result = Match_Success;
5812 for (
auto Variant : getMatchedVariants()) {
5815 MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm, Variant);
5819 if (R == Match_Success || R == Match_MissingFeature ||
5820 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5821 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5822 Result != Match_MissingFeature)) {
5826 if (R == Match_Success)
5830 if (Result == Match_Success) {
5831 if (!validateInstruction(Inst, IDLoc, Operands)) {
5834 emitTargetDirective();
5841 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).
getToken();
5842 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5849 case Match_MissingFeature:
5853 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5855 case Match_InvalidOperand: {
5856 SMLoc ErrorLoc = IDLoc;
5857 if (ErrorInfo != ~0ULL) {
5858 if (ErrorInfo >= Operands.
size()) {
5859 return Error(IDLoc,
"too few operands for instruction");
5861 ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
5862 if (ErrorLoc == SMLoc())
5866 return Error(ErrorLoc,
"invalid VOPDY instruction");
5868 return Error(ErrorLoc,
"invalid operand for instruction");
5871 case Match_MnemonicFail:
5877bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5882 if (getParser().parseAbsoluteExpression(Tmp)) {
5885 Ret =
static_cast<uint32_t
>(Tmp);
5889bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5890 if (!getSTI().getTargetTriple().isAMDGCN())
5891 return TokError(
"directive only supported for amdgcn architecture");
5893 std::string TargetIDDirective;
5894 SMLoc TargetStart = getTok().getLoc();
5895 if (getParser().parseEscapedString(TargetIDDirective))
5898 std::optional<AMDGPU::TargetID> MaybeParsed =
5901 return getParser().Error(TargetStart,
5902 "malformed target id '" + TargetIDDirective +
"'");
5905 const Triple &
TT = getSTI().getTargetTriple();
5911 return getParser().Error(
5912 TargetStart,
"target id '" + TargetIDDirective +
5913 "' specifies a processor that is not valid for "
5915 TT.getArchName() +
"'");
5918 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5919 getTargetStreamer().getTargetID();
5922 const Triple &STITriple = getSTI().getTargetTriple();
5923 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5924 return getParser().Error(
5925 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5926 " is incompatible with " +
5927 Twine(CurrentTargetID->toString()));
5931 StringRef DirectiveProcessor =
5934 if (DirectiveISA != ISA) {
5935 return getParser().Error(TargetStart,
5936 ".amdgcn_target directive processor " +
5937 Twine(DirectiveProcessor) +
5938 " does not match the specified processor " +
5939 Twine(getSTI().
getCPU()));
5945 CurrentTargetID->getXnackSetting())) {
5947 ".amdgcn_target directive has conflicting xnack settings");
5951 CurrentTargetID->getSramEccSetting())) {
5953 ".amdgcn_target directive has conflicting sramecc settings");
5959 getTargetStreamer().getTargetID()->setXnackSetting(
5961 getTargetStreamer().getTargetID()->setSramEccSetting(
5967bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5971bool AMDGPUAsmParser::calculateGPRBlocks(
5972 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5973 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5974 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5975 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5976 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
5981 const MCExpr *
NumSGPRs = NextFreeSGPR;
5982 int64_t EvaluatedSGPRs;
5984 if (
ISA.Major >= 10)
5989 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
5990 !Features.
test(FeatureSGPRInitBug) &&
5991 static_cast<uint64_t
>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
5992 return OutOfRangeError(SGPRRange);
5994 const MCExpr *ExtraSGPRs =
5998 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
5999 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6000 static_cast<uint64_t
>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6001 return OutOfRangeError(SGPRRange);
6003 if (Features.
test(FeatureSGPRInitBug))
6010 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6011 unsigned Granule) ->
const MCExpr * {
6015 const MCExpr *AlignToGPR =
6017 const MCExpr *DivGPR =
6023 VGPRBlocks = GetNumGPRBlocks(
6032bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6033 if (!getSTI().getTargetTriple().isAMDGCN())
6034 return TokError(
"directive only supported for amdgcn architecture");
6037 return TokError(
"directive only supported for amdhsa OS");
6039 StringRef KernelName;
6040 if (getParser().parseIdentifier(KernelName))
6047 AMDGPU::MCKernelDescriptor KD =
6057 const MCExpr *NextFreeVGPR = ZeroExpr;
6059 const MCExpr *NamedBarCnt = ZeroExpr;
6060 uint64_t SharedVGPRCount = 0;
6061 uint64_t PreloadLength = 0;
6062 uint64_t PreloadOffset = 0;
6064 const MCExpr *NextFreeSGPR = ZeroExpr;
6067 unsigned ImpliedUserSGPRCount = 0;
6071 std::optional<unsigned> ExplicitUserSGPRCount;
6072 const MCExpr *ReserveVCC = OneExpr;
6073 const MCExpr *ReserveFlatScr = OneExpr;
6074 std::optional<bool> EnableWavefrontSize32;
6081 SMRange IDRange = getTok().getLocRange();
6082 if (!parseId(
ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6085 if (
ID ==
".end_amdhsa_kernel")
6089 return TokError(
".amdhsa_ directives cannot be repeated");
6091 SMLoc ValStart = getLoc();
6092 const MCExpr *ExprVal;
6093 if (getParser().parseExpression(ExprVal))
6095 SMLoc ValEnd = getLoc();
6096 SMRange ValRange = SMRange(ValStart, ValEnd);
6099 uint64_t Val = IVal;
6100 bool EvaluatableExpr;
6101 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6103 return OutOfRangeError(ValRange);
6107#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6108 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6109 return OutOfRangeError(RANGE); \
6110 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6115#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6117 return Error(IDRange.Start, "directive should have resolvable expression", \
6120 if (
ID ==
".amdhsa_group_segment_fixed_size") {
6123 return OutOfRangeError(ValRange);
6125 }
else if (
ID ==
".amdhsa_private_segment_fixed_size") {
6128 return OutOfRangeError(ValRange);
6130 }
else if (
ID ==
".amdhsa_kernarg_size") {
6132 return OutOfRangeError(ValRange);
6134 }
else if (
ID ==
".amdhsa_user_sgpr_count") {
6136 ExplicitUserSGPRCount = Val;
6137 }
else if (
ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6141 "directive is not supported with architected flat scratch",
6144 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6147 ImpliedUserSGPRCount += 4;
6148 }
else if (
ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6151 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6154 return OutOfRangeError(ValRange);
6158 ImpliedUserSGPRCount += Val;
6159 PreloadLength = Val;
6161 }
else if (
ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6164 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6167 return OutOfRangeError(ValRange);
6171 PreloadOffset = Val;
6172 }
else if (
ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6175 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6178 ImpliedUserSGPRCount += 2;
6179 }
else if (
ID ==
".amdhsa_user_sgpr_queue_ptr") {
6182 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6185 ImpliedUserSGPRCount += 2;
6186 }
else if (
ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6189 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6192 ImpliedUserSGPRCount += 2;
6193 }
else if (
ID ==
".amdhsa_user_sgpr_dispatch_id") {
6196 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6199 ImpliedUserSGPRCount += 2;
6200 }
else if (
ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6203 "directive is not supported with architected flat scratch",
6207 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6210 ImpliedUserSGPRCount += 2;
6211 }
else if (
ID ==
".amdhsa_user_sgpr_private_segment_size") {
6214 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6217 ImpliedUserSGPRCount += 1;
6218 }
else if (
ID ==
".amdhsa_wavefront_size32") {
6221 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6222 EnableWavefrontSize32 = Val;
6224 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6226 }
else if (
ID ==
".amdhsa_uses_dynamic_stack") {
6228 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6230 }
else if (
ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6233 "directive is not supported with architected flat scratch",
6236 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6238 }
else if (
ID ==
".amdhsa_enable_private_segment") {
6242 "directive is not supported without architected flat scratch",
6245 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6247 }
else if (
ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6249 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6251 }
else if (
ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6253 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6255 }
else if (
ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6257 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6259 }
else if (
ID ==
".amdhsa_system_sgpr_workgroup_info") {
6261 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6263 }
else if (
ID ==
".amdhsa_system_vgpr_workitem_id") {
6265 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6267 }
else if (
ID ==
".amdhsa_next_free_vgpr") {
6268 VGPRRange = ValRange;
6269 NextFreeVGPR = ExprVal;
6270 }
else if (
ID ==
".amdhsa_next_free_sgpr") {
6271 SGPRRange = ValRange;
6272 NextFreeSGPR = ExprVal;
6273 }
else if (
ID ==
".amdhsa_accum_offset") {
6275 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6276 AccumOffset = ExprVal;
6277 }
else if (
ID ==
".amdhsa_named_barrier_count") {
6279 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6280 NamedBarCnt = ExprVal;
6281 }
else if (
ID ==
".amdhsa_reserve_vcc") {
6283 return OutOfRangeError(ValRange);
6284 ReserveVCC = ExprVal;
6285 }
else if (
ID ==
".amdhsa_reserve_flat_scratch") {
6287 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6290 "directive is not supported with architected flat scratch",
6293 return OutOfRangeError(ValRange);
6294 ReserveFlatScr = ExprVal;
6295 }
else if (
ID ==
".amdhsa_reserve_xnack_mask") {
6297 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6299 return OutOfRangeError(ValRange);
6300 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6301 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6302 if (Val != XnackOn) {
6303 return getParser().Error(
6305 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6307 }
else if (
ID ==
".amdhsa_float_round_mode_32") {
6309 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6311 }
else if (
ID ==
".amdhsa_float_round_mode_16_64") {
6313 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6315 }
else if (
ID ==
".amdhsa_float_denorm_mode_32") {
6317 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6319 }
else if (
ID ==
".amdhsa_float_denorm_mode_16_64") {
6321 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6323 }
else if (
ID ==
".amdhsa_dx10_clamp") {
6324 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6325 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6328 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6330 }
else if (
ID ==
".amdhsa_ieee_mode") {
6331 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6332 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6335 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6337 }
else if (
ID ==
".amdhsa_fp16_overflow") {
6339 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6341 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6343 }
else if (
ID ==
".amdhsa_tg_split") {
6345 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6348 }
else if (
ID ==
".amdhsa_workgroup_processor_mode") {
6351 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6353 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6355 }
else if (
ID ==
".amdhsa_memory_ordered") {
6357 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6359 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6361 }
else if (
ID ==
".amdhsa_forward_progress") {
6363 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6365 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6367 }
else if (
ID ==
".amdhsa_shared_vgpr_count") {
6369 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6370 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6372 SharedVGPRCount = Val;
6374 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6376 }
else if (
ID ==
".amdhsa_inst_pref_size") {
6378 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6379 if (
ISA.Major == 11) {
6381 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6385 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6388 }
else if (
ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6391 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6393 }
else if (
ID ==
".amdhsa_exception_fp_denorm_src") {
6395 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6397 }
else if (
ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6400 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6402 }
else if (
ID ==
".amdhsa_exception_fp_ieee_overflow") {
6404 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6406 }
else if (
ID ==
".amdhsa_exception_fp_ieee_underflow") {
6408 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6410 }
else if (
ID ==
".amdhsa_exception_fp_ieee_inexact") {
6412 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6414 }
else if (
ID ==
".amdhsa_exception_int_div_zero") {
6416 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6418 }
else if (
ID ==
".amdhsa_round_robin_scheduling") {
6420 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6422 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6425 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6428#undef PARSE_BITS_ENTRY
6431 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6432 return TokError(
".amdhsa_next_free_vgpr directive is required");
6434 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6435 return TokError(
".amdhsa_next_free_sgpr directive is required");
6437 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6439 return TokError(
"too many user SGPRs enabled, found " +
6440 Twine(UserSGPRCount) +
", but only " +
6446 if (PreloadLength) {
6452 const MCExpr *VGPRBlocks;
6453 const MCExpr *SGPRBlocks;
6454 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6455 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6456 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6457 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6460 int64_t EvaluatedVGPRBlocks;
6461 bool VGPRBlocksEvaluatable =
6462 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6463 if (VGPRBlocksEvaluatable &&
6465 static_cast<uint64_t
>(EvaluatedVGPRBlocks))) {
6466 return OutOfRangeError(VGPRRange);
6470 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6471 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6473 int64_t EvaluatedSGPRBlocks;
6474 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6476 static_cast<uint64_t
>(EvaluatedSGPRBlocks)))
6477 return OutOfRangeError(SGPRRange);
6480 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6481 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6483 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6484 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6485 "enabled user SGPRs");
6491 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6492 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6497 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6498 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6503 return TokError(
"Kernarg size should be resolvable");
6504 uint64_t kernarg_size = IVal;
6505 if (PreloadLength && kernarg_size &&
6506 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6507 return TokError(
"Kernarg preload length + offset is larger than the "
6508 "kernarg segment size");
6511 if (!Seen.
contains(
".amdhsa_accum_offset"))
6512 return TokError(
".amdhsa_accum_offset directive is required");
6513 int64_t EvaluatedAccum;
6514 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6515 uint64_t UEvaluatedAccum = EvaluatedAccum;
6516 if (AccumEvaluatable &&
6517 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6518 return TokError(
"accum_offset should be in range [4..256] in "
6521 int64_t EvaluatedNumVGPR;
6522 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6525 alignTo(std::max((uint64_t)1, (uint64_t)EvaluatedNumVGPR), 4))
6526 return TokError(
"accum_offset exceeds total VGPR allocation");
6532 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6533 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6539 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6540 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6543 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6545 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6546 return TokError(
"shared_vgpr_count directive not valid on "
6547 "wavefront size 32");
6550 if (VGPRBlocksEvaluatable &&
6551 (SharedVGPRCount * 2 +
static_cast<uint64_t
>(EvaluatedVGPRBlocks) >
6553 return TokError(
"shared_vgpr_count*2 + "
6554 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6559 emitTargetDirective();
6560 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6561 NextFreeVGPR, NextFreeSGPR,
6562 ReserveVCC, ReserveFlatScr);
6566bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6568 if (ParseAsAbsoluteExpression(
Version))
6571 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6572 emitTargetDirective();
6576bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef
ID,
6577 AMDGPUMCKernelCodeT &
C) {
6580 if (
ID ==
"max_scratch_backing_memory_byte_size") {
6581 Parser.eatToEndOfStatement();
6585 SmallString<40> ErrStr;
6586 raw_svector_ostream Err(ErrStr);
6587 if (!
C.ParseKernelCodeT(
ID, getParser(), Err)) {
6588 return TokError(Err.
str());
6592 if (
ID ==
"enable_wavefront_size32") {
6595 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6597 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6600 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6604 if (
ID ==
"wavefront_size") {
6605 if (
C.wavefront_size == 5) {
6607 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6609 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6610 }
else if (
C.wavefront_size == 6) {
6612 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6619bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6620 AMDGPUMCKernelCodeT KernelCode;
6630 if (!parseId(
ID,
"expected value identifier or .end_amd_kernel_code_t"))
6633 if (
ID ==
".end_amd_kernel_code_t")
6636 if (ParseAMDKernelCodeTValue(
ID, KernelCode))
6641 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6646bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6647 StringRef KernelName;
6648 if (!parseId(KernelName,
"expected symbol name"))
6651 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6658bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6659 if (!getSTI().getTargetTriple().isAMDGCN()) {
6660 return Error(getLoc(),
6661 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6665 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6667 std::optional<AMDGPU::TargetID> MaybeParsed =
6670 return Error(getParser().getTok().getLoc(),
6671 "malformed target id '" + TargetIDDirective +
"'");
6674 const Triple &
TT = getSTI().getTargetTriple();
6680 return Error(getParser().getTok().getLoc(),
6681 "target id '" + TargetIDDirective +
6682 "' specifies a processor that is not valid for subarch '" +
6683 TT.getArchName() +
"'");
6686 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6687 getTargetStreamer().getTargetID();
6690 const Triple &STITriple = getSTI().getTargetTriple();
6691 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6692 return Error(getParser().getTok().getLoc(),
6693 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6694 " is incompatible with " +
6695 Twine(CurrentTargetID->toString()));
6699 StringRef DirectiveProcessor =
6702 if (DirectiveISA != ISA) {
6703 return Error(getParser().getTok().getLoc(),
6704 ".amd_amdgpu_isa directive processor " +
6705 Twine(DirectiveProcessor) +
6706 " does not match the specified processor " +
6707 Twine(getSTI().
getCPU()));
6710 getTargetStreamer().EmitISAVersion();
6716bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6719 std::string HSAMetadataString;
6724 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6725 return Error(getLoc(),
"invalid HSA metadata");
6732bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6733 const char *AssemblerDirectiveEnd,
6734 std::string &CollectString) {
6736 raw_string_ostream CollectStream(CollectString);
6738 getLexer().setSkipSpace(
false);
6740 bool FoundEnd =
false;
6743 CollectStream << getTokenStr();
6747 if (trySkipId(AssemblerDirectiveEnd)) {
6752 CollectStream << Parser.parseStringToEndOfStatement()
6753 <<
getContext().getAsmInfo().getSeparatorString();
6755 Parser.eatToEndOfStatement();
6758 getLexer().setSkipSpace(
true);
6761 return TokError(Twine(
"expected directive ") +
6762 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6769bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6775 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6776 if (!PALMetadata->setFromString(
String))
6777 return Error(getLoc(),
"invalid PAL metadata");
6782bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6785 Twine(
" directive is "
6786 "not available on non-amdpal OSes"))
6790 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6791 PALMetadata->setLegacy();
6794 if (ParseAsAbsoluteExpression(
Key)) {
6795 return TokError(Twine(
"invalid value in ") +
6799 return TokError(Twine(
"expected an even number of values in ") +
6802 if (ParseAsAbsoluteExpression(
Value)) {
6803 return TokError(Twine(
"invalid value in ") +
6806 PALMetadata->setRegister(
Key,
Value);
6815bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6816 if (getParser().checkForValidSection())
6820 SMLoc NameLoc = getLoc();
6821 if (getParser().parseIdentifier(Name))
6822 return TokError(
"expected identifier in directive");
6825 if (getParser().parseComma())
6831 SMLoc SizeLoc = getLoc();
6832 if (getParser().parseAbsoluteExpression(
Size))
6835 return Error(SizeLoc,
"size must be non-negative");
6836 if (
Size > LocalMemorySize)
6837 return Error(SizeLoc,
"size is too large");
6839 int64_t Alignment = 4;
6841 SMLoc AlignLoc = getLoc();
6842 if (getParser().parseAbsoluteExpression(Alignment))
6845 return Error(AlignLoc,
"alignment must be a power of two");
6850 if (Alignment >= 1u << 31)
6851 return Error(AlignLoc,
"alignment is too large");
6857 Symbol->redefineIfPossible();
6858 if (!
Symbol->isUndefined())
6859 return Error(NameLoc,
"invalid symbol redefinition");
6861 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6865bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6866 if (getParser().checkForValidSection())
6870 if (getParser().parseIdentifier(FuncName))
6871 return TokError(
"expected symbol name after .amdgpu_info");
6874 AMDGPU::InfoSectionData ParsedInfoData;
6875 AMDGPU::FuncInfo FI;
6877 bool HasScalarAttrs =
false;
6884 SMLoc IDLoc = getLoc();
6885 if (!parseId(
ID,
"expected directive or .end_amdgpu_info"))
6888 if (
ID ==
".end_amdgpu_info")
6896 return Error(IDLoc,
"unknown .amdgpu_info directive '" +
ID +
"'");
6898 if (Dir ==
"flags") {
6900 if (getParser().parseAbsoluteExpression(Val))
6903 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6905 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6907 HasScalarAttrs =
true;
6908 }
else if (Dir ==
"num_sgpr") {
6910 if (getParser().parseAbsoluteExpression(Val))
6912 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6913 HasScalarAttrs =
true;
6914 }
else if (Dir ==
"num_vgpr") {
6916 if (getParser().parseAbsoluteExpression(Val))
6919 HasScalarAttrs =
true;
6920 }
else if (Dir ==
"num_agpr") {
6922 if (getParser().parseAbsoluteExpression(Val))
6925 HasScalarAttrs =
true;
6926 }
else if (Dir ==
"private_segment_size") {
6928 if (getParser().parseAbsoluteExpression(Val))
6931 HasScalarAttrs =
true;
6932 }
else if (Dir ==
"use") {
6934 if (getParser().parseIdentifier(ResName))
6935 return TokError(
"expected resource symbol for .amdgpu_use");
6936 ParsedInfoData.
Uses.push_back(
6937 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6938 }
else if (Dir ==
"call") {
6940 if (getParser().parseIdentifier(DstName))
6941 return TokError(
"expected callee symbol for .amdgpu_call");
6942 ParsedInfoData.
Calls.push_back(
6943 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6944 }
else if (Dir ==
"indirect_call") {
6946 if (getParser().parseEscapedString(TypeId))
6947 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6948 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6949 }
else if (Dir ==
"typeid") {
6951 if (getParser().parseEscapedString(TypeId))
6952 return TokError(
"expected type ID string for .amdgpu_typeid");
6953 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6955 return Error(IDLoc,
"unknown .amdgpu_info directive '" +
ID +
"'");
6960 ParsedInfoData.
Funcs.push_back(std::move(FI));
6962 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6963 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6964 Data.Funcs.push_back(std::move(Func));
6965 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6966 Data.Uses.push_back(Use);
6967 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6969 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6972 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6973 Data.TypeIds.push_back(std::move(TypeId));
6978void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
6985void AMDGPUAsmParser::checkKernelPrologues() {
6986 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
6987 static const unsigned Required[] = {GLOBAL_PREFETCH_B8_SADDR_gfx1250,
6989 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
6990 if (!AMDHSAKernelSymbols.
contains(Sym))
6992 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
6995 "' does not begin with the required prologue "
6996 "sequence: GLOBAL_PREFETCH_B8 followed by V_NOP");
7000 OpcodeStream.
clear();
7001 OpcodeStreamSymbols.clear();
7002 AMDHSAKernelSymbols.
clear();
7005void AMDGPUAsmParser::onEndOfFile() {
7006 emitTargetDirective();
7007 checkKernelPrologues();
7009 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7012bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7013 StringRef IDVal = DirectiveID.
getString();
7016 if (IDVal ==
".amdhsa_kernel")
7017 return ParseDirectiveAMDHSAKernel();
7019 if (IDVal ==
".amdhsa_code_object_version")
7020 return ParseDirectiveAMDHSACodeObjectVersion();
7024 return ParseDirectiveHSAMetadata();
7026 if (IDVal ==
".amd_kernel_code_t")
7027 return ParseDirectiveAMDKernelCodeT();
7029 if (IDVal ==
".amdgpu_hsa_kernel")
7030 return ParseDirectiveAMDGPUHsaKernel();
7032 if (IDVal ==
".amd_amdgpu_isa")
7033 return ParseDirectiveISAVersion();
7037 Twine(
" directive is "
7038 "not available on non-amdhsa OSes"))
7043 if (IDVal ==
".amdgcn_target")
7044 return ParseDirectiveAMDGCNTarget();
7046 if (IDVal ==
".amdgpu_lds")
7047 return ParseDirectiveAMDGPULDS();
7049 if (IDVal ==
".amdgpu_info")
7050 return ParseDirectiveAMDGPUInfo();
7053 return ParseDirectivePALMetadataBegin();
7056 return ParseDirectivePALMetadata();
7061bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7068 return hasSGPR104_SGPR105();
7071 case SRC_SHARED_BASE_LO:
7072 case SRC_SHARED_BASE:
7073 case SRC_SHARED_LIMIT_LO:
7074 case SRC_SHARED_LIMIT:
7075 case SRC_PRIVATE_BASE_LO:
7076 case SRC_PRIVATE_BASE:
7077 case SRC_PRIVATE_LIMIT_LO:
7078 case SRC_PRIVATE_LIMIT:
7080 case SRC_FLAT_SCRATCH_BASE_LO:
7081 case SRC_FLAT_SCRATCH_BASE_HI:
7082 return hasGloballyAddressableScratch();
7083 case SRC_POPS_EXITING_WAVE_ID:
7096 getTargetStreamer().getTargetID()->isXnackSupported();
7126 return hasSGPR102_SGPR103();
7131ParseStatus AMDGPUAsmParser::parseOperand(
OperandVector &Operands,
7134 ParseStatus Res = parseVOPD(Operands);
7139 Res = MatchOperandParserImpl(Operands, Mnemonic);
7151 SMLoc LBraceLoc = getLoc();
7156 auto Loc = getLoc();
7157 Res = parseReg(Operands);
7159 Error(Loc,
"expected a register");
7163 RBraceLoc = getLoc();
7168 "expected a comma or a closing square bracket"))
7172 if (Operands.
size() - Prefix > 1) {
7174 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7175 Operands.
push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7181 return parseRegOrImm(Operands);
7184StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7186 setForcedEncodingSize(0);
7187 setForcedDPP(
false);
7188 setForcedSDWA(
false);
7190 if (
Name.consume_back(
"_e64_dpp")) {
7192 setForcedEncodingSize(64);
7195 if (
Name.consume_back(
"_e64")) {
7196 setForcedEncodingSize(64);
7199 if (
Name.consume_back(
"_e32")) {
7200 setForcedEncodingSize(32);
7203 if (
Name.consume_back(
"_dpp")) {
7207 if (
Name.consume_back(
"_sdwa")) {
7208 setForcedSDWA(
true);
7216 unsigned VariantID);
7222 Name = parseMnemonicSuffix(Name);
7228 Operands.
push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7230 bool IsMIMG = Name.starts_with(
"image_");
7233 OperandMode
Mode = OperandMode_Default;
7235 Mode = OperandMode_NSA;
7239 checkUnsupportedInstruction(Name, NameLoc);
7240 if (!Parser.hasPendingError()) {
7243 :
"not a valid operand.";
7263ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7266 if (!trySkipId(Name))
7269 Operands.
push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7273ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7282ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7283 const char *Prefix,
OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy,
7284 std::function<
bool(int64_t &)> ConvertResult) {
7288 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7292 if (ConvertResult && !ConvertResult(
Value)) {
7293 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7296 Operands.
push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7300ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7301 const char *Prefix,
OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy,
7302 bool (*ConvertResult)(int64_t &)) {
7311 const unsigned MaxSize = 4;
7315 for (
int I = 0;; ++
I) {
7317 SMLoc Loc = getLoc();
7321 if (
Op != 0 &&
Op != 1)
7322 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7329 if (
I + 1 == MaxSize)
7330 return Error(getLoc(),
"expected a closing square bracket");
7336 Operands.
push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7340ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7342 AMDGPUOperand::ImmTy ImmTy,
7343 bool IgnoreNegative) {
7347 if (trySkipId(Name)) {
7349 }
else if (trySkipId(
"no", Name)) {
7358 return Error(S,
"r128 modifier is not supported on this GPU");
7359 if (Name ==
"a16" && !
hasA16())
7360 return Error(S,
"a16 modifier is not supported on this GPU");
7362 if (Bit == 0 && Name ==
"gds") {
7363 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).
getToken();
7365 return Error(S,
"nogds is not allowed");
7368 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7369 ImmTy = AMDGPUOperand::ImmTyR128A16;
7371 Operands.
push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7375unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7376 bool &Disabling)
const {
7377 Disabling =
Id.consume_front(
"no");
7380 return StringSwitch<unsigned>(Id)
7387 return StringSwitch<unsigned>(Id)
7395ParseStatus AMDGPUAsmParser::parseCPol(
OperandVector &Operands) {
7397 SMLoc StringLoc = getLoc();
7399 int64_t CPolVal = 0;
7408 ResTH = parseTH(Operands, TH);
7419 ResScope = parseScope(Operands, Scope);
7432 if (trySkipId(
"nv")) {
7436 }
else if (trySkipId(
"no",
"nv")) {
7443 if (trySkipId(
"scale_offset")) {
7447 }
else if (trySkipId(
"no",
"scale_offset")) {
7460 Operands.
push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7461 AMDGPUOperand::ImmTyCPol));
7465 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).
getToken();
7466 SMLoc OpLoc = getLoc();
7467 unsigned Enabled = 0, Seen = 0;
7471 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7478 return Error(S,
"dlc modifier is not supported on this GPU");
7481 return Error(S,
"scc modifier is not supported on this GPU");
7484 return Error(S,
"duplicate cache policy modifier");
7496 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7500ParseStatus AMDGPUAsmParser::parseScope(
OperandVector &Operands,
7505 ParseStatus Res = parseStringOrIntWithPrefix(
7506 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7515ParseStatus AMDGPUAsmParser::parseTH(
OperandVector &Operands, int64_t &TH) {
7520 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7524 if (
Value ==
"TH_DEFAULT")
7526 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7527 Value ==
"TH_LOAD_NT_WB") {
7528 return Error(StringLoc,
"invalid th value");
7529 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7531 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7533 }
else if (
Value.consume_front(
"TH_STORE_")) {
7536 return Error(StringLoc,
"invalid th value");
7539 if (
Value ==
"BYPASS")
7544 TH |= StringSwitch<int64_t>(
Value)
7554 .Default(0xffffffff);
7556 TH |= StringSwitch<int64_t>(
Value)
7567 .Default(0xffffffff);
7570 if (TH == 0xffffffff)
7571 return Error(StringLoc,
"invalid th value");
7578 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7579 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7580 std::optional<unsigned> InsertAt = std::nullopt) {
7581 auto i = OptionalIdx.find(ImmT);
7582 if (i != OptionalIdx.end()) {
7583 unsigned Idx = i->second;
7584 const AMDGPUOperand &
Op =
7585 static_cast<const AMDGPUOperand &
>(*Operands[Idx]);
7589 Op.addImmOperands(Inst, 1);
7591 if (InsertAt.has_value())
7598ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7604 StringLoc = getLoc();
7609ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7610 OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids,
7615 SMLoc StringLoc = getLoc();
7619 Value = getTokenStr();
7623 if (
Value == Ids[IntVal])
7628 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7629 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7634ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7635 OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids,
7636 AMDGPUOperand::ImmTy
Type) {
7640 ParseStatus Res = parseStringOrIntWithPrefix(Operands, Name, Ids, IntVal);
7642 Operands.
push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7651bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7654 SMLoc Loc = getLoc();
7656 auto Res = parseIntWithPrefix(Pref, Val);
7662 if (Val < 0 || Val > MaxVal) {
7663 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7671ParseStatus AMDGPUAsmParser::tryParseIndexKey(
OperandVector &Operands,
7672 AMDGPUOperand::ImmTy ImmTy) {
7673 const char *Pref =
"index_key";
7675 SMLoc Loc = getLoc();
7676 auto Res = parseIntWithPrefix(Pref, ImmVal);
7680 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7681 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7682 (ImmVal < 0 || ImmVal > 1))
7683 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7685 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7686 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7688 Operands.
push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7692ParseStatus AMDGPUAsmParser::parseIndexKey8bit(
OperandVector &Operands) {
7693 return tryParseIndexKey(Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7696ParseStatus AMDGPUAsmParser::parseIndexKey16bit(
OperandVector &Operands) {
7697 return tryParseIndexKey(Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7700ParseStatus AMDGPUAsmParser::parseIndexKey32bit(
OperandVector &Operands) {
7701 return tryParseIndexKey(Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7704ParseStatus AMDGPUAsmParser::tryParseMatrixFMT(
OperandVector &Operands,
7706 AMDGPUOperand::ImmTy
Type) {
7711ParseStatus AMDGPUAsmParser::parseMatrixAFMT(
OperandVector &Operands) {
7712 return tryParseMatrixFMT(Operands,
"matrix_a_fmt",
7713 AMDGPUOperand::ImmTyMatrixAFMT);
7716ParseStatus AMDGPUAsmParser::parseMatrixBFMT(
OperandVector &Operands) {
7717 return tryParseMatrixFMT(Operands,
"matrix_b_fmt",
7718 AMDGPUOperand::ImmTyMatrixBFMT);
7721ParseStatus AMDGPUAsmParser::tryParseMatrixScale(
OperandVector &Operands,
7723 AMDGPUOperand::ImmTy
Type) {
7728ParseStatus AMDGPUAsmParser::parseMatrixAScale(
OperandVector &Operands) {
7729 return tryParseMatrixScale(Operands,
"matrix_a_scale",
7730 AMDGPUOperand::ImmTyMatrixAScale);
7733ParseStatus AMDGPUAsmParser::parseMatrixBScale(
OperandVector &Operands) {
7734 return tryParseMatrixScale(Operands,
"matrix_b_scale",
7735 AMDGPUOperand::ImmTyMatrixBScale);
7738ParseStatus AMDGPUAsmParser::tryParseMatrixScaleFmt(
OperandVector &Operands,
7740 AMDGPUOperand::ImmTy
Type) {
7745ParseStatus AMDGPUAsmParser::parseMatrixAScaleFmt(
OperandVector &Operands) {
7746 return tryParseMatrixScaleFmt(Operands,
"matrix_a_scale_fmt",
7747 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7750ParseStatus AMDGPUAsmParser::parseMatrixBScaleFmt(
OperandVector &Operands) {
7751 return tryParseMatrixScaleFmt(Operands,
"matrix_b_scale_fmt",
7752 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7757ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7758 using namespace llvm::AMDGPU::MTBUFFormat;
7764 for (
int I = 0;
I < 2; ++
I) {
7765 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7768 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7773 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7779 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7782 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7783 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7789ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7790 using namespace llvm::AMDGPU::MTBUFFormat;
7794 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7797 if (Fmt == UFMT_UNDEF)
7804bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7805 StringRef FormatStr, SMLoc Loc) {
7806 using namespace llvm::AMDGPU::MTBUFFormat;
7810 if (
Format != DFMT_UNDEF) {
7816 if (
Format != NFMT_UNDEF) {
7821 Error(Loc,
"unsupported format");
7825ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7828 using namespace llvm::AMDGPU::MTBUFFormat;
7832 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7837 SMLoc Loc = getLoc();
7838 if (!parseId(Str,
"expected a format string") ||
7839 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7841 if (Dfmt == DFMT_UNDEF)
7842 return Error(Loc,
"duplicate numeric format");
7843 if (Nfmt == NFMT_UNDEF)
7844 return Error(Loc,
"duplicate data format");
7847 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7848 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7852 if (Ufmt == UFMT_UNDEF)
7853 return Error(FormatLoc,
"unsupported format");
7862ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7865 using namespace llvm::AMDGPU::MTBUFFormat;
7868 if (Id == UFMT_UNDEF)
7872 return Error(Loc,
"unified format is not supported on this GPU");
7878ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7879 using namespace llvm::AMDGPU::MTBUFFormat;
7880 SMLoc Loc = getLoc();
7885 return Error(Loc,
"out of range format");
7890ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7891 using namespace llvm::AMDGPU::MTBUFFormat;
7897 StringRef FormatStr;
7898 SMLoc Loc = getLoc();
7899 if (!parseId(FormatStr,
"expected a format string"))
7902 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7904 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7914 return parseNumericFormat(
Format);
7917ParseStatus AMDGPUAsmParser::parseFORMAT(
OperandVector &Operands) {
7918 using namespace llvm::AMDGPU::MTBUFFormat;
7922 SMLoc Loc = getLoc();
7932 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7944 Res = parseRegOrImm(Operands);
7951 Res = parseSymbolicOrNumericFormat(
Format);
7956 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*Operands[
Size - 2]);
7957 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7964 return Error(getLoc(),
"duplicate format");
7968ParseStatus AMDGPUAsmParser::parseFlatOffset(
OperandVector &Operands) {
7970 parseIntWithPrefix(
"offset", Operands, AMDGPUOperand::ImmTyOffset);
7972 Res = parseIntWithPrefix(
"inst_offset", Operands,
7973 AMDGPUOperand::ImmTyInstOffset);
7978ParseStatus AMDGPUAsmParser::parseR128A16(
OperandVector &Operands) {
7980 parseNamedBit(
"r128", Operands, AMDGPUOperand::ImmTyR128A16);
7982 Res = parseNamedBit(
"a16", Operands, AMDGPUOperand::ImmTyA16);
7986ParseStatus AMDGPUAsmParser::parseBLGP(
OperandVector &Operands) {
7988 parseIntWithPrefix(
"blgp", Operands, AMDGPUOperand::ImmTyBLGP);
7991 parseOperandArrayWithPrefix(
"neg", Operands, AMDGPUOperand::ImmTyBLGP);
8000void AMDGPUAsmParser::cvtExp(MCInst &Inst,
const OperandVector &Operands) {
8001 OptionalImmIndexMap OptionalIdx;
8003 unsigned OperandIdx[4];
8004 unsigned EnMask = 0;
8007 for (
unsigned i = 1, e = Operands.
size(); i != e; ++i) {
8008 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
8013 OperandIdx[SrcIdx] = Inst.
size();
8014 Op.addRegOperands(Inst, 1);
8021 OperandIdx[SrcIdx] = Inst.
size();
8027 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8028 Op.addImmOperands(Inst, 1);
8032 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8036 OptionalIdx[
Op.getImmTy()] = i;
8042 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8049 for (
auto i = 0; i < SrcIdx; ++i) {
8051 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8057 AMDGPUOperand::ImmTyExpCompr);
8067 int64_t CntVal,
bool Saturate,
8073 IntVal =
encode(ISA, IntVal, CntVal);
8074 if (CntVal !=
decode(ISA, IntVal)) {
8076 IntVal =
encode(ISA, IntVal, -1);
8084bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8086 SMLoc CntLoc = getLoc();
8087 StringRef CntName = getTokenStr();
8094 SMLoc ValLoc = getLoc();
8101 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8103 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8105 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8108 Error(CntLoc,
"invalid counter name " + CntName);
8113 Error(ValLoc,
"too large value for " + CntName);
8122 Error(getLoc(),
"expected a counter name");
8130ParseStatus AMDGPUAsmParser::parseSWaitCnt(
OperandVector &Operands) {
8136 if (!parseCnt(Waitcnt))
8144 Operands.
push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8148bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8149 SMLoc FieldLoc = getLoc();
8150 StringRef FieldName = getTokenStr();
8155 SMLoc ValueLoc = getLoc();
8162 if (FieldName ==
"instid0") {
8164 }
else if (FieldName ==
"instskip") {
8166 }
else if (FieldName ==
"instid1") {
8169 Error(FieldLoc,
"invalid field name " + FieldName);
8188 .Case(
"VALU_DEP_1", 1)
8189 .Case(
"VALU_DEP_2", 2)
8190 .Case(
"VALU_DEP_3", 3)
8191 .Case(
"VALU_DEP_4", 4)
8192 .Case(
"TRANS32_DEP_1", 5)
8193 .Case(
"TRANS32_DEP_2", 6)
8194 .Case(
"TRANS32_DEP_3", 7)
8195 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8196 .Case(
"SALU_CYCLE_1", 9)
8197 .Case(
"SALU_CYCLE_2", 10)
8198 .Case(
"SALU_CYCLE_3", 11)
8206 Delay |=
Value << Shift;
8210ParseStatus AMDGPUAsmParser::parseSDelayALU(
OperandVector &Operands) {
8216 if (!parseDelay(Delay))
8224 Operands.
push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8228bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8230bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8236void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8237 StringRef DepCtrName) {
8240 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8243 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8246 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8249 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8256bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8258 using namespace llvm::AMDGPU::DepCtr;
8260 SMLoc DepCtrLoc = getLoc();
8261 StringRef DepCtrName = getTokenStr();
8271 unsigned PrevOprMask = UsedOprMask;
8272 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8275 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8284 Error(getLoc(),
"expected a counter name");
8289 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8290 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8294ParseStatus AMDGPUAsmParser::parseDepCtr(
OperandVector &Operands) {
8295 using namespace llvm::AMDGPU::DepCtr;
8298 SMLoc Loc = getLoc();
8301 unsigned UsedOprMask = 0;
8303 if (!parseDepCtr(DepCtr, UsedOprMask))
8311 Operands.
push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8315bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8321ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8323 OperandInfoTy &Width) {
8324 using namespace llvm::AMDGPU::Hwreg;
8330 HwReg.Loc = getLoc();
8333 HwReg.IsSymbolic =
true;
8335 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8343 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8353 Width.Loc = getLoc();
8361ParseStatus AMDGPUAsmParser::parseHwreg(
OperandVector &Operands) {
8362 using namespace llvm::AMDGPU::Hwreg;
8365 SMLoc Loc = getLoc();
8367 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8369 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8370 HwregOffset::Default);
8371 struct : StructuredOpField {
8372 using StructuredOpField::StructuredOpField;
8373 bool validate(AMDGPUAsmParser &Parser)
const override {
8375 return Error(Parser,
"only values from 1 to 32 are legal");
8378 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8379 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8382 Res = parseHwregFunc(HwReg,
Offset, Width);
8385 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8387 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8391 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8398 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8400 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8404bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8410bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8411 OperandInfoTy &Stream) {
8412 using namespace llvm::AMDGPU::SendMsg;
8417 Msg.IsSymbolic =
true;
8424 Op.IsDefined =
true;
8430 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8435 Stream.IsDefined =
true;
8436 Stream.Loc = getLoc();
8445bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8446 const OperandInfoTy &
Op,
8447 const OperandInfoTy &Stream) {
8448 using namespace llvm::AMDGPU::SendMsg;
8457 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8462 Error(
Msg.Loc,
"invalid message id");
8468 Error(
Op.Loc,
"message does not support operations");
8470 Error(
Msg.Loc,
"missing message operation");
8476 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8478 Error(
Op.Loc,
"invalid operation id");
8483 Error(Stream.Loc,
"message operation does not support streams");
8487 Error(Stream.Loc,
"invalid message stream id");
8493ParseStatus AMDGPUAsmParser::parseSendMsg(
OperandVector &Operands) {
8494 using namespace llvm::AMDGPU::SendMsg;
8497 SMLoc Loc = getLoc();
8501 OperandInfoTy
Op(OP_NONE_);
8502 OperandInfoTy Stream(STREAM_ID_NONE_);
8503 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8508 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8510 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8516 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8520bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8522ParseStatus AMDGPUAsmParser::parseWaitEvent(
OperandVector &Operands) {
8523 using namespace llvm::AMDGPU::WaitEvent;
8525 SMLoc Loc = getLoc();
8528 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8530 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8532 StructuredOpField *TargetBitfield =
8533 isGFX11() ? &DontWaitExportReady : &ExportReady;
8535 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8539 if (!validateStructuredOpFields({TargetBitfield}))
8541 ImmVal = TargetBitfield->Val;
8548 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8550 Operands.
push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8551 AMDGPUOperand::ImmTyWaitEvent));
8555bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8561ParseStatus AMDGPUAsmParser::parseInterpSlot(
OperandVector &Operands) {
8568 int Slot = StringSwitch<int>(Str)
8575 return Error(S,
"invalid interpolation slot");
8578 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8582ParseStatus AMDGPUAsmParser::parseInterpAttr(
OperandVector &Operands) {
8589 if (!Str.starts_with(
"attr"))
8590 return Error(S,
"invalid interpolation attribute");
8592 StringRef Chan = Str.take_back(2);
8593 int AttrChan = StringSwitch<int>(Chan)
8600 return Error(S,
"invalid or missing interpolation attribute channel");
8602 Str = Str.drop_back(2).drop_front(4);
8605 if (Str.getAsInteger(10, Attr))
8606 return Error(S,
"invalid or missing interpolation attribute number");
8609 return Error(S,
"out of bounds interpolation attribute number");
8614 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8615 Operands.
push_back(AMDGPUOperand::CreateImm(
8616 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8624ParseStatus AMDGPUAsmParser::parseExpTgt(
OperandVector &Operands) {
8625 using namespace llvm::AMDGPU::Exp;
8635 return Error(S, (Id == ET_INVALID)
8636 ?
"invalid exp target"
8637 :
"exp target is not supported on this GPU");
8640 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8648bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8652bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8657 return getTokenKind() ==
Kind;
8660StringRef AMDGPUAsmParser::getId()
const {
8664bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8672bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8674 StringRef Tok = getTokenStr();
8683bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8685 if (isId(Id) && peekToken().is(Kind)) {
8694 if (isToken(Kind)) {
8702 const StringRef ErrMsg) {
8703 if (!trySkipToken(Kind)) {
8704 Error(getLoc(), ErrMsg);
8710bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8714 if (Parser.parseExpression(Expr))
8717 if (Expr->evaluateAsAbsolute(
Imm))
8720 if (Expected.empty()) {
8721 Error(S,
"expected absolute expression");
8724 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8733 if (Parser.parseExpression(Expr))
8737 if (Expr->evaluateAsAbsolute(IntVal)) {
8738 Operands.
push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8740 Operands.
push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8745bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8747 Val =
getToken().getStringContents();
8751 Error(getLoc(), ErrMsg);
8755bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8757 Val = getTokenStr();
8761 if (!ErrMsg.
empty())
8762 Error(getLoc(), ErrMsg);
8766AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8768AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8771 : getLexer().peekTok(ShouldSkipSpace);
8775 auto TokCount = getLexer().peekTokens(Tokens);
8777 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8782 return getLexer().getKind();
8785SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8787StringRef AMDGPUAsmParser::getTokenStr()
const {
8791void AMDGPUAsmParser::lex() { Parser.Lex(); }
8793const AMDGPUOperand &
8794AMDGPUAsmParser::findMCOperand(
const OperandVector &Operands,
8795 int MCOpIdx)
const {
8796 for (
const auto &
Op : Operands) {
8797 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8798 if (TargetOp.getMCOpIdx() == MCOpIdx)
8804SMLoc AMDGPUAsmParser::getInstLoc(
const OperandVector &Operands)
const {
8805 return ((AMDGPUOperand &)*Operands[0]).getStartLoc();
8809SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8813SMLoc AMDGPUAsmParser::getOperandLoc(
const OperandVector &Operands,
8814 int MCOpIdx)
const {
8815 return findMCOperand(Operands, MCOpIdx).getStartLoc();
8818SMLoc AMDGPUAsmParser::getOperandLoc(
8819 std::function<
bool(
const AMDGPUOperand &)>
Test,
8821 for (
unsigned i = Operands.
size() - 1; i > 0; --i) {
8822 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
8824 return Op.getStartLoc();
8826 return getInstLoc(Operands);
8829SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8831 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8832 return getOperandLoc(
Test, Operands);
8846 StringRef
Id = getTokenStr();
8847 SMLoc IdLoc = getLoc();
8853 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8854 if (
I == Fields.
end())
8855 return Error(IdLoc,
"unknown field");
8856 if ((*I)->IsDefined)
8857 return Error(IdLoc,
"duplicate field");
8860 (*I)->Loc = getLoc();
8863 (*I)->IsDefined =
true;
8870bool AMDGPUAsmParser::validateStructuredOpFields(
8872 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8873 return F->validate(*
this);
8883 const unsigned XorMask) {
8890bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8891 const unsigned MaxVal,
8892 const Twine &ErrMsg, SMLoc &Loc) {
8908bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8909 const unsigned MinVal,
8910 const unsigned MaxVal,
8911 const StringRef ErrMsg) {
8913 for (
unsigned i = 0; i < OpNum; ++i) {
8914 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8921bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8922 using namespace llvm::AMDGPU::Swizzle;
8925 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8926 "expected a 2-bit lane id")) {
8936bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8937 using namespace llvm::AMDGPU::Swizzle;
8943 if (!parseSwizzleOperand(GroupSize, 2, 32,
8944 "group size must be in the interval [2,32]", Loc)) {
8948 Error(Loc,
"group size must be a power of two");
8951 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8952 "lane id must be in the interval [0,group size - 1]",
8960bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8961 using namespace llvm::AMDGPU::Swizzle;
8966 if (!parseSwizzleOperand(GroupSize, 2, 32,
8967 "group size must be in the interval [2,32]", Loc)) {
8971 Error(Loc,
"group size must be a power of two");
8979bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
8980 using namespace llvm::AMDGPU::Swizzle;
8985 if (!parseSwizzleOperand(GroupSize, 1, 16,
8986 "group size must be in the interval [1,16]", Loc)) {
8990 Error(Loc,
"group size must be a power of two");
8998bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
8999 using namespace llvm::AMDGPU::Swizzle;
9006 SMLoc StrLoc = getLoc();
9007 if (!parseString(Ctl)) {
9010 if (Ctl.
size() != BITMASK_WIDTH) {
9011 Error(StrLoc,
"expected a 5-character mask");
9015 unsigned AndMask = 0;
9016 unsigned OrMask = 0;
9017 unsigned XorMask = 0;
9019 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9023 Error(StrLoc,
"invalid mask");
9044bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9045 using namespace llvm::AMDGPU::Swizzle;
9048 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9054 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9055 "FFT swizzle must be in the interval [0," +
9056 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9064bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9065 using namespace llvm::AMDGPU::Swizzle;
9068 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9075 if (!parseSwizzleOperand(
Direction, 0, 1,
9076 "direction must be 0 (left) or 1 (right)", Loc))
9080 if (!parseSwizzleOperand(
9081 RotateSize, 0, ROTATE_MAX_SIZE,
9082 "number of threads to rotate must be in the interval [0," +
9083 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9088 (RotateSize << ROTATE_SIZE_SHIFT);
9092bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9094 SMLoc OffsetLoc = getLoc();
9100 Error(OffsetLoc,
"expected a 16-bit offset");
9106bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9107 using namespace llvm::AMDGPU::Swizzle;
9111 SMLoc ModeLoc = getLoc();
9114 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9115 Ok = parseSwizzleQuadPerm(
Imm);
9116 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9117 Ok = parseSwizzleBitmaskPerm(
Imm);
9118 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9119 Ok = parseSwizzleBroadcast(
Imm);
9120 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9121 Ok = parseSwizzleSwap(
Imm);
9122 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9123 Ok = parseSwizzleReverse(
Imm);
9124 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9125 Ok = parseSwizzleFFT(
Imm);
9126 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9127 Ok = parseSwizzleRotate(
Imm);
9129 Error(ModeLoc,
"expected a swizzle mode");
9132 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9138ParseStatus AMDGPUAsmParser::parseSwizzle(
OperandVector &Operands) {
9142 if (trySkipId(
"offset")) {
9146 if (trySkipId(
"swizzle")) {
9147 Ok = parseSwizzleMacro(
Imm);
9149 Ok = parseSwizzleOffset(
Imm);
9154 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9161bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9167int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9169 using namespace llvm::AMDGPU::VGPRIndexMode;
9181 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9182 if (trySkipId(IdSymbolic[ModeId])) {
9190 ?
"expected a VGPR index mode or a closing parenthesis"
9191 :
"expected a VGPR index mode");
9196 Error(S,
"duplicate VGPR index mode");
9204 "expected a comma or a closing parenthesis"))
9211ParseStatus AMDGPUAsmParser::parseGPRIdxMode(
OperandVector &Operands) {
9213 using namespace llvm::AMDGPU::VGPRIndexMode;
9219 Imm = parseGPRIdxMacro();
9223 if (getParser().parseAbsoluteExpression(
Imm))
9226 return Error(S,
"invalid immediate: only 4-bit values are legal");
9230 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9234bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9240ParseStatus AMDGPUAsmParser::parseSOPPBrTarget(
OperandVector &Operands) {
9245 if (isRegister() || isModifier())
9251 AMDGPUOperand &Opr = ((AMDGPUOperand &)*Operands[Operands.
size() - 1]);
9252 assert(Opr.isImm() || Opr.isExpr());
9253 SMLoc Loc = Opr.getStartLoc();
9257 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9258 Error(Loc,
"expected an absolute expression or a label");
9259 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9260 Error(Loc,
"expected a 16-bit signed jump offset");
9270ParseStatus AMDGPUAsmParser::parseBoolReg(
OperandVector &Operands) {
9271 return parseReg(Operands);
9278void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
const OperandVector &Operands,
9280 OptionalImmIndexMap OptionalIdx;
9281 unsigned FirstOperandIdx = 1;
9282 bool IsAtomicReturn =
false;
9288 for (
unsigned i = FirstOperandIdx, e = Operands.
size(); i != e; ++i) {
9289 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
9293 Op.addRegOperands(Inst, 1);
9297 if (IsAtomicReturn && i == FirstOperandIdx)
9298 Op.addRegOperands(Inst, 1);
9303 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9304 Op.addImmOperands(Inst, 1);
9316 OptionalIdx[
Op.getImmTy()] = i;
9320 AMDGPUOperand::ImmTyOffset);
9332bool AMDGPUOperand::isSMRDOffset8()
const {
9336bool AMDGPUOperand::isSMEMOffset()
const {
9338 return isImmLiteral();
9341bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9376bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9377 if (BoundCtrl == 0 || BoundCtrl == 1) {
9385void AMDGPUAsmParser::onBeginOfFile() {
9386 if (!getParser().getStreamer().getTargetStreamer())
9389 if (!getTargetStreamer().getTargetID())
9390 getTargetStreamer().initializeTargetID(getSTI(),
9394void AMDGPUAsmParser::emitTargetDirective() {
9395 if (TargetDirectiveEmitted)
9397 TargetDirectiveEmitted =
true;
9399 if (!getParser().getStreamer().getTargetStreamer() ||
9404 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9413bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9417 StringRef TokenId = getTokenStr();
9418 AGVK VK = StringSwitch<AGVK>(TokenId)
9419 .Case(
"max", AGVK::AGVK_Max)
9420 .Case(
"min", AGVK::AGVK_Min)
9421 .Case(
"or", AGVK::AGVK_Or)
9422 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9423 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9424 .Case(
"alignto", AGVK::AGVK_AlignTo)
9425 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9426 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9427 .Default(AGVK::AGVK_None);
9431 uint64_t CommaCount = 0;
9436 if (Exprs.
empty()) {
9438 "empty " + Twine(TokenId) +
" expression");
9441 if (CommaCount + 1 != Exprs.
size()) {
9443 "mismatch of commas in " + Twine(TokenId) +
" expression");
9447 Expected && Exprs.
size() != Expected) {
9448 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9449 Twine(Expected) +
" operands");
9456 if (getParser().parseExpression(Expr, EndLoc))
9460 if (LastTokenWasComma)
9464 "unexpected token in " + Twine(TokenId) +
" expression");
9470 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9473ParseStatus AMDGPUAsmParser::parseOModSI(
OperandVector &Operands) {
9474 StringRef
Name = getTokenStr();
9475 if (Name ==
"mul") {
9476 return parseIntWithPrefix(
"mul", Operands, AMDGPUOperand::ImmTyOModSI,
9480 if (Name ==
"div") {
9481 return parseIntWithPrefix(
"div", Operands, AMDGPUOperand::ImmTyOModSI,
9492 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9497 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9498 AMDGPU::OpName::src2};
9506 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9511 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9513 if (
DstOp.isReg() &&
9518 if ((OpSel & (1 << SrcNum)) != 0)
9524void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9526 cvtVOP3P(Inst, Operands);
9530void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
const OperandVector &Operands,
9531 OptionalImmIndexMap &OptionalIdx) {
9532 cvtVOP3P(Inst, Operands, OptionalIdx);
9541 &&
Desc.NumOperands > (OpNum + 1)
9543 &&
Desc.operands()[OpNum + 1].RegClass != -1
9545 &&
Desc.getOperandConstraint(OpNum + 1,
9549void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9551 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9552 AMDGPU::OpName::src2};
9553 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9554 AMDGPU::OpName::src1_modifiers,
9555 AMDGPU::OpName::src2_modifiers};
9556 for (
int J = 0; J < 3; ++J) {
9557 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9563 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9566 if ((OpSel & (1 << J)) != 0)
9569 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9576void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9578 OptionalImmIndexMap OptionalIdx;
9583 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9584 ((AMDGPUOperand &)*Operands[
I++]).addRegOperands(Inst, 1);
9587 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
9588 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[
I]);
9590 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9591 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9592 Op.isInterpAttrChan()) {
9594 }
else if (
Op.isImmModifier()) {
9595 OptionalIdx[
Op.getImmTy()] =
I;
9603 AMDGPUOperand::ImmTyHigh);
9607 AMDGPUOperand::ImmTyClamp);
9611 AMDGPUOperand::ImmTyOModSI);
9616 AMDGPUOperand::ImmTyOpSel);
9617 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9620 cvtOpSelHelper(Inst, OpSel);
9624void AMDGPUAsmParser::cvtVINTERP(MCInst &Inst,
const OperandVector &Operands) {
9625 OptionalImmIndexMap OptionalIdx;
9630 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9631 ((AMDGPUOperand &)*Operands[
I++]).addRegOperands(Inst, 1);
9634 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
9635 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[
I]);
9637 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9638 }
else if (
Op.isImmModifier()) {
9639 OptionalIdx[
Op.getImmTy()] =
I;
9647 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9650 AMDGPUOperand::ImmTyOpSel);
9653 AMDGPUOperand::ImmTyWaitEXP);
9659 cvtOpSelHelper(Inst, OpSel);
9662void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9664 OptionalImmIndexMap OptionalIdx;
9667 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9671 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9672 static_cast<AMDGPUOperand &
>(*Operands[
I++]).addRegOperands(Inst, 1);
9674 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
9675 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*Operands[
I]);
9680 if (NumOperands == CbszOpIdx) {
9685 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9686 }
else if (
Op.isImmModifier()) {
9687 OptionalIdx[
Op.getImmTy()] =
I;
9689 Op.addRegOrImmOperands(Inst, 1);
9694 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9695 if (CbszIdx != OptionalIdx.end()) {
9696 int CbszVal = ((AMDGPUOperand &)*Operands[CbszIdx->second]).
getImm();
9700 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9701 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9702 if (BlgpIdx != OptionalIdx.end()) {
9703 int BlgpVal = ((AMDGPUOperand &)*Operands[BlgpIdx->second]).
getImm();
9714 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9715 if (OpselIdx != OptionalIdx.end()) {
9716 OpSel =
static_cast<const AMDGPUOperand &
>(*Operands[OpselIdx->second])
9720 unsigned OpSelHi = 0;
9721 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9722 if (OpselHiIdx != OptionalIdx.end()) {
9723 OpSelHi =
static_cast<const AMDGPUOperand &
>(*Operands[OpselHiIdx->second])
9726 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9727 AMDGPU::OpName::src1_modifiers};
9729 for (
unsigned J = 0; J < 2; ++J) {
9730 unsigned ModVal = 0;
9731 if (OpSel & (1 << J))
9733 if (OpSelHi & (1 << J))
9736 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9741void AMDGPUAsmParser::cvtVOP3(MCInst &Inst,
const OperandVector &Operands,
9742 OptionalImmIndexMap &OptionalIdx) {
9747 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9748 ((AMDGPUOperand &)*Operands[
I++]).addRegOperands(Inst, 1);
9751 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
9752 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[
I]);
9754 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9755 }
else if (
Op.isImmModifier()) {
9756 OptionalIdx[
Op.getImmTy()] =
I;
9758 Op.addRegOrImmOperands(Inst, 1);
9764 AMDGPUOperand::ImmTyScaleSel);
9768 AMDGPUOperand::ImmTyClamp);
9774 AMDGPUOperand::ImmTyByteSel);
9779 AMDGPUOperand::ImmTyOModSI);
9786 auto *it = Inst.
begin();
9788 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9796void AMDGPUAsmParser::cvtVOP3(MCInst &Inst,
const OperandVector &Operands) {
9797 OptionalImmIndexMap OptionalIdx;
9798 cvtVOP3(Inst, Operands, OptionalIdx);
9801void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
const OperandVector &Operands,
9802 OptionalImmIndexMap &OptIdx) {
9807 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9808 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9809 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9810 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9811 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9812 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9813 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9814 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9815 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9816 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9825 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9826 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9829 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9830 if (BitOp3Idx != -1) {
9837 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9838 if (OpSelIdx != -1) {
9842 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9843 if (OpSelHiIdx != -1) {
9844 int DefaultVal =
IsPacked ? -1 : 0;
9850 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9851 if (MatrixAFMTIdx != -1) {
9853 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9857 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9858 if (MatrixBFMTIdx != -1) {
9860 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9863 int MatrixAScaleIdx =
9864 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9865 if (MatrixAScaleIdx != -1) {
9867 AMDGPUOperand::ImmTyMatrixAScale, 0);
9870 int MatrixBScaleIdx =
9871 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9872 if (MatrixBScaleIdx != -1) {
9874 AMDGPUOperand::ImmTyMatrixBScale, 0);
9877 int MatrixAScaleFmtIdx =
9878 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9879 if (MatrixAScaleFmtIdx != -1) {
9881 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9884 int MatrixBScaleFmtIdx =
9885 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9886 if (MatrixBScaleFmtIdx != -1) {
9888 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9893 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9897 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9899 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9903 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9907 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9908 AMDGPU::OpName::src2};
9909 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9910 AMDGPU::OpName::src1_modifiers,
9911 AMDGPU::OpName::src2_modifiers};
9914 unsigned OpSelHi = 0;
9921 if (OpSelHiIdx != -1)
9930 for (
int J = 0; J < 3; ++J) {
9931 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9935 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9945 uint32_t ModVal = 0;
9948 if (SrcOp.
isReg() && getMRI()
9955 if ((OpSel & (1 << J)) != 0)
9959 if ((OpSelHi & (1 << J)) != 0)
9962 if ((NegLo & (1 << J)) != 0)
9965 if ((NegHi & (1 << J)) != 0)
9972void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
const OperandVector &Operands) {
9973 OptionalImmIndexMap OptIdx;
9974 cvtVOP3(Inst, Operands, OptIdx);
9975 cvtVOP3P(Inst, Operands, OptIdx);
9979 unsigned i,
unsigned Opc,
9981 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
9982 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
9984 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
9987void AMDGPUAsmParser::cvtSWMMAC(MCInst &Inst,
const OperandVector &Operands) {
9990 ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, 1);
9993 ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, 1);
9994 ((AMDGPUOperand &)*Operands[4]).addRegOperands(Inst, 1);
9996 OptionalImmIndexMap OptIdx;
9997 for (
unsigned i = 5; i < Operands.
size(); ++i) {
9998 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[i]);
9999 OptIdx[
Op.getImmTy()] = i;
10004 AMDGPUOperand::ImmTyIndexKey8bit);
10008 AMDGPUOperand::ImmTyIndexKey16bit);
10012 AMDGPUOperand::ImmTyIndexKey32bit);
10017 cvtVOP3P(Inst, Operands, OptIdx);
10024ParseStatus AMDGPUAsmParser::parseVOPD(
OperandVector &Operands) {
10029 SMLoc S = getLoc();
10032 Operands.
push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10033 SMLoc OpYLoc = getLoc();
10036 Operands.
push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10039 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10045void AMDGPUAsmParser::cvtVOPD(MCInst &Inst,
const OperandVector &Operands) {
10048 auto addOp = [&](uint16_t ParsedOprIdx) {
10049 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[ParsedOprIdx]);
10051 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10055 Op.addRegOperands(Inst, 1);
10059 Op.addImmOperands(Inst, 1);
10071 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10075 const auto &CInfo = InstInfo[CompIdx];
10076 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10077 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10078 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10079 if (CInfo.hasSrc2Acc())
10080 addOp(CInfo.getIndexOfDstInParsedOperands());
10084 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10085 if (BitOp3Idx != -1) {
10086 OptionalImmIndexMap OptIdx;
10087 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands.
back());
10089 OptIdx[
Op.getImmTy()] = Operands.
size() - 1;
10099bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10101bool AMDGPUOperand::isDPPCtrl()
const {
10102 using namespace AMDGPU::DPP;
10104 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10107 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10108 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10109 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10110 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10111 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10112 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10113 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10114 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10115 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10116 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10117 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10118 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10127bool AMDGPUOperand::isBLGP()
const {
10131bool AMDGPUOperand::isS16Imm()
const {
10135bool AMDGPUOperand::isU16Imm()
const {
10143bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10148 SMLoc Loc =
getToken().getEndLoc();
10149 Token = std::string(getTokenStr());
10151 if (getLoc() != Loc)
10156 if (!parseId(Suffix))
10160 StringRef DimId = Token;
10171ParseStatus AMDGPUAsmParser::parseDim(
OperandVector &Operands) {
10175 SMLoc S = getLoc();
10181 SMLoc Loc = getLoc();
10182 if (!parseDimId(Encoding))
10183 return Error(Loc,
"invalid dim value");
10186 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10194ParseStatus AMDGPUAsmParser::parseDPP8(
OperandVector &Operands) {
10195 SMLoc S = getLoc();
10204 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10207 for (
size_t i = 0; i < 8; ++i) {
10211 SMLoc Loc = getLoc();
10212 if (getParser().parseAbsoluteExpression(Sels[i]))
10214 if (0 > Sels[i] || 7 < Sels[i])
10215 return Error(Loc,
"expected a 3-bit value");
10218 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10222 for (
size_t i = 0; i < 8; ++i)
10223 DPP8 |= (Sels[i] << (i * 3));
10226 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10230bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10232 if (Ctrl ==
"row_newbcast")
10235 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10238 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10239 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10242 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10243 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10247int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10250 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10254 for (
int i = 0; i < 4; ++i) {
10259 SMLoc Loc = getLoc();
10260 if (getParser().parseAbsoluteExpression(Temp))
10262 if (Temp < 0 || Temp > 3) {
10263 Error(Loc,
"expected a 2-bit value");
10267 Val += (Temp << i * 2);
10270 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10276int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10277 using namespace AMDGPU::DPP;
10282 SMLoc Loc = getLoc();
10284 if (getParser().parseAbsoluteExpression(Val))
10287 struct DppCtrlCheck {
10293 DppCtrlCheck
Check =
10294 StringSwitch<DppCtrlCheck>(Ctrl)
10295 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10296 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10297 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10298 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10299 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10300 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10301 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10302 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10303 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10304 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10308 if (
Check.Ctrl == -1) {
10309 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10317 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10324ParseStatus AMDGPUAsmParser::parseDPPCtrl(
OperandVector &Operands) {
10325 using namespace AMDGPU::DPP;
10328 !isSupportedDPPCtrl(getTokenStr(), Operands))
10331 SMLoc S = getLoc();
10337 if (Ctrl ==
"row_mirror") {
10338 Val = DppCtrl::ROW_MIRROR;
10339 }
else if (Ctrl ==
"row_half_mirror") {
10340 Val = DppCtrl::ROW_HALF_MIRROR;
10343 if (Ctrl ==
"quad_perm") {
10344 Val = parseDPPCtrlPerm();
10346 Val = parseDPPCtrlSel(Ctrl);
10355 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10359void AMDGPUAsmParser::cvtVOP3DPP(MCInst &Inst,
const OperandVector &Operands,
10361 OptionalImmIndexMap OptionalIdx;
10368 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10370 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10371 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10375 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10376 ((AMDGPUOperand &)*Operands[
I++]).addRegOperands(Inst, 1);
10380 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10381 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10382 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10383 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10384 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10385 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10386 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10387 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10388 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10390 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
10394 if (OldIdx == NumOperands) {
10396 constexpr int DST_IDX = 0;
10398 }
else if (Src2ModIdx == NumOperands) {
10408 if (IsVOP3CvtSrDpp) {
10417 if (TiedTo != -1) {
10422 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[
I]);
10424 if (IsDPP8 &&
Op.isDppFI()) {
10427 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10428 }
else if (
Op.isReg()) {
10429 Op.addRegOperands(Inst, 1);
10430 }
else if (
Op.isImm() &&
10432 Op.addImmOperands(Inst, 1);
10433 }
else if (
Op.isImm()) {
10434 OptionalIdx[
Op.getImmTy()] =
I;
10442 AMDGPUOperand::ImmTyClamp);
10448 AMDGPUOperand::ImmTyByteSel);
10453 AMDGPUOperand::ImmTyOModSI);
10456 cvtVOP3P(Inst, Operands, OptionalIdx);
10458 cvtVOP3OpSel(Inst, Operands, OptionalIdx);
10461 AMDGPUOperand::ImmTyOpSel);
10466 AMDGPUOperand::ImmTyDPP8);
10467 using namespace llvm::AMDGPU::DPP;
10471 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10473 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10475 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10477 AMDGPUOperand::ImmTyDppBoundCtrl);
10481 AMDGPUOperand::ImmTyDppFI);
10485void AMDGPUAsmParser::cvtDPP(MCInst &Inst,
const OperandVector &Operands,
10487 OptionalImmIndexMap OptionalIdx;
10491 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10492 ((AMDGPUOperand &)*Operands[
I++]).addRegOperands(Inst, 1);
10496 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
10499 if (TiedTo != -1) {
10504 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[
I]);
10506 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10514 Op.addImmOperands(Inst, 1);
10516 Op.addRegWithFPInputModsOperands(Inst, 2);
10517 }
else if (
Op.isDppFI()) {
10519 }
else if (
Op.isReg()) {
10520 Op.addRegOperands(Inst, 1);
10526 Op.addRegWithFPInputModsOperands(Inst, 2);
10527 }
else if (
Op.isReg()) {
10528 Op.addRegOperands(Inst, 1);
10529 }
else if (
Op.isDPPCtrl()) {
10530 Op.addImmOperands(Inst, 1);
10531 }
else if (
Op.isImm()) {
10533 OptionalIdx[
Op.getImmTy()] =
I;
10541 using namespace llvm::AMDGPU::DPP;
10545 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10547 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10549 AMDGPUOperand::ImmTyDppBoundCtrl);
10552 AMDGPUOperand::ImmTyDppFI);
10561ParseStatus AMDGPUAsmParser::parseSDWASel(
OperandVector &Operands,
10563 AMDGPUOperand::ImmTy
Type) {
10564 return parseStringOrIntWithPrefix(
10566 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10570ParseStatus AMDGPUAsmParser::parseSDWADstUnused(
OperandVector &Operands) {
10571 return parseStringOrIntWithPrefix(
10572 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10573 AMDGPUOperand::ImmTySDWADstUnused);
10576void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst,
const OperandVector &Operands) {
10577 cvtSDWA(Inst, Operands, SDWAInstType::VOP1);
10580void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst,
const OperandVector &Operands) {
10581 cvtSDWA(Inst, Operands, SDWAInstType::VOP2);
10584void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10586 cvtSDWA(Inst, Operands, SDWAInstType::VOP2,
true,
true);
10589void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10591 cvtSDWA(Inst, Operands, SDWAInstType::VOP2,
false,
true);
10594void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst,
const OperandVector &Operands) {
10595 cvtSDWA(Inst, Operands, SDWAInstType::VOPC,
isVI());
10598void AMDGPUAsmParser::cvtSDWA(MCInst &Inst,
const OperandVector &Operands,
10599 SDWAInstType BasicInstType,
bool SkipDstVcc,
10601 using namespace llvm::AMDGPU::SDWA;
10603 OptionalImmIndexMap OptionalIdx;
10604 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10605 bool SkippedVcc =
false;
10609 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10610 ((AMDGPUOperand &)*Operands[
I++]).addRegOperands(Inst, 1);
10613 for (
unsigned E = Operands.
size();
I !=
E; ++
I) {
10614 AMDGPUOperand &
Op = ((AMDGPUOperand &)*Operands[
I]);
10615 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10616 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10622 if (BasicInstType == SDWAInstType::VOP2 &&
10628 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10634 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10635 }
else if (
Op.isImm()) {
10637 OptionalIdx[
Op.getImmTy()] =
I;
10641 SkippedVcc =
false;
10645 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10646 Opc != AMDGPU::V_NOP_sdwa_vi) {
10648 switch (BasicInstType) {
10649 case SDWAInstType::VOP1:
10652 AMDGPUOperand::ImmTyClamp, 0);
10656 AMDGPUOperand::ImmTyOModSI, 0);
10660 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10664 AMDGPUOperand::ImmTySDWADstUnused,
10665 DstUnused::UNUSED_PRESERVE);
10668 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10671 case SDWAInstType::VOP2:
10673 AMDGPUOperand::ImmTyClamp, 0);
10677 AMDGPUOperand::ImmTyOModSI, 0);
10680 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10682 AMDGPUOperand::ImmTySDWADstUnused,
10683 DstUnused::UNUSED_PRESERVE);
10685 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10687 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10690 case SDWAInstType::VOPC:
10693 AMDGPUOperand::ImmTyClamp, 0);
10695 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10697 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10704 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10705 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10706 auto *it = Inst.
begin();
10708 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10721#define GET_MATCHER_IMPLEMENTATION
10722#define GET_MNEMONIC_SPELL_CHECKER
10723#define GET_MNEMONIC_CHECKER
10724#include "AMDGPUGenAsmMatcher.inc"
10730 return parseTokenOp(
"addr64",
Operands);
10732 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10734 return parseTokenOp(
"idxen",
Operands);
10736 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10739 return parseTokenOp(
"offen",
Operands);
10741 return parseTokenOp(
"off",
Operands);
10742 case MCK_row_95_en:
10743 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10745 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10747 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10749 return tryCustomParseOperand(
Operands, MCK);
10754unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10760 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10763 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10765 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10767 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10769 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10771 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10773 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10775 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10776 case MCK_row_95_en:
10777 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10785 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10787 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10788 case MCK_SOPPBrTarget:
10789 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10790 case MCK_VReg32OrOff:
10791 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10792 case MCK_InterpSlot:
10793 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10794 case MCK_InterpAttr:
10795 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10796 case MCK_InterpAttrChan:
10797 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10799 case MCK_SReg_64_XEXEC:
10809 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10811 return Match_InvalidOperand;
10819ParseStatus AMDGPUAsmParser::parseEndpgm(
OperandVector &Operands) {
10820 SMLoc S = getLoc();
10829 return Error(S,
"expected a 16-bit value");
10832 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10836bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10842bool AMDGPUOperand::isSplitBarrier()
const {
return isInlinableImm(MVT::i32); }
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
MachineInstr unsigned OpIdx
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
StringSet - A set-like wrapper for the StringMap.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
size_t size() const
Get the array size.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
bool regsOverlap(MCRegister RegA, MCRegister RegB) const
Returns true if the two registers are equal or alias each other.
const MCRegisterClass & getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
static constexpr CustomOperand Operands[]
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isPackedFP32Inst(unsigned Opc)
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi)
bool isGFX10_BEncoding(const MCSubtargetInfo &STI)
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
GPUKind
GPU kinds supported by the AMDGPU target.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR, int32_t ArgNumVGPR)
bool isGFX10(const MCSubtargetInfo &STI)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isGFX13(const MCSubtargetInfo &STI)
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, OpName NamedIdx)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool isGFX13Plus(const MCSubtargetInfo &STI)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isGFX1250(const MCSubtargetInfo &STI)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
@ Default
The result value is uniform if and only if all operands are uniform.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size
1.14.0