70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
248 return isRegOrInline(RCID, type) || isLiteralImm(type);
251 bool isRegOrImmWithInt16InputMods()
const {
255 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
257 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
260 bool isRegOrImmWithInt32InputMods()
const {
264 bool isRegOrInlineImmWithInt16InputMods()
const {
265 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFP32InputMods()
const {
294 bool isRegOrImmWithFP64InputMods()
const {
298 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
299 return isRegOrInline(
300 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
303 bool isRegOrInlineImmWithFP32InputMods()
const {
304 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
307 bool isRegOrInlineImmWithFP64InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
311 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
313 bool isVRegWithFP32InputMods()
const {
314 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
317 bool isVRegWithFP64InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
321 bool isPackedFP16InputMods()
const {
325 bool isPackedVGPRFP32InputMods()
const {
329 bool isVReg()
const {
330 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
331 isRegClass(AMDGPU::VReg_64RegClassID) ||
332 isRegClass(AMDGPU::VReg_96RegClassID) ||
333 isRegClass(AMDGPU::VReg_128RegClassID) ||
334 isRegClass(AMDGPU::VReg_160RegClassID) ||
335 isRegClass(AMDGPU::VReg_192RegClassID) ||
336 isRegClass(AMDGPU::VReg_256RegClassID) ||
337 isRegClass(AMDGPU::VReg_512RegClassID) ||
338 isRegClass(AMDGPU::VReg_1024RegClassID);
341 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
343 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
345 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
347 bool isAV_LdSt_32_Align2_RegOp()
const {
348 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
349 isRegClass(AMDGPU::AGPR_32RegClassID);
352 bool isVRegWithInputMods()
const;
353 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
354 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
356 bool isSDWAOperand(MVT type)
const;
357 bool isSDWAFP16Operand()
const;
358 bool isSDWAFP32Operand()
const;
359 bool isSDWAInt16Operand()
const;
360 bool isSDWAInt32Operand()
const;
362 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
364 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
366 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
368 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
370 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
371 bool isDim()
const {
return isImmTy(ImmTyDim); }
372 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
373 bool isOff()
const {
return isImmTy(ImmTyOff); }
374 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
375 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
376 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
377 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
378 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
379 bool isFlatOffset()
const {
380 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
382 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
383 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
384 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
385 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
386 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
387 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
388 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
389 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
390 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
391 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
392 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
393 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
394 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
395 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
396 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
397 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
398 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
399 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
400 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
401 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
402 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
403 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
404 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
405 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
406 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
407 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
408 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
409 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
410 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
411 bool isDone()
const {
return isImmTy(ImmTyDone); }
412 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
414 bool isRegOrImm()
const {
return isReg() || isImm(); }
416 bool isRegClass(
unsigned RCID)
const;
420 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
421 return isRegOrInline(RCID, type) && !hasModifiers();
424 bool isSCSrcB16()
const {
425 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
428 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
430 bool isSCSrc_b32()
const {
431 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
434 bool isSCSrc_b64()
const {
435 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
438 bool isBoolReg()
const;
440 bool isSCSrcF16()
const {
441 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
444 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
446 bool isSCSrcF32()
const {
447 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
450 bool isSCSrcF64()
const {
451 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
454 bool isSSrc_b32()
const {
455 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
458 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
460 bool isSSrcV2B16()
const {
465 bool isSSrc_b64()
const {
468 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
469 (((
const MCTargetAsmParser *)AsmParser)
470 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
474 bool isSSrc_f32()
const {
475 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
478 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
480 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
482 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
484 bool isSSrcV2F16()
const {
489 bool isSSrcV2FP32()
const {
494 bool isSCSrcV2FP32()
const {
499 bool isSSrcV2INT32()
const {
504 bool isSCSrcV2INT32()
const {
506 return isSCSrc_b32();
509 bool isSSrcOrLds_b32()
const {
510 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
511 isLiteralImm(MVT::i32) || isExpr();
514 bool isVCSrc_b32()
const {
515 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
518 bool isVCSrc_b32_Lo256()
const {
519 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
522 bool isVCSrc_b64_Lo256()
const {
523 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
526 bool isVCSrc_b64()
const {
527 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
530 bool isVCSrcT_b16()
const {
531 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
534 bool isVCSrcTB16_Lo128()
const {
535 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
538 bool isVCSrcFake16B16_Lo128()
const {
539 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
542 bool isVCSrc_b16()
const {
543 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
546 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
548 bool isVCSrc_f32()
const {
549 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
552 bool isVCSrc_f64()
const {
553 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
556 bool isVCSrcTBF16()
const {
557 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
560 bool isVCSrcT_f16()
const {
561 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
564 bool isVCSrcT_bf16()
const {
565 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
568 bool isVCSrcTBF16_Lo128()
const {
569 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
572 bool isVCSrcTF16_Lo128()
const {
573 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
576 bool isVCSrcFake16BF16_Lo128()
const {
577 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
580 bool isVCSrcFake16F16_Lo128()
const {
581 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
584 bool isVCSrc_bf16()
const {
585 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
588 bool isVCSrc_f16()
const {
589 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
592 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
594 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
596 bool isVSrc_b32()
const {
597 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
600 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
602 bool isVSrc_v2b64()
const {
603 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
604 isLiteralImm(MVT::i64);
607 bool isVSrc_v2f64()
const {
608 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
609 isLiteralImm(MVT::f64);
612 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
614 bool isVSrcT_b16_Lo128()
const {
615 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
618 bool isVSrcFake16_b16_Lo128()
const {
619 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
622 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
624 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
626 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
628 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
630 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
632 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
634 bool isVSrc_f32()
const {
635 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
638 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
640 bool isVSrcT_bf16()
const {
641 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
644 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
646 bool isVSrcT_bf16_Lo128()
const {
647 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
650 bool isVSrcT_f16_Lo128()
const {
651 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
654 bool isVSrcFake16_bf16_Lo128()
const {
655 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
658 bool isVSrcFake16_f16_Lo128()
const {
659 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
662 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
664 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
666 bool isVSrc_v2bf16()
const {
667 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
670 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
672 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
674 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
676 bool isVISrcB32()
const {
677 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
680 bool isVISrcB16()
const {
681 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
684 bool isVISrcV2B16()
const {
return isVISrcB16(); }
686 bool isVISrcF32()
const {
687 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
690 bool isVISrcF16()
const {
691 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
694 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
696 bool isVISrc_64_bf16()
const {
697 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
700 bool isVISrc_64_f16()
const {
701 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
704 bool isVISrc_64_b32()
const {
705 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
708 bool isVISrc_64B64()
const {
709 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
712 bool isVISrc_64_f64()
const {
713 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
716 bool isVISrc_64V2FP32()
const {
717 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
720 bool isVISrc_64V2INT32()
const {
721 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
724 bool isVISrc_256_b32()
const {
725 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
728 bool isVISrc_256_f32()
const {
729 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
732 bool isVISrc_256B64()
const {
733 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
736 bool isVISrc_256_f64()
const {
737 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
740 bool isVISrc_512_f64()
const {
741 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
744 bool isVISrc_128B16()
const {
745 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
748 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
750 bool isVISrc_128_b32()
const {
751 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
754 bool isVISrc_128_f32()
const {
755 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
758 bool isVISrc_256V2FP32()
const {
759 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
762 bool isVISrc_256V2INT32()
const {
763 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
766 bool isVISrc_512_b32()
const {
767 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
770 bool isVISrc_512B16()
const {
771 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
774 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
776 bool isVISrc_512_f32()
const {
777 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
780 bool isVISrc_512F16()
const {
781 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
784 bool isVISrc_512V2F16()
const {
785 return isVISrc_512F16() || isVISrc_512_b32();
788 bool isVISrc_1024_b32()
const {
789 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
792 bool isVISrc_1024B16()
const {
793 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
796 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
798 bool isVISrc_1024_f32()
const {
799 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
802 bool isVISrc_1024F16()
const {
803 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
806 bool isVISrc_1024V2F16()
const {
807 return isVISrc_1024F16() || isVISrc_1024_b32();
810 bool isAISrcB32()
const {
811 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
814 bool isAISrcB16()
const {
815 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
818 bool isAISrcV2B16()
const {
return isAISrcB16(); }
820 bool isAISrcF32()
const {
821 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
824 bool isAISrcF16()
const {
825 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
828 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
830 bool isAISrc_64B64()
const {
831 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
834 bool isAISrc_64_f64()
const {
835 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
838 bool isAISrc_128_b32()
const {
839 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
842 bool isAISrc_128B16()
const {
843 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
846 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
848 bool isAISrc_128_f32()
const {
849 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
852 bool isAISrc_128F16()
const {
853 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
856 bool isAISrc_128V2F16()
const {
857 return isAISrc_128F16() || isAISrc_128_b32();
860 bool isVISrc_128_bf16()
const {
861 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
864 bool isVISrc_128_f16()
const {
865 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
868 bool isVISrc_128V2F16()
const {
869 return isVISrc_128_f16() || isVISrc_128_b32();
872 bool isAISrc_256B64()
const {
873 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
876 bool isAISrc_256_f64()
const {
877 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
880 bool isAISrc_512_b32()
const {
881 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
884 bool isAISrc_512B16()
const {
885 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
888 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
890 bool isAISrc_512_f32()
const {
891 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
894 bool isAISrc_512F16()
const {
895 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
898 bool isAISrc_512V2F16()
const {
899 return isAISrc_512F16() || isAISrc_512_b32();
902 bool isAISrc_1024_b32()
const {
903 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
906 bool isAISrc_1024B16()
const {
907 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
910 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
912 bool isAISrc_1024_f32()
const {
913 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
916 bool isAISrc_1024F16()
const {
917 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
920 bool isAISrc_1024V2F16()
const {
921 return isAISrc_1024F16() || isAISrc_1024_b32();
924 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
926 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
928 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
930 bool isMem()
const override {
return false; }
932 bool isExpr()
const {
return Kind == Expression; }
934 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
936 bool isSWaitCnt()
const;
937 bool isDepCtr()
const;
938 bool isSDelayALU()
const;
939 bool isHwreg()
const;
940 bool isSendMsg()
const;
941 bool isWaitEvent()
const;
942 bool isSplitBarrier()
const;
943 bool isSwizzle()
const;
944 bool isSMRDOffset8()
const;
945 bool isSMEMOffset()
const;
946 bool isSMRDLiteralOffset()
const;
948 bool isDPPCtrl()
const;
950 bool isGPRIdxMode()
const;
951 bool isS16Imm()
const;
952 bool isU16Imm()
const;
953 bool isEndpgm()
const;
955 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
956 return [
this,
P]() {
return P(*
this); };
961 return StringRef(Tok.Data, Tok.Length);
969 void setImm(int64_t Val) {
974 ImmTy getImmTy()
const {
979 MCRegister
getReg()
const override {
984 SMLoc getStartLoc()
const override {
return StartLoc; }
986 SMLoc getEndLoc()
const override {
return EndLoc; }
988 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
990 int getMCOpIdx()
const {
return MCOpIdx; }
992 Modifiers getModifiers()
const {
993 assert(isRegKind() || isImmTy(ImmTyNone));
994 return isRegKind() ?
Reg.Mods :
Imm.Mods;
997 void setModifiers(Modifiers Mods) {
998 assert(isRegKind() || isImmTy(ImmTyNone));
1005 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1007 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1009 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1011 bool isForcedLit()
const {
1012 return isImmLiteral() && getModifiers().isForcedLit();
1015 bool isForcedLit64()
const {
1016 return isImmLiteral() && getModifiers().isForcedLit64();
1021 void addImmOperands(MCInst &Inst,
unsigned N,
1022 bool ApplyModifiers =
true)
const;
1024 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1025 bool ApplyModifiers)
const;
1027 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1029 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1031 addRegOperands(Inst,
N);
1033 addImmOperands(Inst,
N);
1036 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1037 Modifiers Mods = getModifiers();
1040 addRegOperands(Inst,
N);
1042 addImmOperands(Inst,
N,
false);
1046 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1047 assert(!hasIntModifiers());
1048 addRegOrImmWithInputModsOperands(Inst,
N);
1051 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1052 assert(!hasFPModifiers());
1053 addRegOrImmWithInputModsOperands(Inst,
N);
1056 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1057 Modifiers Mods = getModifiers();
1060 addRegOperands(Inst,
N);
1063 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1064 assert(!hasIntModifiers());
1065 addRegWithInputModsOperands(Inst,
N);
1068 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1069 assert(!hasFPModifiers());
1070 addRegWithInputModsOperands(Inst,
N);
1073 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1076 case ImmTyNone: OS <<
"None";
break;
1077 case ImmTyGDS: OS <<
"GDS";
break;
1078 case ImmTyLDS: OS <<
"LDS";
break;
1079 case ImmTyOffen: OS <<
"Offen";
break;
1080 case ImmTyIdxen: OS <<
"Idxen";
break;
1081 case ImmTyAddr64: OS <<
"Addr64";
break;
1082 case ImmTyOffset: OS <<
"Offset";
break;
1083 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1084 case ImmTyOffset0: OS <<
"Offset0";
break;
1085 case ImmTyOffset1: OS <<
"Offset1";
break;
1086 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1087 case ImmTyCPol: OS <<
"CPol";
break;
1088 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1089 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1090 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1091 case ImmTyTFE: OS <<
"TFE";
break;
1092 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1093 case ImmTyD16: OS <<
"D16";
break;
1094 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1095 case ImmTyClamp: OS <<
"Clamp";
break;
1096 case ImmTyOModSI: OS <<
"OModSI";
break;
1097 case ImmTyDPP8: OS <<
"DPP8";
break;
1098 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1099 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1100 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1101 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1102 case ImmTyDppFI: OS <<
"DppFI";
break;
1103 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1104 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1105 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1106 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1107 case ImmTyDMask: OS <<
"DMask";
break;
1108 case ImmTyDim: OS <<
"Dim";
break;
1109 case ImmTyUNorm: OS <<
"UNorm";
break;
1110 case ImmTyDA: OS <<
"DA";
break;
1111 case ImmTyR128A16: OS <<
"R128A16";
break;
1112 case ImmTyA16: OS <<
"A16";
break;
1113 case ImmTyLWE: OS <<
"LWE";
break;
1114 case ImmTyOff: OS <<
"Off";
break;
1115 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1116 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1117 case ImmTyExpVM: OS <<
"ExpVM";
break;
1118 case ImmTyDone: OS <<
"Done";
break;
1119 case ImmTyRowEn: OS <<
"RowEn";
break;
1120 case ImmTyHwreg: OS <<
"Hwreg";
break;
1121 case ImmTySendMsg: OS <<
"SendMsg";
break;
1122 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1123 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1124 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1125 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1126 case ImmTyOpSel: OS <<
"OpSel";
break;
1127 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1128 case ImmTyNegLo: OS <<
"NegLo";
break;
1129 case ImmTyNegHi: OS <<
"NegHi";
break;
1130 case ImmTySwizzle: OS <<
"Swizzle";
break;
1131 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1132 case ImmTyHigh: OS <<
"High";
break;
1133 case ImmTyBLGP: OS <<
"BLGP";
break;
1134 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1135 case ImmTyABID: OS <<
"ABID";
break;
1136 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1137 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1138 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1139 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1140 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1141 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1142 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1143 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1144 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1145 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1146 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1147 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1148 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1149 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1150 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1151 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1156 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1160 <<
" mods: " <<
Reg.Mods <<
'>';
1164 if (getImmTy() != ImmTyNone) {
1166 printImmTy(OS, getImmTy());
1168 OS <<
" mods: " <<
Imm.Mods <<
'>';
1181 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1182 int64_t Val, SMLoc Loc,
1183 ImmTy
Type = ImmTyNone,
1184 bool IsFPImm =
false) {
1185 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1187 Op->Imm.IsFPImm = IsFPImm;
1189 Op->Imm.Mods = Modifiers();
1195 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1196 StringRef Str, SMLoc Loc,
1197 bool HasExplicitEncodingSize =
true) {
1198 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1199 Res->Tok.Data = Str.data();
1200 Res->Tok.Length = Str.size();
1201 Res->StartLoc = Loc;
1206 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1207 MCRegister
Reg, SMLoc S, SMLoc
E) {
1208 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1209 Op->Reg.RegNo =
Reg;
1210 Op->Reg.Mods = Modifiers();
1216 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1217 const class MCExpr *Expr, SMLoc S) {
1218 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1227 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1236#define GET_REGISTER_MATCHER
1237#include "AMDGPUGenAsmMatcher.inc"
1238#undef GET_REGISTER_MATCHER
1239#undef GET_SUBTARGET_FEATURE_NAME
1244class KernelScopeInfo {
1245 int SgprIndexUnusedMin = -1;
1246 int VgprIndexUnusedMin = -1;
1247 int AgprIndexUnusedMin = -1;
1251 void usesSgprAt(
int i) {
1252 if (i >= SgprIndexUnusedMin) {
1253 SgprIndexUnusedMin = ++i;
1256 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1262 void usesVgprAt(
int i) {
1263 if (i >= VgprIndexUnusedMin) {
1264 VgprIndexUnusedMin = ++i;
1267 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1269 VgprIndexUnusedMin);
1275 void usesAgprAt(
int i) {
1280 if (i >= AgprIndexUnusedMin) {
1281 AgprIndexUnusedMin = ++i;
1284 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1289 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1291 VgprIndexUnusedMin);
1298 KernelScopeInfo() =
default;
1302 MSTI = Ctx->getSubtargetInfo();
1304 usesSgprAt(SgprIndexUnusedMin = -1);
1305 usesVgprAt(VgprIndexUnusedMin = -1);
1307 usesAgprAt(AgprIndexUnusedMin = -1);
1311 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1312 unsigned RegWidth) {
1315 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1318 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1321 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1330 MCAsmParser &Parser;
1332 unsigned ForcedEncodingSize = 0;
1333 bool ForcedDPP =
false;
1334 bool ForcedSDWA =
false;
1335 KernelScopeInfo KernelScope;
1336 const unsigned HwMode;
1338 const AMDGPU::IsaVersion ISA;
1343#define GET_ASSEMBLER_HEADER
1344#include "AMDGPUGenAsmMatcher.inc"
1349 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1351 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1355 std::optional<AMDGPU::InfoSectionData> InfoData;
1362 bool TargetDirectiveEmitted =
false;
1371 SmallVector<unsigned> OpcodeStream;
1373 OpcodeStreamSymbols;
1374 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1377 void checkKernelPrologues();
1380 void createConstantSymbol(StringRef Id, int64_t Val);
1382 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1383 bool OutOfRangeError(SMRange
Range);
1399 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1400 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1401 std::optional<bool> EnableWavefrontSize32,
1402 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1403 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1404 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1405 bool ParseDirectiveAMDGCNTarget();
1406 bool ParseDirectiveAMDHSACodeObjectVersion();
1407 bool ParseDirectiveAMDHSAKernel();
1408 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1409 bool ParseDirectiveAMDKernelCodeT();
1411 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1412 bool ParseDirectiveAMDGPUHsaKernel();
1414 bool ParseDirectiveISAVersion();
1415 bool ParseDirectiveHSAMetadata();
1416 bool ParseDirectivePALMetadataBegin();
1417 bool ParseDirectivePALMetadata();
1418 bool ParseDirectiveAMDGPULDS();
1419 bool ParseDirectiveAMDGPUInfo();
1423 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1424 const char *AssemblerDirectiveEnd,
1425 std::string &CollectString);
1427 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1428 RegisterKind RegKind, MCRegister Reg1,
1429 RegisterKind RegKind1, SMLoc Loc);
1430 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1431 unsigned &RegNum,
unsigned &RegWidth,
1432 bool RestoreOnFailure =
false);
1433 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1434 unsigned &RegNum,
unsigned &RegWidth,
1435 SmallVectorImpl<AsmToken> &Tokens);
1436 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1438 SmallVectorImpl<AsmToken> &Tokens);
1439 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1441 SmallVectorImpl<AsmToken> &Tokens);
1442 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1444 SmallVectorImpl<AsmToken> &Tokens);
1445 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1446 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1447 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1450 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1451 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1452 void initializeGprCountSymbol(RegisterKind RegKind);
1453 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1459 OperandMode_Default,
1463 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1465 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1466 const MCInstrInfo &MII)
1467 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1468 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1473 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1475 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1476 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1477 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1478 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1480 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1481 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1482 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1484 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1485 initializeGprCountSymbol(IS_VGPR);
1486 initializeGprCountSymbol(IS_SGPR);
1491 createConstantSymbol(Symbol, Code);
1493 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1494 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1495 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1543 bool hasBVHRayTracingInsts()
const {
1544 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1549 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1551 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1553 bool hasInv2PiInlineImm()
const {
1554 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1557 bool has64BitLiterals()
const {
1558 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1561 bool hasFlatOffsets()
const {
1562 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1565 bool hasTrue16Insts()
const {
1566 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1570 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1573 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1575 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1577 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1579 bool hasPartialNSAEncoding()
const {
1580 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1583 bool hasGloballyAddressableScratch()
const {
1584 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1597 AMDGPUTargetStreamer &getTargetStreamer() {
1598 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1599 return static_cast<AMDGPUTargetStreamer &
>(TS);
1605 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1608 const MCRegisterInfo *getMRI()
const {
1612 const MCInstrInfo *getMII()
const {
return &MII; }
1616 const FeatureBitset &getFeatureBits()
const {
1617 return getSTI().getFeatureBits();
1620 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1621 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1622 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1624 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1625 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1626 bool isForcedDPP()
const {
return ForcedDPP; }
1627 bool isForcedSDWA()
const {
return ForcedSDWA; }
1628 ArrayRef<unsigned> getMatchedVariants()
const;
1629 StringRef getMatchedVariantName()
const;
1631 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1632 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1633 bool RestoreOnFailure);
1634 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1635 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1636 SMLoc &EndLoc)
override;
1637 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1638 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1639 unsigned Kind)
override;
1640 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1643 bool MatchingInlineAsm)
override;
1644 bool ParseDirective(AsmToken DirectiveID)
override;
1645 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1646 void onEndOfFile()
override;
1648 OperandMode
Mode = OperandMode_Default);
1649 StringRef parseMnemonicSuffix(StringRef Name);
1650 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1656 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1660 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1661 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1663 ParseStatus parseOperandArrayWithPrefix(
1665 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1666 bool (*ConvertResult)(int64_t &) =
nullptr);
1670 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1671 bool IgnoreNegative =
false);
1672 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1676 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1680 ArrayRef<const char *> Ids,
1684 ArrayRef<const char *> Ids,
1685 AMDGPUOperand::ImmTy
Type);
1688 bool isOperandModifier(
const AsmToken &Token,
1689 const AsmToken &NextToken)
const;
1690 bool isRegOrOperandModifier(
const AsmToken &Token,
1691 const AsmToken &NextToken)
const;
1692 bool isNamedOperandModifier(
const AsmToken &Token,
1693 const AsmToken &NextToken)
const;
1694 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1695 const AsmToken &NextToken)
const;
1696 bool parseSP3NegModifier();
1703 bool AllowImm =
true);
1705 bool AllowImm =
true);
1710 AMDGPUOperand::ImmTy ImmTy);
1715 AMDGPUOperand::ImmTy
Type);
1719 AMDGPUOperand::ImmTy
Type);
1723 AMDGPUOperand::ImmTy
Type);
1727 ParseStatus parseDfmtNfmt(int64_t &
Format);
1728 ParseStatus parseUfmt(int64_t &
Format);
1729 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1731 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1734 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1735 ParseStatus parseNumericFormat(int64_t &
Format);
1739 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1740 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1745 bool parseCnt(int64_t &IntVal);
1748 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1749 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1752 bool parseDelay(int64_t &Delay);
1758 struct OperandInfoTy {
1761 bool IsSymbolic =
false;
1762 bool IsDefined =
false;
1764 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1767 struct StructuredOpField : OperandInfoTy {
1771 bool IsDefined =
false;
1773 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1774 unsigned Width, int64_t
Default)
1775 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1776 virtual ~StructuredOpField() =
default;
1778 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1779 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1783 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1785 return Error(Parser,
"not supported on this GPU");
1787 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1795 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1796 OperandInfoTy &Stream);
1797 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1798 const OperandInfoTy &Stream);
1800 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1801 OperandInfoTy &Width);
1806 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1813 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1815 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1819 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1825 bool validateConstantBusLimitations(
const MCInst &Inst,
1827 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1830 bool tryVOPD(
const MCInst &Inst);
1831 bool tryVOPD3(
const MCInst &Inst);
1832 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1834 bool validateIntClampSupported(
const MCInst &Inst);
1835 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1836 bool validateMIMGGatherDMask(
const MCInst &Inst);
1838 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1839 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1840 bool validateMIMGD16(
const MCInst &Inst);
1842 bool validateTensorR128(
const MCInst &Inst);
1843 bool validateMIMGMSAA(
const MCInst &Inst);
1844 bool validateOpSel(
const MCInst &Inst);
1845 bool validateTrue16OpSel(
const MCInst &Inst);
1846 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1848 bool validateVccOperand(MCRegister
Reg)
const;
1853 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1854 bool validateVGPRAlign(
const MCInst &Inst)
const;
1858 bool validateDivScale(
const MCInst &Inst);
1863 const unsigned CPol);
1867 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1868 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1869 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1870 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1872 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1873 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1874 ArrayRef<unsigned> Variants);
1875 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1877 bool isId(
const StringRef Id)
const;
1878 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1880 StringRef getId()
const;
1881 bool trySkipId(
const StringRef Id);
1882 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1886 bool parseString(StringRef &Val,
1887 const StringRef ErrMsg =
"expected a string");
1888 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1894 StringRef getTokenStr()
const;
1895 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1897 SMLoc getLoc()
const;
1901 void onBeginOfFile()
override;
1905 void emitTargetDirective();
1906 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1918 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1919 const unsigned MaxVal,
const Twine &ErrMsg,
1921 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1922 const unsigned MinVal,
const unsigned MaxVal,
1923 const StringRef ErrMsg);
1925 bool parseSwizzleOffset(int64_t &
Imm);
1926 bool parseSwizzleMacro(int64_t &
Imm);
1927 bool parseSwizzleQuadPerm(int64_t &
Imm);
1928 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1929 bool parseSwizzleBroadcast(int64_t &
Imm);
1930 bool parseSwizzleSwap(int64_t &
Imm);
1931 bool parseSwizzleReverse(int64_t &
Imm);
1932 bool parseSwizzleFFT(int64_t &
Imm);
1933 bool parseSwizzleRotate(int64_t &
Imm);
1936 int64_t parseGPRIdxMacro();
1939 cvtMubufImpl(Inst,
Operands,
false);
1942 cvtMubufImpl(Inst,
Operands,
true);
1948 OptionalImmIndexMap &OptionalIdx);
1957 OptionalImmIndexMap &OptionalIdx);
1959 OptionalImmIndexMap &OptionalIdx);
1963 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1965 bool parseDimId(
unsigned &Encoding);
1967 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1971 int64_t parseDPPCtrlSel(StringRef Ctrl);
1972 int64_t parseDPPCtrlPerm();
1978 bool IsDPP8 =
false);
1984 AMDGPUOperand::ImmTy
Type);
1992 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1995 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1996 bool SkipSrcVcc =
false);
2105bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2115 if (!isImmTy(ImmTyNone)) {
2120 if (getModifiers().
Lit != LitModifier::None)
2130 if (type == MVT::f64 || type == MVT::i64) {
2132 AsmParser->hasInv2PiInlineImm());
2135 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2154 APFloat::rmNearestTiesToEven, &Lost);
2161 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2163 AsmParser->hasInv2PiInlineImm());
2168 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2169 AsmParser->hasInv2PiInlineImm());
2173 if (type == MVT::f64 || type == MVT::i64) {
2175 AsmParser->hasInv2PiInlineImm());
2184 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2185 AsmParser->hasInv2PiInlineImm());
2189 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2190 AsmParser->hasInv2PiInlineImm());
2193bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2195 if (!isImmTy(ImmTyNone)) {
2200 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2205 if (type == MVT::f64 && hasFPModifiers()) {
2225 if (type == MVT::f64) {
2230 if (type == MVT::i64) {
2243 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2244 : (type == MVT::v2i16) ? MVT::f32
2245 : (type == MVT::v2f32) ? MVT::f32
2248 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2252bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2253 return isRegKind() &&
2254 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2257bool AMDGPUOperand::isVRegWithInputMods()
const {
2258 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2260 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2261 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2264template <
bool IsFake16>
2265bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2266 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2267 : AMDGPU::VGPR_16_Lo128RegClassID);
2270template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2271 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2272 : AMDGPU::VGPR_16RegClassID);
2275bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2276 if (AsmParser->isVI())
2278 if (AsmParser->isGFX9Plus())
2279 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2283bool AMDGPUOperand::isSDWAFP16Operand()
const {
2284 return isSDWAOperand(MVT::f16);
2287bool AMDGPUOperand::isSDWAFP32Operand()
const {
2288 return isSDWAOperand(MVT::f32);
2291bool AMDGPUOperand::isSDWAInt16Operand()
const {
2292 return isSDWAOperand(MVT::i16);
2295bool AMDGPUOperand::isSDWAInt32Operand()
const {
2296 return isSDWAOperand(MVT::i32);
2299bool AMDGPUOperand::isBoolReg()
const {
2300 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2301 (AsmParser->isWave32() && isSCSrc_b32()));
2305 unsigned Size)
const {
2306 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2321void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2322 bool ApplyModifiers)
const {
2332 addLiteralImmOperand(Inst,
Imm.Val,
2333 ApplyModifiers & isImmTy(ImmTyNone) &&
2334 Imm.Mods.hasFPModifiers());
2336 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2341void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2342 bool ApplyModifiers)
const {
2343 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2348 if (ApplyModifiers) {
2350 const unsigned Size =
2352 Val = applyInputFPModifiers(Val,
Size);
2356 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2358 bool CanUse64BitLiterals =
2361 MCContext &Ctx = AsmParser->getContext();
2372 if (
Lit == LitModifier::None &&
2374 AsmParser->hasInv2PiInlineImm())) {
2382 bool HasMandatoryLiteral =
2385 if (
Literal.getLoBits(32) != 0 &&
2386 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2387 !HasMandatoryLiteral) {
2388 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2390 "Can't encode literal as exact 64-bit floating-point operand. "
2391 "Low 32-bits will be set to zero");
2392 Val &= 0xffffffff00000000u;
2398 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2404 Lit = LitModifier::Lit64;
2405 }
else if (
Lit == LitModifier::Lit) {
2419 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2421 Lit = LitModifier::Lit64;
2428 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2429 Literal == 0x3fc45f306725feed) {
2468 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2475 if (
Lit != LitModifier::None) {
2506 if (
Lit == LitModifier::None &&
2516 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2524 if (
Lit == LitModifier::None &&
2532 if (!AsmParser->has64BitLiterals()) {
2533 Val =
static_cast<uint64_t>(Val) << 32;
2540 if (
Lit == LitModifier::Lit ||
2542 Val =
static_cast<uint64_t>(Val) << 32;
2546 if (
Lit == LitModifier::Lit)
2572 if (
Lit != LitModifier::None) {
2580void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2586bool AMDGPUOperand::isInlineValue()
const {
2594void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2605 if (Is == IS_VGPR) {
2610 return AMDGPU::VGPR_32RegClassID;
2612 return AMDGPU::VReg_64RegClassID;
2614 return AMDGPU::VReg_96RegClassID;
2616 return AMDGPU::VReg_128RegClassID;
2618 return AMDGPU::VReg_160RegClassID;
2620 return AMDGPU::VReg_192RegClassID;
2622 return AMDGPU::VReg_224RegClassID;
2624 return AMDGPU::VReg_256RegClassID;
2626 return AMDGPU::VReg_288RegClassID;
2628 return AMDGPU::VReg_320RegClassID;
2630 return AMDGPU::VReg_352RegClassID;
2632 return AMDGPU::VReg_384RegClassID;
2634 return AMDGPU::VReg_512RegClassID;
2636 return AMDGPU::VReg_1024RegClassID;
2638 }
else if (Is == IS_TTMP) {
2643 return AMDGPU::TTMP_32RegClassID;
2645 return AMDGPU::TTMP_64RegClassID;
2647 return AMDGPU::TTMP_128RegClassID;
2649 return AMDGPU::TTMP_256RegClassID;
2651 return AMDGPU::TTMP_512RegClassID;
2653 }
else if (Is == IS_SGPR) {
2658 return AMDGPU::SGPR_32RegClassID;
2660 return AMDGPU::SGPR_64RegClassID;
2662 return AMDGPU::SGPR_96RegClassID;
2664 return AMDGPU::SGPR_128RegClassID;
2666 return AMDGPU::SGPR_160RegClassID;
2668 return AMDGPU::SGPR_192RegClassID;
2670 return AMDGPU::SGPR_224RegClassID;
2672 return AMDGPU::SGPR_256RegClassID;
2674 return AMDGPU::SGPR_288RegClassID;
2676 return AMDGPU::SGPR_320RegClassID;
2678 return AMDGPU::SGPR_352RegClassID;
2680 return AMDGPU::SGPR_384RegClassID;
2682 return AMDGPU::SGPR_512RegClassID;
2684 }
else if (Is == IS_AGPR) {
2689 return AMDGPU::AGPR_32RegClassID;
2691 return AMDGPU::AReg_64RegClassID;
2693 return AMDGPU::AReg_96RegClassID;
2695 return AMDGPU::AReg_128RegClassID;
2697 return AMDGPU::AReg_160RegClassID;
2699 return AMDGPU::AReg_192RegClassID;
2701 return AMDGPU::AReg_224RegClassID;
2703 return AMDGPU::AReg_256RegClassID;
2705 return AMDGPU::AReg_288RegClassID;
2707 return AMDGPU::AReg_320RegClassID;
2709 return AMDGPU::AReg_352RegClassID;
2711 return AMDGPU::AReg_384RegClassID;
2713 return AMDGPU::AReg_512RegClassID;
2715 return AMDGPU::AReg_1024RegClassID;
2723 .
Case(
"exec", AMDGPU::EXEC)
2724 .
Case(
"vcc", AMDGPU::VCC)
2725 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2726 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2727 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2728 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2729 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2730 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2731 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2732 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2733 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2734 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2735 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2736 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2737 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2738 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2739 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2740 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2741 .
Case(
"m0", AMDGPU::M0)
2742 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2743 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2744 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2745 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2746 .
Case(
"scc", AMDGPU::SRC_SCC)
2747 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2748 .
Case(
"tba", AMDGPU::TBA)
2749 .
Case(
"tma", AMDGPU::TMA)
2750 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2751 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2752 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2753 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2754 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2755 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2756 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2757 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2758 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2759 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2760 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2761 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2762 .
Case(
"pc", AMDGPU::PC_REG)
2763 .
Case(
"null", AMDGPU::SGPR_NULL)
2767bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2768 SMLoc &EndLoc,
bool RestoreOnFailure) {
2769 auto R = parseRegister();
2773 RegNo =
R->getReg();
2774 StartLoc =
R->getStartLoc();
2775 EndLoc =
R->getEndLoc();
2779bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2781 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2784ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2786 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2787 bool PendingErrors = getParser().hasPendingError();
2788 getParser().clearPendingErrors();
2796bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2797 RegisterKind RegKind,
2799 RegisterKind RegKind1, SMLoc Loc) {
2801 if (RegKind == IS_SGPR) {
2802 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2803 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2804 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2810 if (RegKind != RegKind1) {
2811 Error(Loc,
"registers in a list must be of the same kind");
2817 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2822 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2823 Reg = AMDGPU::FLAT_SCR;
2827 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2828 Reg = AMDGPU::XNACK_MASK;
2832 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2837 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2842 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2847 Error(Loc,
"register does not fit in the list");
2853 if (Reg1 !=
Reg + RegWidth / 32) {
2854 Error(Loc,
"registers in a list must have consecutive indices");
2870 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2871 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2875 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2881 if (Str.starts_with(
Reg.Name))
2887 return !Str.getAsInteger(10, Num);
2890bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2891 const AsmToken &NextToken)
const {
2906 StringRef RegSuffix = Str.substr(
RegName.size());
2907 if (!RegSuffix.
empty()) {
2924bool AMDGPUAsmParser::isRegister() {
2925 return isRegister(
getToken(), peekToken());
2928MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2929 unsigned SubReg,
unsigned RegWidth,
2933 unsigned AlignSize = 1;
2934 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2940 if (RegNum % AlignSize != 0) {
2941 Error(Loc,
"invalid register alignment");
2942 return MCRegister();
2945 unsigned RegIdx = RegNum / AlignSize;
2948 Error(Loc,
"invalid or unsupported register size");
2949 return MCRegister();
2953 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2954 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2955 Error(Loc,
"register index is out of range");
2956 return AMDGPU::NoRegister;
2959 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2960 Error(Loc,
"register index is out of range");
2961 return MCRegister();
2977bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2979 int64_t RegLo, RegHi;
2983 SMLoc FirstIdxLoc = getLoc();
2990 SecondIdxLoc = getLoc();
3001 Error(FirstIdxLoc,
"invalid register index");
3006 Error(SecondIdxLoc,
"invalid register index");
3010 if (RegLo > RegHi) {
3011 Error(FirstIdxLoc,
"first register index should not exceed second index");
3015 if (RegHi == RegLo) {
3016 StringRef RegSuffix = getTokenStr();
3017 if (RegSuffix ==
".l") {
3018 SubReg = AMDGPU::lo16;
3020 }
else if (RegSuffix ==
".h") {
3021 SubReg = AMDGPU::hi16;
3026 Num =
static_cast<unsigned>(RegLo);
3027 RegWidth = 32 * ((RegHi - RegLo) + 1);
3032MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3035 SmallVectorImpl<AsmToken> &Tokens) {
3041 RegKind = IS_SPECIAL;
3048MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3051 SmallVectorImpl<AsmToken> &Tokens) {
3053 StringRef
RegName = getTokenStr();
3054 auto Loc = getLoc();
3058 Error(Loc,
"invalid register name");
3059 return MCRegister();
3067 unsigned SubReg = NoSubRegister;
3068 bool IsRange =
false;
3069 if (!RegSuffix.
empty()) {
3071 SubReg = AMDGPU::lo16;
3073 SubReg = AMDGPU::hi16;
3077 Error(Loc,
"invalid register index");
3078 return MCRegister();
3084 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3085 return MCRegister();
3089 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3090 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3091 if (RegKind == IS_SGPR && IsRange
3092 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3093 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3094 Error(Loc,
"register index is out of range");
3095 return MCRegister();
3101MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3102 unsigned &RegNum,
unsigned &RegWidth,
3103 SmallVectorImpl<AsmToken> &Tokens) {
3105 auto ListLoc = getLoc();
3108 "expected a register or a list of registers")) {
3109 return MCRegister();
3114 auto Loc = getLoc();
3115 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3116 return MCRegister();
3117 if (RegWidth != 32) {
3118 Error(Loc,
"expected a single 32-bit register");
3119 return MCRegister();
3123 RegisterKind NextRegKind;
3125 unsigned NextRegNum, NextRegWidth;
3128 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3130 return MCRegister();
3132 if (NextRegWidth != 32) {
3133 Error(Loc,
"expected a single 32-bit register");
3134 return MCRegister();
3136 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3138 return MCRegister();
3142 "expected a comma or a closing square bracket")) {
3143 return MCRegister();
3147 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3152bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3153 MCRegister &
Reg,
unsigned &RegNum,
3155 SmallVectorImpl<AsmToken> &Tokens) {
3156 auto Loc = getLoc();
3160 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3162 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3164 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3169 assert(Parser.hasPendingError());
3173 if (!subtargetHasRegister(*
TRI,
Reg)) {
3174 if (
Reg == AMDGPU::SGPR_NULL) {
3175 Error(Loc,
"'null' operand is not supported on this GPU");
3178 " register not available on this GPU");
3186bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3187 MCRegister &
Reg,
unsigned &RegNum,
3189 bool RestoreOnFailure ) {
3193 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3194 if (RestoreOnFailure) {
3195 while (!Tokens.
empty()) {
3204std::optional<StringRef>
3205AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3208 return StringRef(
".amdgcn.next_free_vgpr");
3210 return StringRef(
".amdgcn.next_free_sgpr");
3212 return std::nullopt;
3216void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3217 auto SymbolName = getGprCountSymbolName(RegKind);
3218 assert(SymbolName &&
"initializing invalid register kind");
3224bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3225 unsigned DwordRegIndex,
3226 unsigned RegWidth) {
3231 auto SymbolName = getGprCountSymbolName(RegKind);
3236 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3240 return !
Error(getLoc(),
3241 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3245 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3247 if (OldCount <= NewMax)
3253std::unique_ptr<AMDGPUOperand>
3254AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3256 SMLoc StartLoc = Tok.getLoc();
3257 SMLoc EndLoc = Tok.getEndLoc();
3258 RegisterKind RegKind;
3260 unsigned RegNum, RegWidth;
3262 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3266 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3269 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3270 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3277 if (isRegister() || isModifier())
3280 if (
Lit == LitModifier::None) {
3281 if (trySkipId(
"lit"))
3282 Lit = LitModifier::Lit;
3283 else if (trySkipId(
"lit64"))
3284 Lit = LitModifier::Lit64;
3286 if (
Lit != LitModifier::None) {
3289 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3298 const auto &NextTok = peekToken();
3301 bool Negate =
false;
3309 AMDGPUOperand::Modifiers Mods;
3317 StringRef Num = getTokenStr();
3320 APFloat RealVal(APFloat::IEEEdouble());
3321 auto roundMode = APFloat::rmNearestTiesToEven;
3322 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3325 RealVal.changeSign();
3328 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3329 S, AMDGPUOperand::ImmTyNone,
true));
3330 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3331 Op.setModifiers(Mods);
3340 if (HasSP3AbsModifier) {
3349 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3352 if (Parser.parseExpression(Expr))
3356 if (Expr->evaluateAsAbsolute(IntVal)) {
3358 return Error(S,
"literal value out of range");
3359 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3360 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3361 Op.setModifiers(Mods);
3363 if (
Lit != LitModifier::None)
3365 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3378 if (
auto R = parseRegister()) {
3388 ParseStatus Res = parseReg(
Operands);
3396bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3397 const AsmToken &NextToken)
const {
3400 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3405bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3406 const AsmToken &NextToken)
const {
3410bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3411 const AsmToken &NextToken)
const {
3412 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3415bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3416 const AsmToken &NextToken)
const {
3417 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3433bool AMDGPUAsmParser::isModifier() {
3436 AsmToken NextToken[2];
3437 peekTokens(NextToken);
3439 return isOperandModifier(Tok, NextToken[0]) ||
3441 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3442 isOpcodeModifierWithVal(Tok, NextToken[0]);
3467bool AMDGPUAsmParser::parseSP3NegModifier() {
3469 AsmToken NextToken[2];
3470 peekTokens(NextToken);
3473 (isRegister(NextToken[0], NextToken[1]) ||
3491 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3493 SP3Neg = parseSP3NegModifier();
3496 Neg = trySkipId(
"neg");
3498 return Error(Loc,
"expected register or immediate");
3502 Abs = trySkipId(
"abs");
3507 if (trySkipId(
"lit")) {
3508 Lit = LitModifier::Lit;
3511 }
else if (trySkipId(
"lit64")) {
3512 Lit = LitModifier::Lit64;
3515 if (!has64BitLiterals())
3516 return Error(Loc,
"lit64 is not supported on this GPU");
3522 return Error(Loc,
"expected register or immediate");
3531 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3535 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3536 Error(Loc,
"expected immediate with lit modifier");
3538 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3544 if (
Lit != LitModifier::None &&
3548 AMDGPUOperand::Modifiers Mods;
3549 Mods.Abs = Abs || SP3Abs;
3550 Mods.Neg = Neg || SP3Neg;
3553 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3554 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3556 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3557 Op.setModifiers(Mods);
3565 bool Sext = trySkipId(
"sext");
3566 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3581 AMDGPUOperand::Modifiers Mods;
3584 if (Mods.hasIntModifiers()) {
3585 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3587 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3588 Op.setModifiers(Mods);
3595 return parseRegOrImmWithFPInputMods(
Operands,
false);
3599 return parseRegOrImmWithIntInputMods(
Operands,
false);
3603 auto Loc = getLoc();
3604 if (trySkipId(
"off")) {
3606 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3613 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3622unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3627 return Match_InvalidOperand;
3629 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3630 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3633 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3635 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3636 return Match_InvalidOperand;
3644 if (tryAnotherVOPDEncoding(Inst))
3645 return Match_InvalidOperand;
3647 return Match_Success;
3651 static const unsigned Variants[] = {
3660ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3661 if (isForcedDPP() && isForcedVOP3()) {
3665 if (getForcedEncodingSize() == 32) {
3670 if (isForcedVOP3()) {
3675 if (isForcedSDWA()) {
3681 if (isForcedDPP()) {
3689StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3690 if (isForcedDPP() && isForcedVOP3())
3693 if (getForcedEncodingSize() == 32)
3709AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3713 case AMDGPU::FLAT_SCR:
3715 case AMDGPU::VCC_LO:
3716 case AMDGPU::VCC_HI:
3723 return MCRegister();
3730bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3731 unsigned OpIdx)
const {
3739 const MCOperand &MO = Inst.
getOperand(OpIdx);
3788unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3794 case AMDGPU::V_LSHLREV_B64_e64:
3795 case AMDGPU::V_LSHLREV_B64_gfx10:
3796 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3797 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3798 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3799 case AMDGPU::V_LSHRREV_B64_e64:
3800 case AMDGPU::V_LSHRREV_B64_gfx10:
3801 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3802 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3803 case AMDGPU::V_ASHRREV_I64_e64:
3804 case AMDGPU::V_ASHRREV_I64_gfx10:
3805 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3806 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3807 case AMDGPU::V_LSHL_B64_e64:
3808 case AMDGPU::V_LSHR_B64_e64:
3809 case AMDGPU::V_ASHR_I64_e64:
3822 bool AddMandatoryLiterals =
false) {
3825 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3829 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3831 return {getNamedOperandIdx(Opcode, OpName::src0X),
3832 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3833 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3834 getNamedOperandIdx(Opcode, OpName::src0Y),
3835 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3836 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3841 return {getNamedOperandIdx(Opcode, OpName::src0),
3842 getNamedOperandIdx(Opcode, OpName::src1),
3843 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3846bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3847 const MCOperand &MO = Inst.
getOperand(OpIdx);
3849 return !isInlineConstant(Inst, OpIdx);
3856 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3867 const unsigned Opcode = Inst.
getOpcode();
3868 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3871 if (!LaneSelOp.
isReg())
3874 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3877bool AMDGPUAsmParser::validateConstantBusLimitations(
3879 const unsigned Opcode = Inst.
getOpcode();
3880 const MCInstrDesc &
Desc = MII.
get(Opcode);
3881 MCRegister LastSGPR;
3882 unsigned ConstantBusUseCount = 0;
3883 unsigned NumLiterals = 0;
3884 unsigned LiteralSize;
3900 SmallDenseSet<MCRegister> SGPRsUsed;
3901 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3903 SGPRsUsed.
insert(SGPRUsed);
3904 ++ConstantBusUseCount;
3909 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3911 for (
int OpIdx : OpIndices) {
3915 const MCOperand &MO = Inst.
getOperand(OpIdx);
3916 if (usesConstantBus(Inst, OpIdx)) {
3925 if (SGPRsUsed.
insert(LastSGPR).second) {
3926 ++ConstantBusUseCount;
3946 if (NumLiterals == 0) {
3949 }
else if (LiteralSize !=
Size) {
3955 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3957 "invalid operand (violates constant bus restrictions)");
3964std::optional<unsigned>
3965AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3967 const unsigned Opcode = Inst.
getOpcode();
3973 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3974 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3983 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3984 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3985 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3986 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3987 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3988 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3992 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3993 int I = getNamedOperandIdx(Opcode, OpName);
3997 int64_t
Imm =
Op.getImm();
4003 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4004 OpName::vsrc2Y, OpName::imm}) {
4005 int I = getNamedOperandIdx(Opcode, OpName);
4015 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4016 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4018 return InvalidCompOprIdx;
4021bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4028 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
4029 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4030 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4032 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4036 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4037 if (!InvalidCompOprIdx.has_value())
4040 auto CompOprIdx = *InvalidCompOprIdx;
4043 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4044 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4047 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
4048 if (CompOprIdx == VOPD::Component::DST) {
4050 Error(Loc,
"dst registers must be distinct");
4052 Error(Loc,
"one dst register must be even and the other odd");
4054 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4055 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4056 " operands must use different VGPR banks");
4064bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4066 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4067 if (!InvalidCompOprIdx.has_value())
4071 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4072 if (InvalidCompOprIdx.has_value()) {
4077 if (*InvalidCompOprIdx == VOPD::Component::DST)
4090bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4091 const unsigned Opcode = Inst.
getOpcode();
4106 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4107 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4108 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4109 int I = getNamedOperandIdx(Opcode, OpName);
4116 return !tryVOPD3(Inst);
4121bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4126 return tryVOPD(Inst);
4127 return tryVOPD3(Inst);
4130bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4135 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4143bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4151 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4152 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4153 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4158 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4159 hasBVHRayTracingInsts())
4162 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4163 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4168 bool IsPackedD16 =
false;
4171 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4172 IsPackedD16 = D16Idx >= 0;
4174 DataSize = (DataSize + 1) / 2;
4177 if ((VDataSize / 4) == DataSize + TFESize)
4182 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4184 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4186 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4190bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4199 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4201 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4202 AMDGPU::OpName RSrcOpName =
4204 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4205 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4206 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4210 assert(SrsrcIdx > VAddr0Idx);
4213 if (BaseOpcode->
BVH) {
4214 if (IsA16 == BaseOpcode->
A16)
4216 Error(IDLoc,
"image address size does not match a16");
4222 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4223 unsigned ActualAddrSize =
4224 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4226 unsigned ExpectedAddrSize =
4230 if (hasPartialNSAEncoding() &&
4232 int VAddrLastIdx = SrsrcIdx - 1;
4233 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4235 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4238 if (ExpectedAddrSize > 12)
4239 ExpectedAddrSize = 16;
4244 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4248 if (ActualAddrSize == ExpectedAddrSize)
4251 Error(IDLoc,
"image address size does not match dim and a16");
4255bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4262 if (!
Desc.mayLoad() || !
Desc.mayStore())
4265 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4272 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4275bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4282 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4290 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4293bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4307 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4308 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4315bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4322 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4325 if (!BaseOpcode->
MSAA)
4328 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4334 return DimInfo->
MSAA;
4339 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4340 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4341 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4351bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4359 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4362 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4370 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4374bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4379 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4382 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4385 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4393 "source operand must be either a VGPR or an inline constant");
4400bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4405 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4408 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4412 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4414 "inline constants are not allowed for this operand");
4421bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4429 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4430 if (BlgpIdx != -1) {
4431 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4432 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4442 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4444 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4449 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4451 "wrong register tuple size for blgp value " + Twine(BLGP));
4459 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4463 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4467 MCRegister Src2Reg = Src2.
getReg();
4469 if (Src2Reg == DstReg)
4474 .getSizeInBits() <= 128)
4477 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4479 "source 2 operand must not partially overlap with dst");
4486bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4490 case V_DIV_SCALE_F32_gfx6_gfx7:
4491 case V_DIV_SCALE_F32_vi:
4492 case V_DIV_SCALE_F32_gfx10:
4493 case V_DIV_SCALE_F64_gfx6_gfx7:
4494 case V_DIV_SCALE_F64_vi:
4495 case V_DIV_SCALE_F64_gfx10:
4502 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4503 AMDGPU::OpName::src2_modifiers}) {
4514bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4521 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4530bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4536 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4543 case AMDGPU::V_SUBREV_F32_e32:
4544 case AMDGPU::V_SUBREV_F32_e64:
4545 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4546 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4547 case AMDGPU::V_SUBREV_F32_e32_vi:
4548 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4549 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4550 case AMDGPU::V_SUBREV_F32_e64_vi:
4552 case AMDGPU::V_SUBREV_CO_U32_e32:
4553 case AMDGPU::V_SUBREV_CO_U32_e64:
4554 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4555 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4557 case AMDGPU::V_SUBBREV_U32_e32:
4558 case AMDGPU::V_SUBBREV_U32_e64:
4559 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4560 case AMDGPU::V_SUBBREV_U32_e32_vi:
4561 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4562 case AMDGPU::V_SUBBREV_U32_e64_vi:
4564 case AMDGPU::V_SUBREV_U32_e32:
4565 case AMDGPU::V_SUBREV_U32_e64:
4566 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4567 case AMDGPU::V_SUBREV_U32_e32_vi:
4568 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4569 case AMDGPU::V_SUBREV_U32_e64_vi:
4571 case AMDGPU::V_SUBREV_F16_e32:
4572 case AMDGPU::V_SUBREV_F16_e64:
4573 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4574 case AMDGPU::V_SUBREV_F16_e32_vi:
4575 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4576 case AMDGPU::V_SUBREV_F16_e64_vi:
4578 case AMDGPU::V_SUBREV_U16_e32:
4579 case AMDGPU::V_SUBREV_U16_e64:
4580 case AMDGPU::V_SUBREV_U16_e32_vi:
4581 case AMDGPU::V_SUBREV_U16_e64_vi:
4583 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4584 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4585 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4587 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4588 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4590 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4591 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4593 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4594 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4596 case AMDGPU::V_LSHRREV_B32_e32:
4597 case AMDGPU::V_LSHRREV_B32_e64:
4598 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4599 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4600 case AMDGPU::V_LSHRREV_B32_e32_vi:
4601 case AMDGPU::V_LSHRREV_B32_e64_vi:
4602 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4603 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4605 case AMDGPU::V_ASHRREV_I32_e32:
4606 case AMDGPU::V_ASHRREV_I32_e64:
4607 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4608 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4609 case AMDGPU::V_ASHRREV_I32_e32_vi:
4610 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4611 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4612 case AMDGPU::V_ASHRREV_I32_e64_vi:
4614 case AMDGPU::V_LSHLREV_B32_e32:
4615 case AMDGPU::V_LSHLREV_B32_e64:
4616 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4617 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4618 case AMDGPU::V_LSHLREV_B32_e32_vi:
4619 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4620 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4621 case AMDGPU::V_LSHLREV_B32_e64_vi:
4623 case AMDGPU::V_LSHLREV_B16_e32:
4624 case AMDGPU::V_LSHLREV_B16_e64:
4625 case AMDGPU::V_LSHLREV_B16_e32_vi:
4626 case AMDGPU::V_LSHLREV_B16_e64_vi:
4627 case AMDGPU::V_LSHLREV_B16_gfx10:
4629 case AMDGPU::V_LSHRREV_B16_e32:
4630 case AMDGPU::V_LSHRREV_B16_e64:
4631 case AMDGPU::V_LSHRREV_B16_e32_vi:
4632 case AMDGPU::V_LSHRREV_B16_e64_vi:
4633 case AMDGPU::V_LSHRREV_B16_gfx10:
4635 case AMDGPU::V_ASHRREV_I16_e32:
4636 case AMDGPU::V_ASHRREV_I16_e64:
4637 case AMDGPU::V_ASHRREV_I16_e32_vi:
4638 case AMDGPU::V_ASHRREV_I16_e64_vi:
4639 case AMDGPU::V_ASHRREV_I16_gfx10:
4641 case AMDGPU::V_LSHLREV_B64_e64:
4642 case AMDGPU::V_LSHLREV_B64_gfx10:
4643 case AMDGPU::V_LSHLREV_B64_vi:
4645 case AMDGPU::V_LSHRREV_B64_e64:
4646 case AMDGPU::V_LSHRREV_B64_gfx10:
4647 case AMDGPU::V_LSHRREV_B64_vi:
4649 case AMDGPU::V_ASHRREV_I64_e64:
4650 case AMDGPU::V_ASHRREV_I64_gfx10:
4651 case AMDGPU::V_ASHRREV_I64_vi:
4653 case AMDGPU::V_PK_LSHLREV_B16:
4654 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4655 case AMDGPU::V_PK_LSHLREV_B16_vi:
4657 case AMDGPU::V_PK_LSHRREV_B16:
4658 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4659 case AMDGPU::V_PK_LSHRREV_B16_vi:
4660 case AMDGPU::V_PK_ASHRREV_I16:
4661 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4662 case AMDGPU::V_PK_ASHRREV_I16_vi:
4669bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4671 const unsigned Opcode = Inst.
getOpcode();
4680 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4681 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4685 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4689 "lds_direct is not supported on this GPU");
4695 "lds_direct cannot be used with this instruction");
4699 if (SrcName != OpName::src0) {
4701 "lds_direct may be used as src0 only");
4711 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4712 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4713 if (
Op.isFlatOffset())
4714 return Op.getStartLoc();
4719bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4722 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4727 return validateFlatOffset(Inst,
Operands);
4730 return validateSMEMOffset(Inst,
Operands);
4735 const unsigned OffsetSize = 24;
4736 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4738 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4739 "-bit unsigned offset for buffer ops");
4743 const unsigned OffsetSize = 16;
4744 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4746 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4753bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4759 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4763 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4765 "flat offset modifier is not supported on this GPU");
4772 bool AllowNegative =
4774 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4776 Twine(
"expected a ") +
4777 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4778 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4787 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4788 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4789 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4790 return Op.getStartLoc();
4795bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4804 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4820 ?
"expected a 23-bit unsigned offset for buffer ops"
4821 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4822 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4823 :
"expected a 21-bit signed offset");
4828bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4831 const MCInstrDesc &
Desc = MII.
get(Opcode);
4835 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4836 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4838 const int OpIndices[] = {Src0Idx, Src1Idx};
4840 unsigned NumExprs = 0;
4841 unsigned NumLiterals = 0;
4844 for (
int OpIdx : OpIndices) {
4848 const MCOperand &MO = Inst.
getOperand(OpIdx);
4852 std::optional<int64_t>
Imm;
4855 }
else if (MO.
isExpr()) {
4864 if (!
Imm.has_value()) {
4866 }
else if (!isInlineConstant(Inst, OpIdx)) {
4870 if (NumLiterals == 0 || LiteralValue !=
Value) {
4878 if (NumLiterals + NumExprs <= 1)
4882 "only one unique literal operand is allowed");
4886bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4889 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4897 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4898 if (OpSelIdx != -1) {
4902 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4903 if (OpSelHiIdx != -1) {
4912 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4922 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4923 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4924 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4925 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4927 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4928 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4934 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4936 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4946 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4947 if (Src2Idx != -1) {
4948 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4958bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4959 if (!hasTrue16Insts())
4961 const MCRegisterInfo *MRI = getMRI();
4963 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4969 if (OpSelOpValue == 0)
4971 unsigned OpCount = 0;
4972 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4973 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4974 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4981 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4982 if (OpSelOpIsHi != VGPRSuffixIsHi)
4991bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4992 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5004 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5015 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5016 AMDGPU::OpName::src1_modifiers,
5017 AMDGPU::OpName::src2_modifiers};
5019 for (
unsigned i = 0; i < 3; ++i) {
5029bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5032 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5033 if (DppCtrlIdx >= 0) {
5040 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
5041 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5042 :
"DP ALU dpp only supports row_newbcast");
5047 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5048 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5051 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5053 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5057 "invalid operand for instruction");
5062 "src1 immediate operand invalid for instruction");
5072bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5073 return (
Reg == AMDGPU::VCC && isWave64()) ||
5074 (
Reg == AMDGPU::VCC_LO && isWave32());
5078bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5081 const MCInstrDesc &
Desc = MII.
get(Opcode);
5082 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5089 std::optional<unsigned> LiteralOpIdx;
5092 for (
int OpIdx : OpIndices) {
5096 const MCOperand &MO = Inst.
getOperand(OpIdx);
5102 std::optional<int64_t>
Imm;
5108 bool IsAnotherLiteral =
false;
5109 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5110 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5111 if (!
Imm.has_value()) {
5113 IsAnotherLiteral =
true;
5114 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5119 HasMandatoryLiteral);
5120 unsigned OpTy =
Desc.operands()[OpIdx].OperandType;
5130 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5131 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5133 "invalid operand for instruction");
5138 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5139 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5141 "invalid operand for instruction");
5145 if (IsFP64 && IsValid32Op && !IsForcedFP64)
5152 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5153 !getFeatureBits()[FeatureVOP3Literal]) {
5155 "literal operands are not supported");
5159 if (LiteralOpIdx && IsAnotherLiteral) {
5161 getOperandLoc(
Operands, *LiteralOpIdx)),
5162 "only one unique literal operand is allowed");
5166 if (IsAnotherLiteral)
5167 LiteralOpIdx = OpIdx;
5176 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5190bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5196 ? AMDGPU::OpName::data0
5197 : AMDGPU::OpName::vdata;
5199 const MCRegisterInfo *MRI = getMRI();
5200 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5204 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5205 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5209 auto FB = getFeatureBits();
5210 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5211 if (DataAreg < 0 || DstAreg < 0)
5213 return DstAreg == DataAreg;
5216 return DstAreg < 1 && DataAreg < 1;
5219bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5220 auto FB = getFeatureBits();
5221 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5225 const MCRegisterInfo *MRI = getMRI();
5228 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5231 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5235 case AMDGPU::DS_LOAD_TR6_B96:
5236 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5240 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5241 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5245 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5246 if (VAddrIdx != -1) {
5249 if ((
Sub - AMDGPU::VGPR0) & 1)
5254 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5255 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5260 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5261 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5281 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5282 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5284 return Op.getStartLoc();
5289bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5292 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5295 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5298 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5299 auto FB = getFeatureBits();
5300 bool UsesNeg =
false;
5301 if (FB[AMDGPU::FeatureGFX940Insts]) {
5303 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5304 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5305 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5306 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5311 if (IsNeg == UsesNeg)
5314 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5315 :
"invalid modifier: neg is not supported");
5320bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5326 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5327 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5328 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5329 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5332 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5335 if (
Reg == AMDGPU::SGPR_NULL)
5338 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5342bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5347 return validateGWS(Inst,
Operands);
5352 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5357 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5358 Error(S,
"gds modifier is not supported on this GPU");
5366bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5368 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5372 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5373 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5376 const MCRegisterInfo *MRI = getMRI();
5377 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5379 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5382 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5384 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5391bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5395 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5403 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5406 Error(S,
"scale_offset is not supported on this GPU");
5409 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5412 Error(S,
"nv is not supported on this GPU");
5417 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5420 Error(S,
"scale_offset is not supported for this instruction");
5424 return validateTHAndScopeBits(Inst,
Operands, CPol);
5428 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5429 Error(S,
"cache policy is not supported for SMRD instructions");
5433 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5440 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5444 "scc modifier is not supported for this instruction on this GPU");
5455 :
"instruction must use glc");
5460 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5463 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5465 :
"instruction must not use glc");
5473bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5475 const unsigned CPol) {
5480 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5487 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5492 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5500 return PrintError(
"invalid th value for SMEM instruction");
5507 return PrintError(
"scope and th combination is not valid");
5513 return PrintError(
"invalid th value for atomic instructions");
5516 return PrintError(
"invalid th value for store instructions");
5519 return PrintError(
"invalid th value for load instructions");
5525bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5529 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5531 Error(Loc,
"TFE modifier has no meaning for store instructions");
5539bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5545 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5549 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5552 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5553 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5562 "wrong register tuple size for " +
5567 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5568 !validateFmt(BFmt, AMDGPU::OpName::src1))
5572 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5573 if (AScaleIdx == -1)
5577 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5581 "invalid matrix and scale format combination");
5588bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5590 if (!validateLdsDirect(Inst,
Operands))
5592 if (!validateTrue16OpSel(Inst)) {
5594 "op_sel operand conflicts with 16-bit operand suffix");
5597 if (!validateSOPLiteral(Inst,
Operands))
5599 if (!validateVOPLiteral(Inst,
Operands)) {
5602 if (!validateConstantBusLimitations(Inst,
Operands)) {
5605 if (!validateVOPD(Inst,
Operands)) {
5608 if (!validateIntClampSupported(Inst)) {
5610 "integer clamping is not supported on this GPU");
5613 if (!validateOpSel(Inst)) {
5615 "invalid op_sel operand");
5618 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5620 "invalid neg_lo operand");
5623 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5625 "invalid neg_hi operand");
5628 if (!validateDPP(Inst,
Operands)) {
5632 if (!validateMIMGD16(Inst)) {
5634 "d16 modifier is not supported on this GPU");
5637 if (!validateMIMGDim(Inst,
Operands)) {
5638 Error(IDLoc,
"missing dim operand");
5641 if (!validateTensorR128(Inst)) {
5643 "instruction must set modifier r128=0");
5646 if (!validateMIMGMSAA(Inst)) {
5648 "invalid dim; must be MSAA type");
5651 if (!validateMIMGDataSize(Inst, IDLoc)) {
5654 if (!validateMIMGAddrSize(Inst, IDLoc))
5656 if (!validateMIMGAtomicDMask(Inst)) {
5658 "invalid atomic image dmask");
5661 if (!validateMIMGGatherDMask(Inst)) {
5663 "invalid image_gather dmask: only one bit must be set");
5666 if (!validateMovrels(Inst,
Operands)) {
5669 if (!validateOffset(Inst,
Operands)) {
5672 if (!validateMAIAccWrite(Inst,
Operands)) {
5675 if (!validateMAISrc2(Inst,
Operands)) {
5678 if (!validateMFMA(Inst,
Operands)) {
5681 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5685 if (!validateAGPRLdSt(Inst)) {
5688 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5689 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5690 :
"invalid register class: agpr loads and stores not supported on "
5694 if (!validateVGPRAlign(Inst)) {
5695 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5702 if (!validateBLGP(Inst,
Operands)) {
5706 if (!validateDivScale(Inst)) {
5707 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5710 if (!validateWaitCnt(Inst,
Operands)) {
5713 if (!validateTFE(Inst,
Operands)) {
5716 if (!validateWMMA(Inst,
Operands)) {
5725 unsigned VariantID = 0);
5729 unsigned VariantID);
5731bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5736bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5737 const FeatureBitset &FBS,
5738 ArrayRef<unsigned> Variants) {
5739 for (
auto Variant : Variants) {
5747bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5749 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5752 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5757 getParser().clearPendingErrors();
5761 StringRef VariantName = getMatchedVariantName();
5762 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5763 return Error(IDLoc, Twine(VariantName,
5764 " variant of this instruction is not supported"));
5768 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5769 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5771 FeatureBitset FeaturesWS32 = getFeatureBits();
5772 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5773 .
flip(AMDGPU::FeatureWavefrontSize32);
5774 FeatureBitset AvailableFeaturesWS32 =
5775 ComputeAvailableFeatures(FeaturesWS32);
5777 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5778 return Error(IDLoc,
"instruction requires wavesize=32");
5782 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5783 return Error(IDLoc,
"instruction not supported on this GPU (" +
5784 getSTI().
getCPU() +
")" +
": " + Mnemo);
5789 return Error(IDLoc,
"invalid instruction" + Suggestion);
5795 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5796 if (
Op.isToken() && InvalidOprIdx > 1) {
5797 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5798 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5803bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5807 bool MatchingInlineAsm) {
5810 unsigned Result = Match_Success;
5811 for (
auto Variant : getMatchedVariants()) {
5814 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5818 if (R == Match_Success || R == Match_MissingFeature ||
5819 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5820 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5821 Result != Match_MissingFeature)) {
5825 if (R == Match_Success)
5829 if (Result == Match_Success) {
5830 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5833 emitTargetDirective();
5841 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5848 case Match_MissingFeature:
5852 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5854 case Match_InvalidOperand: {
5855 SMLoc ErrorLoc = IDLoc;
5856 if (ErrorInfo != ~0ULL) {
5857 if (ErrorInfo >=
Operands.size()) {
5858 return Error(IDLoc,
"too few operands for instruction");
5860 ErrorLoc = ((AMDGPUOperand &)*
Operands[ErrorInfo]).getStartLoc();
5861 if (ErrorLoc == SMLoc())
5865 return Error(ErrorLoc,
"invalid VOPDY instruction");
5867 return Error(ErrorLoc,
"invalid operand for instruction");
5870 case Match_MnemonicFail:
5876bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5881 if (getParser().parseAbsoluteExpression(Tmp)) {
5884 Ret =
static_cast<uint32_t
>(Tmp);
5888bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5889 if (!getSTI().getTargetTriple().isAMDGCN())
5890 return TokError(
"directive only supported for amdgcn architecture");
5892 std::string TargetIDDirective;
5893 SMLoc TargetStart = getTok().getLoc();
5894 if (getParser().parseEscapedString(TargetIDDirective))
5897 std::optional<AMDGPU::TargetID> MaybeParsed =
5900 return getParser().Error(TargetStart,
5901 "malformed target id '" + TargetIDDirective +
"'");
5904 const Triple &
TT = getSTI().getTargetTriple();
5910 return getParser().Error(
5911 TargetStart,
"target id '" + TargetIDDirective +
5912 "' specifies a processor that is not valid for "
5914 TT.getArchName() +
"'");
5917 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5918 getTargetStreamer().getTargetID();
5921 const Triple &STITriple = getSTI().getTargetTriple();
5922 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5923 return getParser().Error(
5924 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5925 " is incompatible with " +
5926 Twine(CurrentTargetID->toString()));
5930 StringRef DirectiveProcessor =
5933 if (DirectiveISA != ISA) {
5934 return getParser().Error(TargetStart,
5935 ".amdgcn_target directive processor " +
5936 Twine(DirectiveProcessor) +
5937 " does not match the specified processor " +
5938 Twine(getSTI().
getCPU()));
5944 CurrentTargetID->getXnackSetting())) {
5946 ".amdgcn_target directive has conflicting xnack settings");
5950 CurrentTargetID->getSramEccSetting())) {
5952 ".amdgcn_target directive has conflicting sramecc settings");
5958 getTargetStreamer().getTargetID()->setXnackSetting(
5960 getTargetStreamer().getTargetID()->setSramEccSetting(
5966bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5970bool AMDGPUAsmParser::calculateGPRBlocks(
5971 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5972 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5973 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5974 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5975 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
5980 const MCExpr *
NumSGPRs = NextFreeSGPR;
5981 int64_t EvaluatedSGPRs;
5983 if (
ISA.Major >= 10)
5988 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
5989 !Features.
test(FeatureSGPRInitBug) &&
5990 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
5991 return OutOfRangeError(SGPRRange);
5993 const MCExpr *ExtraSGPRs =
5997 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
5998 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
5999 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6000 return OutOfRangeError(SGPRRange);
6002 if (Features.
test(FeatureSGPRInitBug))
6009 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6010 unsigned Granule) ->
const MCExpr * {
6014 const MCExpr *AlignToGPR =
6016 const MCExpr *DivGPR =
6022 VGPRBlocks = GetNumGPRBlocks(
6031bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6032 if (!getSTI().getTargetTriple().isAMDGCN())
6033 return TokError(
"directive only supported for amdgcn architecture");
6036 return TokError(
"directive only supported for amdhsa OS");
6038 StringRef KernelName;
6039 if (getParser().parseIdentifier(KernelName))
6046 AMDGPU::MCKernelDescriptor KD =
6056 const MCExpr *NextFreeVGPR = ZeroExpr;
6058 const MCExpr *NamedBarCnt = ZeroExpr;
6063 const MCExpr *NextFreeSGPR = ZeroExpr;
6066 unsigned ImpliedUserSGPRCount = 0;
6070 std::optional<unsigned> ExplicitUserSGPRCount;
6071 const MCExpr *ReserveVCC = OneExpr;
6072 const MCExpr *ReserveFlatScr = OneExpr;
6073 std::optional<bool> EnableWavefrontSize32;
6080 SMRange IDRange = getTok().getLocRange();
6081 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6084 if (ID ==
".end_amdhsa_kernel")
6087 if (!Seen.
insert(ID).second)
6088 return TokError(
".amdhsa_ directives cannot be repeated");
6090 SMLoc ValStart = getLoc();
6091 const MCExpr *ExprVal;
6092 if (getParser().parseExpression(ExprVal))
6094 SMLoc ValEnd = getLoc();
6095 SMRange ValRange = SMRange(ValStart, ValEnd);
6099 bool EvaluatableExpr;
6100 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6102 return OutOfRangeError(ValRange);
6106#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6107 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6108 return OutOfRangeError(RANGE); \
6109 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6114#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6116 return Error(IDRange.Start, "directive should have resolvable expression", \
6119 if (ID ==
".amdhsa_group_segment_fixed_size") {
6122 return OutOfRangeError(ValRange);
6124 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6127 return OutOfRangeError(ValRange);
6129 }
else if (ID ==
".amdhsa_kernarg_size") {
6131 return OutOfRangeError(ValRange);
6133 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6135 ExplicitUserSGPRCount = Val;
6136 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6140 "directive is not supported with architected flat scratch",
6143 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6146 ImpliedUserSGPRCount += 4;
6147 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6150 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6153 return OutOfRangeError(ValRange);
6157 ImpliedUserSGPRCount += Val;
6158 PreloadLength = Val;
6160 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6163 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6166 return OutOfRangeError(ValRange);
6170 PreloadOffset = Val;
6171 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6174 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6177 ImpliedUserSGPRCount += 2;
6178 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6181 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6184 ImpliedUserSGPRCount += 2;
6185 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6188 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6191 ImpliedUserSGPRCount += 2;
6192 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6195 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6198 ImpliedUserSGPRCount += 2;
6199 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6202 "directive is not supported with architected flat scratch",
6206 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6209 ImpliedUserSGPRCount += 2;
6210 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6213 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6216 ImpliedUserSGPRCount += 1;
6217 }
else if (ID ==
".amdhsa_wavefront_size32") {
6220 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6221 EnableWavefrontSize32 = Val;
6223 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6225 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6227 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6229 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6232 "directive is not supported with architected flat scratch",
6235 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6237 }
else if (ID ==
".amdhsa_enable_private_segment") {
6241 "directive is not supported without architected flat scratch",
6244 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6246 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6248 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6250 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6252 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6254 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6256 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6258 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6260 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6262 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6264 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6266 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6267 VGPRRange = ValRange;
6268 NextFreeVGPR = ExprVal;
6269 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6270 SGPRRange = ValRange;
6271 NextFreeSGPR = ExprVal;
6272 }
else if (ID ==
".amdhsa_accum_offset") {
6274 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6275 AccumOffset = ExprVal;
6276 }
else if (ID ==
".amdhsa_named_barrier_count") {
6278 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6279 NamedBarCnt = ExprVal;
6280 }
else if (ID ==
".amdhsa_reserve_vcc") {
6282 return OutOfRangeError(ValRange);
6283 ReserveVCC = ExprVal;
6284 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6286 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6289 "directive is not supported with architected flat scratch",
6292 return OutOfRangeError(ValRange);
6293 ReserveFlatScr = ExprVal;
6294 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6296 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6298 return OutOfRangeError(ValRange);
6299 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6300 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6301 if (Val != XnackOn) {
6302 return getParser().Error(
6304 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6306 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6308 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6310 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6312 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6314 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6316 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6318 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6320 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6322 }
else if (ID ==
".amdhsa_dx10_clamp") {
6323 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6324 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6327 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6329 }
else if (ID ==
".amdhsa_ieee_mode") {
6330 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6331 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6334 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6336 }
else if (ID ==
".amdhsa_fp16_overflow") {
6338 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6340 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6342 }
else if (ID ==
".amdhsa_tg_split") {
6344 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6347 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6350 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6352 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6354 }
else if (ID ==
".amdhsa_memory_ordered") {
6356 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6358 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6360 }
else if (ID ==
".amdhsa_forward_progress") {
6362 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6364 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6366 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6368 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6369 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6371 SharedVGPRCount = Val;
6373 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6375 }
else if (ID ==
".amdhsa_inst_pref_size") {
6377 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6378 if (
ISA.Major == 11) {
6380 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6384 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6387 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6390 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6392 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6394 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6396 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6399 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6401 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6403 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6405 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6407 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6409 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6411 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6413 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6415 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6417 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6419 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6421 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6424 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6427#undef PARSE_BITS_ENTRY
6430 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6431 return TokError(
".amdhsa_next_free_vgpr directive is required");
6433 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6434 return TokError(
".amdhsa_next_free_sgpr directive is required");
6436 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6438 return TokError(
"too many user SGPRs enabled, found " +
6439 Twine(UserSGPRCount) +
", but only " +
6445 if (PreloadLength) {
6451 const MCExpr *VGPRBlocks;
6452 const MCExpr *SGPRBlocks;
6453 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6454 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6455 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6456 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6459 int64_t EvaluatedVGPRBlocks;
6460 bool VGPRBlocksEvaluatable =
6461 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6462 if (VGPRBlocksEvaluatable &&
6464 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6465 return OutOfRangeError(VGPRRange);
6469 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6470 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6472 int64_t EvaluatedSGPRBlocks;
6473 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6475 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6476 return OutOfRangeError(SGPRRange);
6479 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6480 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6482 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6483 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6484 "enabled user SGPRs");
6490 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6491 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6496 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6497 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6502 return TokError(
"Kernarg size should be resolvable");
6504 if (PreloadLength && kernarg_size &&
6505 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6506 return TokError(
"Kernarg preload length + offset is larger than the "
6507 "kernarg segment size");
6510 if (!Seen.
contains(
".amdhsa_accum_offset"))
6511 return TokError(
".amdhsa_accum_offset directive is required");
6512 int64_t EvaluatedAccum;
6513 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6514 uint64_t UEvaluatedAccum = EvaluatedAccum;
6515 if (AccumEvaluatable &&
6516 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6517 return TokError(
"accum_offset should be in range [4..256] in "
6520 int64_t EvaluatedNumVGPR;
6521 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6525 return TokError(
"accum_offset exceeds total VGPR allocation");
6531 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6532 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6538 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6539 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6542 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6544 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6545 return TokError(
"shared_vgpr_count directive not valid on "
6546 "wavefront size 32");
6549 if (VGPRBlocksEvaluatable &&
6550 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6552 return TokError(
"shared_vgpr_count*2 + "
6553 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6558 emitTargetDirective();
6559 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6560 NextFreeVGPR, NextFreeSGPR,
6561 ReserveVCC, ReserveFlatScr);
6565bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6567 if (ParseAsAbsoluteExpression(
Version))
6570 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6571 emitTargetDirective();
6575bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6576 AMDGPUMCKernelCodeT &
C) {
6579 if (ID ==
"max_scratch_backing_memory_byte_size") {
6580 Parser.eatToEndOfStatement();
6584 SmallString<40> ErrStr;
6585 raw_svector_ostream Err(ErrStr);
6586 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6587 return TokError(Err.
str());
6591 if (ID ==
"enable_wavefront_size32") {
6594 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6596 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6599 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6603 if (ID ==
"wavefront_size") {
6604 if (
C.wavefront_size == 5) {
6606 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6608 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6609 }
else if (
C.wavefront_size == 6) {
6611 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6618bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6619 AMDGPUMCKernelCodeT KernelCode;
6629 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6632 if (ID ==
".end_amd_kernel_code_t")
6635 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6640 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6645bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6646 StringRef KernelName;
6647 if (!parseId(KernelName,
"expected symbol name"))
6650 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6657bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6658 if (!getSTI().getTargetTriple().isAMDGCN()) {
6659 return Error(getLoc(),
6660 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6664 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6666 std::optional<AMDGPU::TargetID> MaybeParsed =
6669 return Error(getParser().getTok().getLoc(),
6670 "malformed target id '" + TargetIDDirective +
"'");
6673 const Triple &
TT = getSTI().getTargetTriple();
6679 return Error(getParser().getTok().getLoc(),
6680 "target id '" + TargetIDDirective +
6681 "' specifies a processor that is not valid for subarch '" +
6682 TT.getArchName() +
"'");
6685 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6686 getTargetStreamer().getTargetID();
6689 const Triple &STITriple = getSTI().getTargetTriple();
6690 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6691 return Error(getParser().getTok().getLoc(),
6692 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6693 " is incompatible with " +
6694 Twine(CurrentTargetID->toString()));
6698 StringRef DirectiveProcessor =
6701 if (DirectiveISA != ISA) {
6702 return Error(getParser().getTok().getLoc(),
6703 ".amd_amdgpu_isa directive processor " +
6704 Twine(DirectiveProcessor) +
6705 " does not match the specified processor " +
6706 Twine(getSTI().
getCPU()));
6709 getTargetStreamer().EmitISAVersion();
6715bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6718 std::string HSAMetadataString;
6723 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6724 return Error(getLoc(),
"invalid HSA metadata");
6731bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6732 const char *AssemblerDirectiveEnd,
6733 std::string &CollectString) {
6735 raw_string_ostream CollectStream(CollectString);
6737 getLexer().setSkipSpace(
false);
6739 bool FoundEnd =
false;
6742 CollectStream << getTokenStr();
6746 if (trySkipId(AssemblerDirectiveEnd)) {
6751 CollectStream << Parser.parseStringToEndOfStatement()
6752 <<
getContext().getAsmInfo().getSeparatorString();
6754 Parser.eatToEndOfStatement();
6757 getLexer().setSkipSpace(
true);
6760 return TokError(Twine(
"expected directive ") +
6761 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6768bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6774 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6775 if (!PALMetadata->setFromString(
String))
6776 return Error(getLoc(),
"invalid PAL metadata");
6781bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6784 Twine(
" directive is "
6785 "not available on non-amdpal OSes"))
6789 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6790 PALMetadata->setLegacy();
6793 if (ParseAsAbsoluteExpression(
Key)) {
6794 return TokError(Twine(
"invalid value in ") +
6798 return TokError(Twine(
"expected an even number of values in ") +
6801 if (ParseAsAbsoluteExpression(
Value)) {
6802 return TokError(Twine(
"invalid value in ") +
6805 PALMetadata->setRegister(
Key,
Value);
6814bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6815 if (getParser().checkForValidSection())
6819 SMLoc NameLoc = getLoc();
6820 if (getParser().parseIdentifier(Name))
6821 return TokError(
"expected identifier in directive");
6824 if (getParser().parseComma())
6830 SMLoc SizeLoc = getLoc();
6831 if (getParser().parseAbsoluteExpression(
Size))
6834 return Error(SizeLoc,
"size must be non-negative");
6835 if (
Size > LocalMemorySize)
6836 return Error(SizeLoc,
"size is too large");
6840 SMLoc AlignLoc = getLoc();
6841 if (getParser().parseAbsoluteExpression(Alignment))
6844 return Error(AlignLoc,
"alignment must be a power of two");
6849 if (Alignment >= 1u << 31)
6850 return Error(AlignLoc,
"alignment is too large");
6856 Symbol->redefineIfPossible();
6857 if (!
Symbol->isUndefined())
6858 return Error(NameLoc,
"invalid symbol redefinition");
6860 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6864bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6865 if (getParser().checkForValidSection())
6869 if (getParser().parseIdentifier(FuncName))
6870 return TokError(
"expected symbol name after .amdgpu_info");
6873 AMDGPU::InfoSectionData ParsedInfoData;
6874 AMDGPU::FuncInfo FI;
6876 bool HasScalarAttrs =
false;
6883 SMLoc IDLoc = getLoc();
6884 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6887 if (ID ==
".end_amdgpu_info")
6895 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6897 if (Dir ==
"flags") {
6899 if (getParser().parseAbsoluteExpression(Val))
6902 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6904 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6906 HasScalarAttrs =
true;
6907 }
else if (Dir ==
"num_sgpr") {
6909 if (getParser().parseAbsoluteExpression(Val))
6911 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6912 HasScalarAttrs =
true;
6913 }
else if (Dir ==
"num_vgpr") {
6915 if (getParser().parseAbsoluteExpression(Val))
6918 HasScalarAttrs =
true;
6919 }
else if (Dir ==
"num_agpr") {
6921 if (getParser().parseAbsoluteExpression(Val))
6924 HasScalarAttrs =
true;
6925 }
else if (Dir ==
"private_segment_size") {
6927 if (getParser().parseAbsoluteExpression(Val))
6930 HasScalarAttrs =
true;
6931 }
else if (Dir ==
"use") {
6933 if (getParser().parseIdentifier(ResName))
6934 return TokError(
"expected resource symbol for .amdgpu_use");
6935 ParsedInfoData.
Uses.push_back(
6936 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6937 }
else if (Dir ==
"call") {
6939 if (getParser().parseIdentifier(DstName))
6940 return TokError(
"expected callee symbol for .amdgpu_call");
6941 ParsedInfoData.
Calls.push_back(
6942 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6943 }
else if (Dir ==
"indirect_call") {
6945 if (getParser().parseEscapedString(TypeId))
6946 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6947 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6948 }
else if (Dir ==
"typeid") {
6950 if (getParser().parseEscapedString(TypeId))
6951 return TokError(
"expected type ID string for .amdgpu_typeid");
6952 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6954 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6959 ParsedInfoData.
Funcs.push_back(std::move(FI));
6961 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6962 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6963 Data.Funcs.push_back(std::move(Func));
6964 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6965 Data.Uses.push_back(Use);
6966 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6968 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6971 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6972 Data.TypeIds.push_back(std::move(TypeId));
6977void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
6984void AMDGPUAsmParser::checkKernelPrologues() {
6985 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
6986 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
6987 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
6988 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
6989 if (!AMDHSAKernelSymbols.
contains(Sym))
6991 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
6992 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
6996 "' does not begin with the required prologue "
6997 "sequence: s_mov_b64 followed by v_nop and "
6998 "global_prefetch_b8");
7002 OpcodeStream.
clear();
7003 OpcodeStreamSymbols.clear();
7004 AMDHSAKernelSymbols.
clear();
7007void AMDGPUAsmParser::onEndOfFile() {
7008 emitTargetDirective();
7009 checkKernelPrologues();
7011 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7014bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7015 StringRef IDVal = DirectiveID.
getString();
7018 if (IDVal ==
".amdhsa_kernel")
7019 return ParseDirectiveAMDHSAKernel();
7021 if (IDVal ==
".amdhsa_code_object_version")
7022 return ParseDirectiveAMDHSACodeObjectVersion();
7026 return ParseDirectiveHSAMetadata();
7028 if (IDVal ==
".amd_kernel_code_t")
7029 return ParseDirectiveAMDKernelCodeT();
7031 if (IDVal ==
".amdgpu_hsa_kernel")
7032 return ParseDirectiveAMDGPUHsaKernel();
7034 if (IDVal ==
".amd_amdgpu_isa")
7035 return ParseDirectiveISAVersion();
7039 Twine(
" directive is "
7040 "not available on non-amdhsa OSes"))
7045 if (IDVal ==
".amdgcn_target")
7046 return ParseDirectiveAMDGCNTarget();
7048 if (IDVal ==
".amdgpu_lds")
7049 return ParseDirectiveAMDGPULDS();
7051 if (IDVal ==
".amdgpu_info")
7052 return ParseDirectiveAMDGPUInfo();
7055 return ParseDirectivePALMetadataBegin();
7058 return ParseDirectivePALMetadata();
7063bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7070 return hasSGPR104_SGPR105();
7073 case SRC_SHARED_BASE_LO:
7074 case SRC_SHARED_BASE:
7075 case SRC_SHARED_LIMIT_LO:
7076 case SRC_SHARED_LIMIT:
7078 case SRC_PRIVATE_BASE_LO:
7079 case SRC_PRIVATE_BASE:
7080 case SRC_PRIVATE_LIMIT_LO:
7081 case SRC_PRIVATE_LIMIT:
7083 case SRC_FLAT_SCRATCH_BASE_LO:
7084 case SRC_FLAT_SCRATCH_BASE_HI:
7085 return hasGloballyAddressableScratch();
7086 case SRC_POPS_EXITING_WAVE_ID:
7099 getTargetStreamer().getTargetID()->isXnackSupported();
7129 return hasSGPR102_SGPR103();
7137 ParseStatus Res = parseVOPD(
Operands);
7142 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7154 SMLoc LBraceLoc = getLoc();
7159 auto Loc = getLoc();
7162 Error(Loc,
"expected a register");
7166 RBraceLoc = getLoc();
7171 "expected a comma or a closing square bracket"))
7175 if (
Operands.size() - Prefix > 1) {
7177 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7178 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7187StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7189 setForcedEncodingSize(0);
7190 setForcedDPP(
false);
7191 setForcedSDWA(
false);
7193 if (
Name.consume_back(
"_e64_dpp")) {
7195 setForcedEncodingSize(64);
7198 if (
Name.consume_back(
"_e64")) {
7199 setForcedEncodingSize(64);
7202 if (
Name.consume_back(
"_e32")) {
7203 setForcedEncodingSize(32);
7206 if (
Name.consume_back(
"_dpp")) {
7210 if (
Name.consume_back(
"_sdwa")) {
7211 setForcedSDWA(
true);
7219 unsigned VariantID);
7225 Name = parseMnemonicSuffix(Name);
7231 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7233 bool IsMIMG = Name.starts_with(
"image_");
7236 OperandMode
Mode = OperandMode_Default;
7238 Mode = OperandMode_NSA;
7242 checkUnsupportedInstruction(Name, NameLoc);
7243 if (!Parser.hasPendingError()) {
7246 :
"not a valid operand.";
7266ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7269 if (!trySkipId(Name))
7272 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7276ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7285ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7287 std::function<
bool(int64_t &)> ConvertResult) {
7291 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7295 if (ConvertResult && !ConvertResult(
Value)) {
7296 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7299 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7303ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7305 bool (*ConvertResult)(int64_t &)) {
7314 const unsigned MaxSize = 4;
7318 for (
int I = 0;; ++
I) {
7320 SMLoc Loc = getLoc();
7324 if (
Op != 0 &&
Op != 1)
7325 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7332 if (
I + 1 == MaxSize)
7333 return Error(getLoc(),
"expected a closing square bracket");
7339 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7343ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7345 AMDGPUOperand::ImmTy ImmTy,
7346 bool IgnoreNegative) {
7350 if (trySkipId(Name)) {
7352 }
else if (trySkipId(
"no", Name)) {
7361 return Error(S,
"r128 modifier is not supported on this GPU");
7362 if (Name ==
"a16" && !
hasA16())
7363 return Error(S,
"a16 modifier is not supported on this GPU");
7365 if (Bit == 0 && Name ==
"gds") {
7368 return Error(S,
"nogds is not allowed");
7371 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7372 ImmTy = AMDGPUOperand::ImmTyR128A16;
7374 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7378unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7379 bool &Disabling)
const {
7380 Disabling =
Id.consume_front(
"no");
7383 return StringSwitch<unsigned>(Id)
7390 return StringSwitch<unsigned>(Id)
7400 SMLoc StringLoc = getLoc();
7402 int64_t CPolVal = 0;
7422 ResScope = parseScope(
Operands, Scope);
7435 if (trySkipId(
"nv")) {
7439 }
else if (trySkipId(
"no",
"nv")) {
7446 if (trySkipId(
"scale_offset")) {
7450 }
else if (trySkipId(
"no",
"scale_offset")) {
7463 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7464 AMDGPUOperand::ImmTyCPol));
7469 SMLoc OpLoc = getLoc();
7470 unsigned Enabled = 0, Seen = 0;
7474 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7481 return Error(S,
"dlc modifier is not supported on this GPU");
7484 return Error(S,
"scc modifier is not supported on this GPU");
7487 return Error(S,
"duplicate cache policy modifier");
7499 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7508 ParseStatus Res = parseStringOrIntWithPrefix(
7509 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7523 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7527 if (
Value ==
"TH_DEFAULT")
7529 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7530 Value ==
"TH_LOAD_NT_WB") {
7531 return Error(StringLoc,
"invalid th value");
7532 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7534 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7536 }
else if (
Value.consume_front(
"TH_STORE_")) {
7539 return Error(StringLoc,
"invalid th value");
7542 if (
Value ==
"BYPASS")
7547 TH |= StringSwitch<int64_t>(
Value)
7557 .Default(0xffffffff);
7559 TH |= StringSwitch<int64_t>(
Value)
7570 .Default(0xffffffff);
7573 if (TH == 0xffffffff)
7574 return Error(StringLoc,
"invalid th value");
7581 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7582 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7583 std::optional<unsigned> InsertAt = std::nullopt) {
7584 auto i = OptionalIdx.find(ImmT);
7585 if (i != OptionalIdx.end()) {
7586 unsigned Idx = i->second;
7587 const AMDGPUOperand &
Op =
7588 static_cast<const AMDGPUOperand &
>(*
Operands[Idx]);
7592 Op.addImmOperands(Inst, 1);
7594 if (InsertAt.has_value())
7601ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7607 StringLoc = getLoc();
7612ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7618 SMLoc StringLoc = getLoc();
7622 Value = getTokenStr();
7626 if (
Value == Ids[IntVal])
7631 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7632 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7637ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7639 AMDGPUOperand::ImmTy
Type) {
7643 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7645 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7654bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7657 SMLoc Loc = getLoc();
7659 auto Res = parseIntWithPrefix(Pref, Val);
7665 if (Val < 0 || Val > MaxVal) {
7666 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7675 AMDGPUOperand::ImmTy ImmTy) {
7676 const char *Pref =
"index_key";
7678 SMLoc Loc = getLoc();
7679 auto Res = parseIntWithPrefix(Pref, ImmVal);
7683 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7684 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7685 (ImmVal < 0 || ImmVal > 1))
7686 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7688 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7689 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7691 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7696 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7700 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7704 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7709 AMDGPUOperand::ImmTy
Type) {
7715 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7716 AMDGPUOperand::ImmTyMatrixAFMT);
7720 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7721 AMDGPUOperand::ImmTyMatrixBFMT);
7726 AMDGPUOperand::ImmTy
Type) {
7732 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7733 AMDGPUOperand::ImmTyMatrixAScale);
7737 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7738 AMDGPUOperand::ImmTyMatrixBScale);
7743 AMDGPUOperand::ImmTy
Type) {
7749 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7750 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7754 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7755 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7760ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7761 using namespace llvm::AMDGPU::MTBUFFormat;
7767 for (
int I = 0;
I < 2; ++
I) {
7768 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7771 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7776 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7782 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7785 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7786 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7792ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7793 using namespace llvm::AMDGPU::MTBUFFormat;
7797 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7800 if (Fmt == UFMT_UNDEF)
7807bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7808 StringRef FormatStr, SMLoc Loc) {
7809 using namespace llvm::AMDGPU::MTBUFFormat;
7813 if (
Format != DFMT_UNDEF) {
7819 if (
Format != NFMT_UNDEF) {
7824 Error(Loc,
"unsupported format");
7828ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7831 using namespace llvm::AMDGPU::MTBUFFormat;
7835 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7840 SMLoc Loc = getLoc();
7841 if (!parseId(Str,
"expected a format string") ||
7842 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7844 if (Dfmt == DFMT_UNDEF)
7845 return Error(Loc,
"duplicate numeric format");
7846 if (Nfmt == NFMT_UNDEF)
7847 return Error(Loc,
"duplicate data format");
7850 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7851 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7855 if (Ufmt == UFMT_UNDEF)
7856 return Error(FormatLoc,
"unsupported format");
7865ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7868 using namespace llvm::AMDGPU::MTBUFFormat;
7871 if (Id == UFMT_UNDEF)
7875 return Error(Loc,
"unified format is not supported on this GPU");
7881ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7882 using namespace llvm::AMDGPU::MTBUFFormat;
7883 SMLoc Loc = getLoc();
7888 return Error(Loc,
"out of range format");
7893ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7894 using namespace llvm::AMDGPU::MTBUFFormat;
7900 StringRef FormatStr;
7901 SMLoc Loc = getLoc();
7902 if (!parseId(FormatStr,
"expected a format string"))
7905 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7907 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7917 return parseNumericFormat(
Format);
7921 using namespace llvm::AMDGPU::MTBUFFormat;
7925 SMLoc Loc = getLoc();
7935 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7954 Res = parseSymbolicOrNumericFormat(
Format);
7959 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
7960 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7967 return Error(getLoc(),
"duplicate format");
7973 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
7975 Res = parseIntWithPrefix(
"inst_offset",
Operands,
7976 AMDGPUOperand::ImmTyInstOffset);
7983 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
7985 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
7991 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
7994 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8004 OptionalImmIndexMap OptionalIdx;
8006 unsigned OperandIdx[4];
8007 unsigned EnMask = 0;
8010 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8011 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8016 OperandIdx[SrcIdx] = Inst.
size();
8017 Op.addRegOperands(Inst, 1);
8024 OperandIdx[SrcIdx] = Inst.
size();
8030 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8031 Op.addImmOperands(Inst, 1);
8035 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8039 OptionalIdx[
Op.getImmTy()] = i;
8045 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8052 for (
auto i = 0; i < SrcIdx; ++i) {
8054 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8060 AMDGPUOperand::ImmTyExpCompr);
8070 int64_t CntVal,
bool Saturate,
8076 IntVal =
encode(ISA, IntVal, CntVal);
8077 if (CntVal !=
decode(ISA, IntVal)) {
8079 IntVal =
encode(ISA, IntVal, -1);
8087bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8089 SMLoc CntLoc = getLoc();
8090 StringRef CntName = getTokenStr();
8097 SMLoc ValLoc = getLoc();
8104 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8106 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8108 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8111 Error(CntLoc,
"invalid counter name " + CntName);
8116 Error(ValLoc,
"too large value for " + CntName);
8125 Error(getLoc(),
"expected a counter name");
8139 if (!parseCnt(Waitcnt))
8147 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8151bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8152 SMLoc FieldLoc = getLoc();
8153 StringRef FieldName = getTokenStr();
8158 SMLoc ValueLoc = getLoc();
8165 if (FieldName ==
"instid0") {
8167 }
else if (FieldName ==
"instskip") {
8169 }
else if (FieldName ==
"instid1") {
8172 Error(FieldLoc,
"invalid field name " + FieldName);
8191 .Case(
"VALU_DEP_1", 1)
8192 .Case(
"VALU_DEP_2", 2)
8193 .Case(
"VALU_DEP_3", 3)
8194 .Case(
"VALU_DEP_4", 4)
8195 .Case(
"TRANS32_DEP_1", 5)
8196 .Case(
"TRANS32_DEP_2", 6)
8197 .Case(
"TRANS32_DEP_3", 7)
8198 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8199 .Case(
"SALU_CYCLE_1", 9)
8200 .Case(
"SALU_CYCLE_2", 10)
8201 .Case(
"SALU_CYCLE_3", 11)
8209 Delay |=
Value << Shift;
8219 if (!parseDelay(Delay))
8227 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8231bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8233bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8239void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8240 StringRef DepCtrName) {
8243 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8246 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8249 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8252 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8259bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8261 using namespace llvm::AMDGPU::DepCtr;
8263 SMLoc DepCtrLoc = getLoc();
8264 StringRef DepCtrName = getTokenStr();
8274 unsigned PrevOprMask = UsedOprMask;
8275 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8278 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8287 Error(getLoc(),
"expected a counter name");
8292 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8293 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8298 using namespace llvm::AMDGPU::DepCtr;
8301 SMLoc Loc = getLoc();
8304 unsigned UsedOprMask = 0;
8306 if (!parseDepCtr(DepCtr, UsedOprMask))
8314 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8318bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8324ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8326 OperandInfoTy &Width) {
8327 using namespace llvm::AMDGPU::Hwreg;
8333 HwReg.Loc = getLoc();
8336 HwReg.IsSymbolic =
true;
8338 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8346 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8356 Width.Loc = getLoc();
8365 using namespace llvm::AMDGPU::Hwreg;
8368 SMLoc Loc = getLoc();
8370 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8372 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8373 HwregOffset::Default);
8374 struct : StructuredOpField {
8375 using StructuredOpField::StructuredOpField;
8376 bool validate(AMDGPUAsmParser &Parser)
const override {
8378 return Error(Parser,
"only values from 1 to 32 are legal");
8381 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8382 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8385 Res = parseHwregFunc(HwReg,
Offset, Width);
8388 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8390 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8394 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8401 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8403 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8407bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8413bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8414 OperandInfoTy &Stream) {
8415 using namespace llvm::AMDGPU::SendMsg;
8420 Msg.IsSymbolic =
true;
8427 Op.IsDefined =
true;
8433 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8438 Stream.IsDefined =
true;
8439 Stream.Loc = getLoc();
8448bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8449 const OperandInfoTy &
Op,
8450 const OperandInfoTy &Stream) {
8451 using namespace llvm::AMDGPU::SendMsg;
8460 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8465 Error(
Msg.Loc,
"invalid message id");
8471 Error(
Op.Loc,
"message does not support operations");
8473 Error(
Msg.Loc,
"missing message operation");
8479 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8481 Error(
Op.Loc,
"invalid operation id");
8486 Error(Stream.Loc,
"message operation does not support streams");
8490 Error(Stream.Loc,
"invalid message stream id");
8497 using namespace llvm::AMDGPU::SendMsg;
8500 SMLoc Loc = getLoc();
8504 OperandInfoTy
Op(OP_NONE_);
8505 OperandInfoTy Stream(STREAM_ID_NONE_);
8506 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8511 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8513 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8519 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8523bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8526 using namespace llvm::AMDGPU::WaitEvent;
8528 SMLoc Loc = getLoc();
8531 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8533 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8535 StructuredOpField *TargetBitfield =
8536 isGFX11() ? &DontWaitExportReady : &ExportReady;
8538 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8542 if (!validateStructuredOpFields({TargetBitfield}))
8544 ImmVal = TargetBitfield->Val;
8551 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8553 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8554 AMDGPUOperand::ImmTyWaitEvent));
8558bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8571 int Slot = StringSwitch<int>(Str)
8578 return Error(S,
"invalid interpolation slot");
8581 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8592 if (!Str.starts_with(
"attr"))
8593 return Error(S,
"invalid interpolation attribute");
8595 StringRef Chan = Str.take_back(2);
8596 int AttrChan = StringSwitch<int>(Chan)
8603 return Error(S,
"invalid or missing interpolation attribute channel");
8605 Str = Str.drop_back(2).drop_front(4);
8608 if (Str.getAsInteger(10, Attr))
8609 return Error(S,
"invalid or missing interpolation attribute number");
8612 return Error(S,
"out of bounds interpolation attribute number");
8617 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8618 Operands.push_back(AMDGPUOperand::CreateImm(
8619 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8628 using namespace llvm::AMDGPU::Exp;
8638 return Error(S, (Id == ET_INVALID)
8639 ?
"invalid exp target"
8640 :
"exp target is not supported on this GPU");
8643 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8651bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8655bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8660 return getTokenKind() ==
Kind;
8663StringRef AMDGPUAsmParser::getId()
const {
8667bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8675bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8677 StringRef Tok = getTokenStr();
8686bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8688 if (isId(Id) && peekToken().is(Kind)) {
8697 if (isToken(Kind)) {
8705 const StringRef ErrMsg) {
8706 if (!trySkipToken(Kind)) {
8707 Error(getLoc(), ErrMsg);
8713bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8717 if (Parser.parseExpression(Expr))
8720 if (Expr->evaluateAsAbsolute(
Imm))
8723 if (Expected.empty()) {
8724 Error(S,
"expected absolute expression");
8727 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8736 if (Parser.parseExpression(Expr))
8740 if (Expr->evaluateAsAbsolute(IntVal)) {
8741 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8743 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8748bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8750 Val =
getToken().getStringContents();
8754 Error(getLoc(), ErrMsg);
8758bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8760 Val = getTokenStr();
8764 if (!ErrMsg.
empty())
8765 Error(getLoc(), ErrMsg);
8769AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8771AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8774 : getLexer().peekTok(ShouldSkipSpace);
8778 auto TokCount = getLexer().peekTokens(Tokens);
8780 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8785 return getLexer().getKind();
8788SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8790StringRef AMDGPUAsmParser::getTokenStr()
const {
8794void AMDGPUAsmParser::lex() { Parser.Lex(); }
8796const AMDGPUOperand &
8798 int MCOpIdx)
const {
8800 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8801 if (TargetOp.getMCOpIdx() == MCOpIdx)
8808 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8812SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8817 int MCOpIdx)
const {
8818 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8821SMLoc AMDGPUAsmParser::getOperandLoc(
8822 std::function<
bool(
const AMDGPUOperand &)>
Test,
8824 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8825 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8827 return Op.getStartLoc();
8832SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8834 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8849 StringRef
Id = getTokenStr();
8850 SMLoc IdLoc = getLoc();
8856 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8857 if (
I == Fields.
end())
8858 return Error(IdLoc,
"unknown field");
8859 if ((*I)->IsDefined)
8860 return Error(IdLoc,
"duplicate field");
8863 (*I)->Loc = getLoc();
8866 (*I)->IsDefined =
true;
8873bool AMDGPUAsmParser::validateStructuredOpFields(
8875 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8876 return F->validate(*
this);
8886 const unsigned XorMask) {
8893bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8894 const unsigned MaxVal,
8895 const Twine &ErrMsg, SMLoc &Loc) {
8911bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8912 const unsigned MinVal,
8913 const unsigned MaxVal,
8914 const StringRef ErrMsg) {
8916 for (
unsigned i = 0; i < OpNum; ++i) {
8917 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8924bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8925 using namespace llvm::AMDGPU::Swizzle;
8928 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8929 "expected a 2-bit lane id")) {
8939bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8940 using namespace llvm::AMDGPU::Swizzle;
8946 if (!parseSwizzleOperand(GroupSize, 2, 32,
8947 "group size must be in the interval [2,32]", Loc)) {
8951 Error(Loc,
"group size must be a power of two");
8954 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8955 "lane id must be in the interval [0,group size - 1]",
8963bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8964 using namespace llvm::AMDGPU::Swizzle;
8969 if (!parseSwizzleOperand(GroupSize, 2, 32,
8970 "group size must be in the interval [2,32]", Loc)) {
8974 Error(Loc,
"group size must be a power of two");
8982bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
8983 using namespace llvm::AMDGPU::Swizzle;
8988 if (!parseSwizzleOperand(GroupSize, 1, 16,
8989 "group size must be in the interval [1,16]", Loc)) {
8993 Error(Loc,
"group size must be a power of two");
9001bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9002 using namespace llvm::AMDGPU::Swizzle;
9009 SMLoc StrLoc = getLoc();
9010 if (!parseString(Ctl)) {
9013 if (Ctl.
size() != BITMASK_WIDTH) {
9014 Error(StrLoc,
"expected a 5-character mask");
9018 unsigned AndMask = 0;
9019 unsigned OrMask = 0;
9020 unsigned XorMask = 0;
9022 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9026 Error(StrLoc,
"invalid mask");
9047bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9048 using namespace llvm::AMDGPU::Swizzle;
9051 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9057 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9058 "FFT swizzle must be in the interval [0," +
9059 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9067bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9068 using namespace llvm::AMDGPU::Swizzle;
9071 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9078 if (!parseSwizzleOperand(
Direction, 0, 1,
9079 "direction must be 0 (left) or 1 (right)", Loc))
9083 if (!parseSwizzleOperand(
9084 RotateSize, 0, ROTATE_MAX_SIZE,
9085 "number of threads to rotate must be in the interval [0," +
9086 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9091 (RotateSize << ROTATE_SIZE_SHIFT);
9095bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9097 SMLoc OffsetLoc = getLoc();
9103 Error(OffsetLoc,
"expected a 16-bit offset");
9109bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9110 using namespace llvm::AMDGPU::Swizzle;
9114 SMLoc ModeLoc = getLoc();
9117 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9118 Ok = parseSwizzleQuadPerm(
Imm);
9119 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9120 Ok = parseSwizzleBitmaskPerm(
Imm);
9121 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9122 Ok = parseSwizzleBroadcast(
Imm);
9123 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9124 Ok = parseSwizzleSwap(
Imm);
9125 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9126 Ok = parseSwizzleReverse(
Imm);
9127 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9128 Ok = parseSwizzleFFT(
Imm);
9129 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9130 Ok = parseSwizzleRotate(
Imm);
9132 Error(ModeLoc,
"expected a swizzle mode");
9135 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9145 if (trySkipId(
"offset")) {
9149 if (trySkipId(
"swizzle")) {
9150 Ok = parseSwizzleMacro(
Imm);
9152 Ok = parseSwizzleOffset(
Imm);
9157 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9164bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9170int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9172 using namespace llvm::AMDGPU::VGPRIndexMode;
9184 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9185 if (trySkipId(IdSymbolic[ModeId])) {
9193 ?
"expected a VGPR index mode or a closing parenthesis"
9194 :
"expected a VGPR index mode");
9199 Error(S,
"duplicate VGPR index mode");
9207 "expected a comma or a closing parenthesis"))
9216 using namespace llvm::AMDGPU::VGPRIndexMode;
9222 Imm = parseGPRIdxMacro();
9226 if (getParser().parseAbsoluteExpression(
Imm))
9229 return Error(S,
"invalid immediate: only 4-bit values are legal");
9233 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9237bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9248 if (isRegister() || isModifier())
9255 assert(Opr.isImm() || Opr.isExpr());
9256 SMLoc Loc = Opr.getStartLoc();
9260 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9261 Error(Loc,
"expected an absolute expression or a label");
9262 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9263 Error(Loc,
"expected a 16-bit signed jump offset");
9283 OptionalImmIndexMap OptionalIdx;
9284 unsigned FirstOperandIdx = 1;
9285 bool IsAtomicReturn =
false;
9291 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9292 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9296 Op.addRegOperands(Inst, 1);
9300 if (IsAtomicReturn && i == FirstOperandIdx)
9301 Op.addRegOperands(Inst, 1);
9306 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9307 Op.addImmOperands(Inst, 1);
9319 OptionalIdx[
Op.getImmTy()] = i;
9323 AMDGPUOperand::ImmTyOffset);
9335bool AMDGPUOperand::isSMRDOffset8()
const {
9339bool AMDGPUOperand::isSMEMOffset()
const {
9341 return isImmLiteral();
9344bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9379bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9380 if (BoundCtrl == 0 || BoundCtrl == 1) {
9388void AMDGPUAsmParser::onBeginOfFile() {
9389 if (!getParser().getStreamer().getTargetStreamer())
9392 if (!getTargetStreamer().getTargetID())
9393 getTargetStreamer().initializeTargetID(getSTI(),
9397void AMDGPUAsmParser::emitTargetDirective() {
9398 if (TargetDirectiveEmitted)
9400 TargetDirectiveEmitted =
true;
9402 if (!getParser().getStreamer().getTargetStreamer() ||
9407 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9416bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9420 StringRef TokenId = getTokenStr();
9421 AGVK VK = StringSwitch<AGVK>(TokenId)
9422 .Case(
"max", AGVK::AGVK_Max)
9423 .Case(
"min", AGVK::AGVK_Min)
9424 .Case(
"or", AGVK::AGVK_Or)
9425 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9426 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9427 .Case(
"alignto", AGVK::AGVK_AlignTo)
9428 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9429 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9430 .Default(AGVK::AGVK_None);
9439 if (Exprs.
empty()) {
9441 "empty " + Twine(TokenId) +
" expression");
9444 if (CommaCount + 1 != Exprs.
size()) {
9446 "mismatch of commas in " + Twine(TokenId) +
" expression");
9450 Expected && Exprs.
size() != Expected) {
9451 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9452 Twine(Expected) +
" operands");
9459 if (getParser().parseExpression(Expr, EndLoc))
9463 if (LastTokenWasComma)
9467 "unexpected token in " + Twine(TokenId) +
" expression");
9473 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9477 StringRef
Name = getTokenStr();
9478 if (Name ==
"mul") {
9479 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9483 if (Name ==
"div") {
9484 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9495 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9500 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9501 AMDGPU::OpName::src2};
9509 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9514 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9516 if (
DstOp.isReg() &&
9521 if ((OpSel & (1 << SrcNum)) != 0)
9527void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9534 OptionalImmIndexMap &OptionalIdx) {
9535 cvtVOP3P(Inst,
Operands, OptionalIdx);
9544 &&
Desc.NumOperands > (OpNum + 1)
9546 &&
Desc.operands()[OpNum + 1].RegClass != -1
9548 &&
Desc.getOperandConstraint(OpNum + 1,
9552void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9554 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9555 AMDGPU::OpName::src2};
9556 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9557 AMDGPU::OpName::src1_modifiers,
9558 AMDGPU::OpName::src2_modifiers};
9559 for (
int J = 0; J < 3; ++J) {
9560 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9566 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9569 if ((OpSel & (1 << J)) != 0)
9572 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9579void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9581 OptionalImmIndexMap OptionalIdx;
9586 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9587 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9591 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9593 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9594 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9595 Op.isInterpAttrChan()) {
9597 }
else if (
Op.isImmModifier()) {
9598 OptionalIdx[
Op.getImmTy()] =
I;
9606 AMDGPUOperand::ImmTyHigh);
9610 AMDGPUOperand::ImmTyClamp);
9614 AMDGPUOperand::ImmTyOModSI);
9619 AMDGPUOperand::ImmTyOpSel);
9620 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9623 cvtOpSelHelper(Inst, OpSel);
9628 OptionalImmIndexMap OptionalIdx;
9633 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9634 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9638 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9640 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9641 }
else if (
Op.isImmModifier()) {
9642 OptionalIdx[
Op.getImmTy()] =
I;
9650 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9653 AMDGPUOperand::ImmTyOpSel);
9656 AMDGPUOperand::ImmTyWaitEXP);
9662 cvtOpSelHelper(Inst, OpSel);
9665void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9667 OptionalImmIndexMap OptionalIdx;
9670 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9674 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9675 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9678 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9683 if (NumOperands == CbszOpIdx) {
9688 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9689 }
else if (
Op.isImmModifier()) {
9690 OptionalIdx[
Op.getImmTy()] =
I;
9692 Op.addRegOrImmOperands(Inst, 1);
9697 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9698 if (CbszIdx != OptionalIdx.end()) {
9699 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9703 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9704 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9705 if (BlgpIdx != OptionalIdx.end()) {
9706 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9717 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9718 if (OpselIdx != OptionalIdx.end()) {
9719 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9723 unsigned OpSelHi = 0;
9724 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9725 if (OpselHiIdx != OptionalIdx.end()) {
9726 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9729 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9730 AMDGPU::OpName::src1_modifiers};
9732 for (
unsigned J = 0; J < 2; ++J) {
9733 unsigned ModVal = 0;
9734 if (OpSel & (1 << J))
9736 if (OpSelHi & (1 << J))
9739 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9745 OptionalImmIndexMap &OptionalIdx) {
9750 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9751 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9755 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9757 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9758 }
else if (
Op.isImmModifier()) {
9759 OptionalIdx[
Op.getImmTy()] =
I;
9761 Op.addRegOrImmOperands(Inst, 1);
9767 AMDGPUOperand::ImmTyScaleSel);
9771 AMDGPUOperand::ImmTyClamp);
9777 AMDGPUOperand::ImmTyByteSel);
9782 AMDGPUOperand::ImmTyOModSI);
9789 auto *it = Inst.
begin();
9791 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9800 OptionalImmIndexMap OptionalIdx;
9801 cvtVOP3(Inst,
Operands, OptionalIdx);
9805 OptionalImmIndexMap &OptIdx) {
9810 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9811 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9812 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9813 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9814 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9815 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9816 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9817 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9818 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9819 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9828 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9829 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9832 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9833 if (BitOp3Idx != -1) {
9840 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9841 if (OpSelIdx != -1) {
9845 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9846 if (OpSelHiIdx != -1) {
9847 int DefaultVal =
IsPacked ? -1 : 0;
9853 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9854 if (MatrixAFMTIdx != -1) {
9856 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9860 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9861 if (MatrixBFMTIdx != -1) {
9863 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9866 int MatrixAScaleIdx =
9867 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9868 if (MatrixAScaleIdx != -1) {
9870 AMDGPUOperand::ImmTyMatrixAScale, 0);
9873 int MatrixBScaleIdx =
9874 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9875 if (MatrixBScaleIdx != -1) {
9877 AMDGPUOperand::ImmTyMatrixBScale, 0);
9880 int MatrixAScaleFmtIdx =
9881 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9882 if (MatrixAScaleFmtIdx != -1) {
9884 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9887 int MatrixBScaleFmtIdx =
9888 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9889 if (MatrixBScaleFmtIdx != -1) {
9891 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9896 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9900 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9902 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9906 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9910 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9911 AMDGPU::OpName::src2};
9912 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9913 AMDGPU::OpName::src1_modifiers,
9914 AMDGPU::OpName::src2_modifiers};
9917 unsigned OpSelHi = 0;
9924 if (OpSelHiIdx != -1)
9933 for (
int J = 0; J < 3; ++J) {
9934 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9938 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9948 uint32_t ModVal = 0;
9950 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
9951 if (SrcOp.
isReg() && getMRI()
9958 if ((OpSel & (1 << J)) != 0)
9962 if ((OpSelHi & (1 << J)) != 0)
9965 if ((NegLo & (1 << J)) != 0)
9968 if ((NegHi & (1 << J)) != 0)
9976 OptionalImmIndexMap OptIdx;
9982 unsigned i,
unsigned Opc,
9984 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
9985 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
9987 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
9993 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9996 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9997 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
9999 OptionalImmIndexMap OptIdx;
10000 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10001 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10002 OptIdx[
Op.getImmTy()] = i;
10007 AMDGPUOperand::ImmTyIndexKey8bit);
10011 AMDGPUOperand::ImmTyIndexKey16bit);
10015 AMDGPUOperand::ImmTyIndexKey32bit);
10032 SMLoc S = getLoc();
10035 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10036 SMLoc OpYLoc = getLoc();
10039 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10042 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10051 auto addOp = [&](uint16_t ParsedOprIdx) {
10052 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10054 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10058 Op.addRegOperands(Inst, 1);
10062 Op.addImmOperands(Inst, 1);
10074 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10078 const auto &CInfo = InstInfo[CompIdx];
10079 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10080 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10081 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10082 if (CInfo.hasSrc2Acc())
10083 addOp(CInfo.getIndexOfDstInParsedOperands());
10087 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10088 if (BitOp3Idx != -1) {
10089 OptionalImmIndexMap OptIdx;
10090 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10092 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10102bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10104bool AMDGPUOperand::isDPPCtrl()
const {
10105 using namespace AMDGPU::DPP;
10107 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10110 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10111 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10112 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10113 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10114 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10115 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10116 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10117 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10118 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10119 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10120 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10121 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10130bool AMDGPUOperand::isBLGP()
const {
10134bool AMDGPUOperand::isS16Imm()
const {
10138bool AMDGPUOperand::isU16Imm()
const {
10146bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10151 SMLoc Loc =
getToken().getEndLoc();
10152 Token = std::string(getTokenStr());
10154 if (getLoc() != Loc)
10159 if (!parseId(Suffix))
10163 StringRef DimId = Token;
10178 SMLoc S = getLoc();
10184 SMLoc Loc = getLoc();
10185 if (!parseDimId(Encoding))
10186 return Error(Loc,
"invalid dim value");
10189 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10198 SMLoc S = getLoc();
10207 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10210 for (
size_t i = 0; i < 8; ++i) {
10214 SMLoc Loc = getLoc();
10215 if (getParser().parseAbsoluteExpression(Sels[i]))
10217 if (0 > Sels[i] || 7 < Sels[i])
10218 return Error(Loc,
"expected a 3-bit value");
10221 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10225 for (
size_t i = 0; i < 8; ++i)
10226 DPP8 |= (Sels[i] << (i * 3));
10229 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10233bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10235 if (Ctrl ==
"row_newbcast")
10238 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10241 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10242 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10245 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10246 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10250int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10253 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10257 for (
int i = 0; i < 4; ++i) {
10262 SMLoc Loc = getLoc();
10263 if (getParser().parseAbsoluteExpression(Temp))
10265 if (Temp < 0 || Temp > 3) {
10266 Error(Loc,
"expected a 2-bit value");
10270 Val += (Temp << i * 2);
10273 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10279int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10280 using namespace AMDGPU::DPP;
10285 SMLoc Loc = getLoc();
10287 if (getParser().parseAbsoluteExpression(Val))
10290 struct DppCtrlCheck {
10296 DppCtrlCheck
Check =
10297 StringSwitch<DppCtrlCheck>(Ctrl)
10298 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10299 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10300 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10301 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10302 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10303 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10304 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10305 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10306 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10307 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10311 if (
Check.Ctrl == -1) {
10312 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10320 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10328 using namespace AMDGPU::DPP;
10331 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10334 SMLoc S = getLoc();
10340 if (Ctrl ==
"row_mirror") {
10341 Val = DppCtrl::ROW_MIRROR;
10342 }
else if (Ctrl ==
"row_half_mirror") {
10343 Val = DppCtrl::ROW_HALF_MIRROR;
10346 if (Ctrl ==
"quad_perm") {
10347 Val = parseDPPCtrlPerm();
10349 Val = parseDPPCtrlSel(Ctrl);
10358 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10364 OptionalImmIndexMap OptionalIdx;
10371 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10373 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10374 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10378 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10379 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10383 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10384 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10385 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10386 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10387 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10388 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10389 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10390 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10391 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10397 if (OldIdx == NumOperands) {
10399 constexpr int DST_IDX = 0;
10401 }
else if (Src2ModIdx == NumOperands) {
10411 if (IsVOP3CvtSrDpp) {
10420 if (TiedTo != -1) {
10425 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10427 if (IsDPP8 &&
Op.isDppFI()) {
10430 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10431 }
else if (
Op.isReg()) {
10432 Op.addRegOperands(Inst, 1);
10433 }
else if (
Op.isImm() &&
10435 Op.addImmOperands(Inst, 1);
10436 }
else if (
Op.isImm()) {
10437 OptionalIdx[
Op.getImmTy()] =
I;
10445 AMDGPUOperand::ImmTyClamp);
10451 AMDGPUOperand::ImmTyByteSel);
10456 AMDGPUOperand::ImmTyOModSI);
10459 cvtVOP3P(Inst,
Operands, OptionalIdx);
10461 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10464 AMDGPUOperand::ImmTyOpSel);
10469 AMDGPUOperand::ImmTyDPP8);
10470 using namespace llvm::AMDGPU::DPP;
10474 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10476 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10478 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10480 AMDGPUOperand::ImmTyDppBoundCtrl);
10484 AMDGPUOperand::ImmTyDppFI);
10490 OptionalImmIndexMap OptionalIdx;
10494 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10495 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10502 if (TiedTo != -1) {
10507 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10509 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10517 Op.addImmOperands(Inst, 1);
10519 Op.addRegWithFPInputModsOperands(Inst, 2);
10520 }
else if (
Op.isDppFI()) {
10522 }
else if (
Op.isReg()) {
10523 Op.addRegOperands(Inst, 1);
10529 Op.addRegWithFPInputModsOperands(Inst, 2);
10530 }
else if (
Op.isReg()) {
10531 Op.addRegOperands(Inst, 1);
10532 }
else if (
Op.isDPPCtrl()) {
10533 Op.addImmOperands(Inst, 1);
10534 }
else if (
Op.isImm()) {
10536 OptionalIdx[
Op.getImmTy()] =
I;
10544 using namespace llvm::AMDGPU::DPP;
10548 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10550 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10552 AMDGPUOperand::ImmTyDppBoundCtrl);
10555 AMDGPUOperand::ImmTyDppFI);
10566 AMDGPUOperand::ImmTy
Type) {
10567 return parseStringOrIntWithPrefix(
10569 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10574 return parseStringOrIntWithPrefix(
10575 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10576 AMDGPUOperand::ImmTySDWADstUnused);
10580 cvtSDWA(Inst,
Operands, SDWAInstType::VOP1);
10584 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2);
10587void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10589 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
true,
true);
10592void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10594 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
false,
true);
10602 SDWAInstType BasicInstType,
bool SkipDstVcc,
10604 using namespace llvm::AMDGPU::SDWA;
10606 OptionalImmIndexMap OptionalIdx;
10607 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10608 bool SkippedVcc =
false;
10612 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10613 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10617 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10618 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10619 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10625 if (BasicInstType == SDWAInstType::VOP2 &&
10631 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10637 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10638 }
else if (
Op.isImm()) {
10640 OptionalIdx[
Op.getImmTy()] =
I;
10644 SkippedVcc =
false;
10648 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10649 Opc != AMDGPU::V_NOP_sdwa_vi) {
10651 switch (BasicInstType) {
10652 case SDWAInstType::VOP1:
10655 AMDGPUOperand::ImmTyClamp, 0);
10659 AMDGPUOperand::ImmTyOModSI, 0);
10663 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10667 AMDGPUOperand::ImmTySDWADstUnused,
10668 DstUnused::UNUSED_PRESERVE);
10671 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10674 case SDWAInstType::VOP2:
10676 AMDGPUOperand::ImmTyClamp, 0);
10680 AMDGPUOperand::ImmTyOModSI, 0);
10683 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10685 AMDGPUOperand::ImmTySDWADstUnused,
10686 DstUnused::UNUSED_PRESERVE);
10688 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10690 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10693 case SDWAInstType::VOPC:
10696 AMDGPUOperand::ImmTyClamp, 0);
10698 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10700 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10707 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10708 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10709 auto *it = Inst.
begin();
10711 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10724#define GET_MATCHER_IMPLEMENTATION
10725#define GET_MNEMONIC_SPELL_CHECKER
10726#define GET_MNEMONIC_CHECKER
10727#include "AMDGPUGenAsmMatcher.inc"
10733 return parseTokenOp(
"addr64",
Operands);
10735 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10737 return parseTokenOp(
"idxen",
Operands);
10739 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10742 return parseTokenOp(
"offen",
Operands);
10744 return parseTokenOp(
"off",
Operands);
10745 case MCK_row_95_en:
10746 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10748 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10750 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10752 return tryCustomParseOperand(
Operands, MCK);
10757unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10763 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10766 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10768 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10770 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10772 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10774 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10776 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10778 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10779 case MCK_row_95_en:
10780 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10788 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10790 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10791 case MCK_SOPPBrTarget:
10792 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10793 case MCK_VReg32OrOff:
10794 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10795 case MCK_InterpSlot:
10796 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10797 case MCK_InterpAttr:
10798 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10799 case MCK_InterpAttrChan:
10800 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10802 case MCK_SReg_64_XEXEC:
10812 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10814 return Match_InvalidOperand;
10823 SMLoc S = getLoc();
10832 return Error(S,
"expected a 16-bit value");
10835 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10839bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10845bool AMDGPUOperand::isSplitBarrier()
const {
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
bool regsOverlap(MCRegister RegA, MCRegister RegB) const
Returns true if the two registers are equal or alias each other.
const MCRegisterClass & getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi)
bool isGFX10_BEncoding(const MCSubtargetInfo &STI)
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR, int32_t ArgNumVGPR)
bool isGFX10(const MCSubtargetInfo &STI)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
bool isGFX13(const MCSubtargetInfo &STI)
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, OpName NamedIdx)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
GPUKind
GPU kinds supported by the AMDGPU target.
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool isGFX13Plus(const MCSubtargetInfo &STI)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isGFX1250(const MCSubtargetInfo &STI)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
EnumSet< Modifier, Modifier_enumSize > Modifiers
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
@ Default
The result value is uniform if and only if all operands are uniform.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size
1.14.0