18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsPowerPC.h"
26#include "llvm/IR/IntrinsicsR600.h"
27#include "llvm/IR/IntrinsicsRISCV.h"
28#include "llvm/IR/IntrinsicsS390.h"
29#include "llvm/IR/IntrinsicsSPIRV.h"
30#include "llvm/IR/IntrinsicsVE.h"
31#include "llvm/IR/IntrinsicsX86.h"
32#include "llvm/IR/IntrinsicsXCore.h"
44 unsigned NumArgs,
bool IsVarArg,
47 unsigned NumMissingTrailingParams = 0);
50#define GET_INTRINSIC_NAME_TABLE
51#include "llvm/IR/IntrinsicImpl.inc"
54#define GET_INTRINSIC_TARGET_FEATURES_TABLE
55#include "llvm/IR/IntrinsicImpl.inc"
58 assert(
id < num_intrinsics &&
"Invalid intrinsic ID!");
59 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
63 assert(
id < num_intrinsics &&
"invalid intrinsic ID!");
64 return IntrinsicTargetFeaturesTable[IntrinsicTargetFeaturesOffsetTable[id]];
68 assert(
id < num_intrinsics &&
"Invalid intrinsic ID!");
70 "This version of getName does not support overloading");
89 Result +=
"p" +
utostr(PTyp->getAddressSpace());
91 Result +=
"a" +
utostr(ATyp->getNumElements()) +
94 if (!STyp->isLiteral()) {
97 Result += STyp->getName();
99 HasUnnamedType =
true;
102 for (
auto *Elem : STyp->elements())
109 for (
size_t i = 0; i < FT->getNumParams(); i++)
119 Result +=
"v" +
utostr(EC.getKnownMinValue()) +
123 Result += TETy->getName();
124 for (
Type *ParamTy : TETy->type_params())
126 for (
unsigned IntParam : TETy->int_params())
127 Result +=
"_" +
utostr(IntParam);
131 switch (Ty->getTypeID()) {
138 Result +=
"Metadata";
178 bool EarlyModuleCheck) {
180 assert(Id < Intrinsic::num_intrinsics &&
"Invalid intrinsic ID!");
182 "This version of getName is for overloaded intrinsics only");
183 (void)EarlyModuleCheck;
184 assert((!EarlyModuleCheck || M ||
186 "Intrinsic overloading on pointer types need to provide a Module");
187 bool HasUnnamedType =
false;
189 for (
Type *Ty : OverloadTys)
191 if (HasUnnamedType) {
192 assert(M &&
"unnamed types need a module");
197 "Provided FunctionType must match arguments");
198 return M->getUniqueIntrinsicName(Result, Id, FT);
205 assert(M &&
"We need to have a Module");
219#define GET_INTRINSIC_IITINFO
220#include "llvm/IR/IntrinsicImpl.inc"
223static_assert(IIT_Done == 0,
"IIT_Done expected to be 0");
230 auto IsScalableVector = [&]() {
232 if (NextInfo != IIT_SCALABLE_VEC)
243 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Void, 0));
246 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::VarArg, 0));
249 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::MMX, 0));
252 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::AMX, 0));
255 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Token, 0));
258 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Metadata, 0));
261 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Half, 0));
264 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::BFloat, 0));
267 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Float, 0));
270 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Double, 0));
273 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Quad, 0));
276 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::PPCQuad, 0));
279 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 1));
282 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 2));
285 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 4));
287 case IIT_AARCH64_SVCOUNT:
288 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::AArch64Svcount, 0));
291 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 8));
294 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 16));
297 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 32));
300 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 64));
303 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Integer, 128));
306 OutputTable.
push_back(IITDescriptor::getVector(1, IsScalableVector()));
310 OutputTable.
push_back(IITDescriptor::getVector(2, IsScalableVector()));
314 OutputTable.
push_back(IITDescriptor::getVector(3, IsScalableVector()));
318 OutputTable.
push_back(IITDescriptor::getVector(4, IsScalableVector()));
322 OutputTable.
push_back(IITDescriptor::getVector(6, IsScalableVector()));
326 OutputTable.
push_back(IITDescriptor::getVector(8, IsScalableVector()));
330 OutputTable.
push_back(IITDescriptor::getVector(10, IsScalableVector()));
334 OutputTable.
push_back(IITDescriptor::getVector(16, IsScalableVector()));
338 OutputTable.
push_back(IITDescriptor::getVector(32, IsScalableVector()));
342 OutputTable.
push_back(IITDescriptor::getVector(64, IsScalableVector()));
346 OutputTable.
push_back(IITDescriptor::getVector(128, IsScalableVector()));
350 OutputTable.
push_back(IITDescriptor::getVector(256, IsScalableVector()));
354 OutputTable.
push_back(IITDescriptor::getVector(512, IsScalableVector()));
358 OutputTable.
push_back(IITDescriptor::getVector(1024, IsScalableVector()));
362 OutputTable.
push_back(IITDescriptor::getVector(2048, IsScalableVector()));
366 OutputTable.
push_back(IITDescriptor::getVector(4096, IsScalableVector()));
370 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::WasmExternref, 0));
373 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::WasmFuncref, 0));
376 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::Pointer, 0));
380 IITDescriptor::get(IITDescriptor::Pointer, Infos[NextElt++]));
383 unsigned OverloadIndex = Infos[NextElt++];
384 unsigned ArgKindEnums = Infos[NextElt++];
385 unsigned Packed = (ArgKindEnums << 8) | OverloadIndex;
387 IITDescriptor::get(IITDescriptor::Overloaded, Packed));
391 unsigned OverloadIndex = Infos[NextElt++];
393 IITDescriptor::get(IITDescriptor::Match, OverloadIndex));
396 case IIT_EXTEND_ARG: {
397 unsigned OverloadIndex = Infos[NextElt++];
399 IITDescriptor::get(IITDescriptor::Extend, OverloadIndex));
402 case IIT_TRUNC_ARG: {
403 unsigned OverloadIndex = Infos[NextElt++];
405 IITDescriptor::get(IITDescriptor::Trunc, OverloadIndex));
408 case IIT_ONE_NTH_ELTS_VEC_ARG: {
409 unsigned short OverloadIndex = Infos[NextElt++];
410 unsigned short N = Infos[NextElt++];
411 OutputTable.
push_back(IITDescriptor::get(IITDescriptor::OneNthEltsVec,
415 case IIT_SAME_VEC_WIDTH_ARG: {
416 unsigned OverloadIndex = Infos[NextElt++];
418 IITDescriptor::get(IITDescriptor::SameVecWidth, OverloadIndex));
423 case IIT_ANY_SAME_VEC_WIDTH_ARG: {
424 unsigned short OverloadIndex = Infos[NextElt++];
425 unsigned short RefOverloadIndex = Infos[NextElt++];
426 unsigned short ElementConstraint = Infos[NextElt++];
429 OutputTable.
push_back(IITDescriptor::get(
430 IITDescriptor::AnySameVecWidth, RefOverloadIndex,
431 ElementConstraint << 4 | OverloadIndex));
435 unsigned StructElts = Infos[NextElt++] + 2;
438 IITDescriptor::get(IITDescriptor::Struct, StructElts));
440 for (
unsigned i = 0; i != StructElts; ++i)
444 case IIT_SUBDIVIDE2_ARG: {
445 unsigned OverloadIndex = Infos[NextElt++];
447 IITDescriptor::get(IITDescriptor::Subdivide2, OverloadIndex));
450 case IIT_SUBDIVIDE4_ARG: {
451 unsigned OverloadIndex = Infos[NextElt++];
453 IITDescriptor::get(IITDescriptor::Subdivide4, OverloadIndex));
456 case IIT_VEC_ELEMENT: {
457 unsigned OverloadIndex = Infos[NextElt++];
459 IITDescriptor::get(IITDescriptor::VecElement, OverloadIndex));
462 case IIT_VEC_OF_BITCASTS_TO_INT: {
463 unsigned OverloadIndex = Infos[NextElt++];
465 IITDescriptor::get(IITDescriptor::VecOfBitcastsToInt, OverloadIndex));
468 case IIT_SCALABLE_VEC:
474#define GET_INTRINSIC_GENERATOR_GLOBAL
475#include "llvm/IR/IntrinsicImpl.inc"
477std::tuple<ArrayRef<Intrinsic::IITDescriptor>,
unsigned,
bool>
483 constexpr unsigned FixedEncodingBits =
sizeof(FixedEncodingTy) * CHAR_BIT;
484 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
486 constexpr unsigned Mask = (1U << MSBPosition) - 1;
488 FixedEncodingTy TableVal = IIT_Table[
id - 1];
498 unsigned char IITValues[FixedEncodingBits / 4 + 1] = {0};
501 unsigned NextElt = 0;
504 if (TableVal >> MSBPosition) {
506 IITEntries = IIT_LongEncodingTable;
509 NextElt = TableVal & Mask;
514 IITValues[NextElt++] = TableVal & 0xF;
518 IITEntries = IITValues;
524 unsigned NumArgs = 0;
525 while (IITEntries[NextElt] != IIT_Done) {
532 bool IsVarArg =
false;
538 return {
TableRef, NumArgs, IsVarArg};
549 case IITDescriptor::Void:
551 case IITDescriptor::MMX:
553 case IITDescriptor::AMX:
555 case IITDescriptor::Token:
557 case IITDescriptor::Metadata:
559 case IITDescriptor::Half:
561 case IITDescriptor::BFloat:
563 case IITDescriptor::Float:
565 case IITDescriptor::Double:
567 case IITDescriptor::Quad:
569 case IITDescriptor::PPCQuad:
571 case IITDescriptor::AArch64Svcount:
573 case IITDescriptor::WasmExternref:
575 case IITDescriptor::WasmFuncref:
577 case IITDescriptor::Integer:
579 case IITDescriptor::Vector:
582 case IITDescriptor::Pointer:
584 case IITDescriptor::Struct: {
586 for (
unsigned i = 0, e =
D.StructNumElements; i != e; ++i)
593 case IITDescriptor::Overloaded:
594 case IITDescriptor::AnySameVecWidth:
595 case IITDescriptor::Match:
596 return OverloadTys[
D.getOverloadIndex()];
597 case IITDescriptor::Extend:
598 return OverloadTys[
D.getOverloadIndex()]->getExtendedType();
599 case IITDescriptor::Trunc:
600 return OverloadTys[
D.getOverloadIndex()]->getTruncatedType();
601 case IITDescriptor::Subdivide2:
602 case IITDescriptor::Subdivide4: {
603 Type *Ty = OverloadTys[
D.getOverloadIndex()];
605 assert(VTy &&
"Expected overload type to be a Vector Type");
606 int SubDivs =
D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
609 case IITDescriptor::OneNthEltsVec:
612 D.getVectorDivisor());
613 case IITDescriptor::SameVecWidth: {
615 Type *Ty = OverloadTys[
D.getOverloadIndex()];
620 case IITDescriptor::VecElement: {
621 Type *Ty = OverloadTys[
D.getOverloadIndex()];
623 return VTy->getElementType();
626 case IITDescriptor::VecOfBitcastsToInt: {
627 Type *Ty = OverloadTys[
D.getOverloadIndex()];
629 assert(VTy &&
"Expected overload type to be a Vector Type");
632 case IITDescriptor::VarArg:
654#define GET_INTRINSIC_OVERLOAD_TABLE
655#include "llvm/IR/IntrinsicImpl.inc"
659#define GET_INTRINSIC_SCALARIZABLE_TABLE
660#include "llvm/IR/IntrinsicImpl.inc"
664#define GET_INTRINSIC_PRETTY_PRINT_TABLE
665#include "llvm/IR/IntrinsicImpl.inc"
669#define GET_INTRINSIC_TARGET_DATA
670#include "llvm/IR/IntrinsicImpl.inc"
673 return IID > TargetInfos[0].Count;
682 assert(Name.starts_with(
"llvm.") &&
"Unexpected intrinsic prefix");
683 assert(Name.drop_front(5).starts_with(
Target) &&
"Unexpected target");
694 CmpEnd += 1 +
Target.size();
696 const unsigned *
Low = NameOffsetTable.
begin();
697 const unsigned *
High = NameOffsetTable.
end();
698 const unsigned *LastLow =
Low;
699 while (CmpEnd < Name.size() &&
High -
Low > 0) {
700 size_t CmpStart = CmpEnd;
701 CmpEnd = Name.find(
'.', CmpStart + 1);
703 auto Cmp = [CmpStart, CmpEnd](
auto LHS,
auto RHS) {
708 if constexpr (std::is_integral_v<
decltype(
LHS)>)
709 LHSStr = IntrinsicNameTable.getCString(
LHS);
714 if constexpr (std::is_integral_v<
decltype(
RHS)>)
715 RHSStr = IntrinsicNameTable.getCString(
RHS);
719 return strncmp(LHSStr + CmpStart, RHSStr + CmpStart, CmpEnd - CmpStart) <
723 std::tie(
Low,
High) = std::equal_range(
Low,
High, Name.data(), Cmp);
728 if (LastLow == NameOffsetTable.
end())
730 StringRef NameFound = IntrinsicNameTable[*LastLow];
731 if (Name == NameFound ||
732 (Name.starts_with(NameFound) && Name[NameFound.
size()] ==
'.'))
733 return LastLow - NameOffsetTable.
begin();
742static std::pair<ArrayRef<unsigned>,
StringRef>
744 assert(Name.starts_with(
"llvm."));
751 Targets, [=](
const IntrinsicTargetInfo &TI) {
return TI.Name <
Target; });
754 const auto &TI = It != Targets.
end() && It->Name ==
Target ? *It : Targets[0];
755 return {
ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
769 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
774 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
775 assert(Name.size() >= MatchSize &&
"Expected either exact or prefix match");
776 bool IsExactMatch = Name.size() == MatchSize;
782#define GET_INTRINSIC_ATTRIBUTES
783#include "llvm/IR/IntrinsicImpl.inc"
789 std::string Name = OverloadTys.
empty()
793 if (
F->getFunctionType() == FT)
801 F->setName(
F->getName() +
".invalid");
802 return cast<Function>(M->getOrInsertFunction(Name, FT).getCallee());
828 assert(IsValid &&
"intrinsic signature mismatch");
843#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
844#include "llvm/IR/IntrinsicImpl.inc"
847#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
848#include "llvm/IR/IntrinsicImpl.inc"
852#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
853 case Intrinsic::INTRINSIC:
854#include "llvm/IR/ConstrainedOps.def"
864#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
865 case Intrinsic::INTRINSIC: \
866 return ROUND_MODE == 1;
867#include "llvm/IR/ConstrainedOps.def"
891struct MatchPosition {
895 uint16_t IsStruct : 1;
901 static constexpr unsigned INDEX_TABLE_SIZE = 2;
902 Index Indices[INDEX_TABLE_SIZE];
906 assert(NumIndices > 0 &&
"cannot pop from empty indices");
910 void push_struct_element(
unsigned ElementNum) {
911 assert(NumIndices < INDEX_TABLE_SIZE &&
"index table overflow");
913 Indices[NumIndices].IsStruct =
true;
914 Indices[NumIndices++].Num = ElementNum;
917 void push_vector_element() {
918 assert(NumIndices < INDEX_TABLE_SIZE &&
"index table overflow");
919 Indices[NumIndices].IsStruct =
false;
920 Indices[NumIndices++].Num = 0;
931 OS <<
"argument " << Pos.Num;
933 for (
const MatchPosition::Index &Idx :
934 ArrayRef(Pos.Indices).take_front(Pos.NumIndices)) {
936 OS <<
" struct element " << Idx.Num;
938 OS <<
" vector element";
944 std::tuple<Type *, ArrayRef<Intrinsic::IITDescriptor>, MatchPosition>;
948static std::pair<bool, StringRef>
950 Intrinsic::IITDescriptor::AnyKindElementConstraint EC) {
955 case IITDescriptor::EC_None:
956 return {
true,
"value"};
957 case IITDescriptor::EC_Integer:
959 case IITDescriptor::EC_Float:
961 case IITDescriptor::EC_Pointer:
976 OS << Position <<
" too many "
977 << (Position.IsRet ?
"returns" :
"arguments");
982 auto InfosRef = Infos;
983 auto DeferCheck = [&DeferredChecks, &InfosRef, &Position](
Type *
T) {
992 auto PrintMsg = [&OS, &Position,
994 std::optional<unsigned> OIdx = std::nullopt) ->
bool {
997 OS << Position <<
" type";
999 OS <<
" (overload type " << *OIdx <<
")";
1000 OS <<
" expected " <<
Expected <<
", but got " << *Ty;
1007 auto PrintMsgInvalidOverloadTy =
1008 [&OS, &Position, &OverloadTys](
const Twine &DependentQualifier,
1010 unsigned OIdx) ->
bool {
1011 OS << Position <<
" is " << DependentQualifier <<
" overload type " << OIdx
1012 <<
", so overload type " << OIdx <<
" expected " <<
Expected
1013 <<
", but got " << *OverloadTys[OIdx];
1018 auto PrintMsgInvalidDepType =
1019 [&OS, &Position, &OverloadTys,
1021 unsigned OIdx) ->
bool {
1024 bool IsMatching = DependentQualifier.isSingleStringRef() &&
1025 DependentQualifier.getSingleStringRef() ==
"matching";
1026 OS << Position <<
" type (" << DependentQualifier <<
" overload type "
1027 << OIdx <<
") expected " <<
Expected;
1029 OS <<
" (overload type " << OIdx <<
" is " << *OverloadTys[OIdx] <<
")";
1030 OS <<
", but got " << *Ty;
1035 case IITDescriptor::Void:
1036 assert(Position.IsRet && Position.NumIndices == 0 &&
1037 "void descriptor expected only for return type");
1038 return PrintMsg(Ty->isVoidTy(),
"void");
1039 case IITDescriptor::MMX: {
1043 "x86_mmx (<1 x i64>)");
1045 case IITDescriptor::AMX:
1046 return PrintMsg(Ty->isX86_AMXTy(),
"x86_amx");
1047 case IITDescriptor::Token:
1048 return PrintMsg(Ty->isTokenTy(),
"token");
1049 case IITDescriptor::Metadata:
1050 return PrintMsg(Ty->isMetadataTy(),
"metadata");
1051 case IITDescriptor::Half:
1052 return PrintMsg(Ty->isHalfTy(),
"half");
1053 case IITDescriptor::BFloat:
1054 return PrintMsg(Ty->isBFloatTy(),
"bfloat");
1055 case IITDescriptor::Float:
1056 return PrintMsg(Ty->isFloatTy(),
"float");
1057 case IITDescriptor::Double:
1058 return PrintMsg(Ty->isDoubleTy(),
"double");
1059 case IITDescriptor::Quad:
1060 return PrintMsg(Ty->isFP128Ty(),
"fp128");
1061 case IITDescriptor::PPCQuad:
1062 return PrintMsg(Ty->isPPC_FP128Ty(),
"ppc_fp128");
1063 case IITDescriptor::Integer:
1064 return PrintMsg(Ty->isIntegerTy(
D.IntegerWidth),
1065 "i" +
Twine(
D.IntegerWidth));
1066 case IITDescriptor::AArch64Svcount:
1070 case IITDescriptor::WasmExternref:
1074 case IITDescriptor::WasmFuncref:
1078 case IITDescriptor::Vector: {
1080 StringRef Scalable =
D.VectorWidth.isScalable() ?
"vscale " :
"";
1082 PrintMsg(VT && VT->getElementCount() ==
D.VectorWidth,
1083 Twine(Scalable) +
"vector with " +
1084 Twine(
D.VectorWidth.getKnownMinValue()) +
" elements");
1087 Position.push_vector_element();
1089 OverloadTys, DeferredChecks, IsDeferredCheck, OS);
1091 case IITDescriptor::Pointer: {
1093 unsigned AS =
D.PointerAddressSpace;
1094 bool IsValid = PT && PT->getAddressSpace() == AS;
1096 return PrintMsg(IsValid,
"ptr");
1097 return PrintMsg(IsValid,
"ptr addrspace(" +
Twine(AS) +
")");
1100 case IITDescriptor::Struct: {
1102 unsigned EC =
D.StructNumElements;
1103 bool HasError = PrintMsg(
1104 ST && ST->isLiteral() && !ST->isPacked() && ST->getNumElements() == EC,
1105 "literal non-packed struct with " +
Twine(EC) +
" elements");
1110 Position.push_struct_element(Idx);
1112 IsDeferredCheck, OS))
1114 Position.pop_index();
1119 case IITDescriptor::Overloaded: {
1120 unsigned OIdx =
D.getOverloadIndex();
1121 assert(OIdx == OverloadTys.
size() && !IsDeferredCheck &&
1122 "Table consistency error");
1128 if (Ty->isTokenTy())
1129 return PrintMsg(
false,
"any manglable type", OIdx);
1131 IITDescriptor::AnyKindVectorConstraint VC;
1132 IITDescriptor::AnyKindElementConstraint EC;
1133 std::tie(VC, EC) =
D.getOverloadConstraints();
1135 bool IsValid = [&]() {
1137 case IITDescriptor::VC_None:
1139 case IITDescriptor::VC_Vector:
1141 case IITDescriptor::VC_Scalar:
1148 IsValid &= ElemValid;
1159 if (EC == IITDescriptor::EC_None) {
1163 return PrintMsg(
false,
formatv(
"any {} type", VK), OIdx);
1167 case IITDescriptor::VC_None:
1169 return PrintMsg(
false,
formatv(
"any {0} or {0} vector", EK), OIdx);
1170 case IITDescriptor::VC_Vector:
1171 return PrintMsg(
false,
formatv(
"any {} vector", EK), OIdx);
1172 case IITDescriptor::VC_Scalar:
1173 return PrintMsg(
false,
formatv(
"any {} type", EK), OIdx);
1178 case IITDescriptor::Match: {
1179 unsigned OIdx =
D.getOverloadIndex();
1180 if (OIdx >= OverloadTys.
size())
1181 return IsDeferredCheck || DeferCheck(Ty);
1182 return PrintMsgInvalidDepType(Ty == OverloadTys[OIdx],
"matching",
1183 formatv(
"{}", *OverloadTys[OIdx]), OIdx);
1186 case IITDescriptor::Extend:
1187 case IITDescriptor::Trunc: {
1188 unsigned OIdx =
D.getOverloadIndex();
1190 if (OIdx >= OverloadTys.
size())
1191 return IsDeferredCheck || DeferCheck(Ty);
1193 Type *OTy = OverloadTys[OIdx];
1194 bool IsExtend =
D.Kind == IITDescriptor::Extend;
1195 StringRef Qualifier = IsExtend ?
"extended" :
"truncated";
1197 return PrintMsgInvalidOverloadTy(Qualifier,
"int or vector of int", OIdx);
1200 return PrintMsgInvalidDepType(Ty == NewTy, Qualifier,
formatv(
"{}", *NewTy),
1203 case IITDescriptor::OneNthEltsVec: {
1204 unsigned OIdx =
D.getOverloadIndex();
1205 unsigned Divisor =
D.getVectorDivisor();
1207 if (OIdx >= OverloadTys.
size())
1208 return IsDeferredCheck || DeferCheck(Ty);
1209 Type *OTy = OverloadTys[OIdx];
1211 auto Qualifier =
formatv(
"1/nth (n={}) elements vector of", Divisor);
1213 return PrintMsgInvalidOverloadTy(Qualifier,
"vector", OIdx);
1214 if (!OVecTy->getElementCount().isKnownMultipleOf(Divisor))
1215 return PrintMsgInvalidOverloadTy(
1216 Qualifier,
formatv(
"vector with multiple of {} elements", Divisor),
1219 return PrintMsgInvalidDepType(
Expected == Ty, Qualifier,
1222 case IITDescriptor::SameVecWidth: {
1223 unsigned OIdx =
D.getOverloadIndex();
1224 if (OIdx >= OverloadTys.
size()) {
1227 return IsDeferredCheck || DeferCheck(Ty);
1232 StringRef Qualifier =
"same vector width of";
1233 if (OVecTy && !ThisArgVecType)
1234 return PrintMsgInvalidDepType(
false, Qualifier,
"vector", OIdx);
1235 if (!OVecTy && ThisArgVecType)
1236 return PrintMsgInvalidDepType(
false, Qualifier,
"scalar", OIdx);
1238 if (ThisArgVecType) {
1240 if (
Expected != ThisArgVecType->getElementCount())
1241 return PrintMsgInvalidDepType(
1244 EltTy = ThisArgVecType->getElementType();
1245 Position.push_vector_element();
1248 DeferredChecks, IsDeferredCheck, OS);
1250 case IITDescriptor::AnySameVecWidth: {
1251 unsigned RefOverloadIndex =
D.getRefOverloadIndex();
1252 if (RefOverloadIndex >= OverloadTys.
size()) {
1253 if (IsDeferredCheck)
1257 assert(
D.getOverloadIndex() == OverloadTys.
size() &&
1258 "Table consistency error");
1260 return DeferCheck(Ty);
1263 if (!IsDeferredCheck) {
1264 assert(
D.getOverloadIndex() == OverloadTys.
size() &&
1265 "Table consistency error");
1269 auto [ElemValid, Kind] =
1274 StringRef Qualifier =
"same vector width of";
1278 formatv(
"{0} vector with {1} elements", Kind, ExpectedCount);
1280 return PrintMsgInvalidDepType(
false, Qualifier,
Expected,
1283 ThisArgVecTy->getElementCount() == ExpectedCount && ElemValid;
1284 return PrintMsgInvalidDepType(IsValid, Qualifier,
Expected,
1290 return PrintMsgInvalidDepType(
false, Qualifier,
Expected,
1292 return PrintMsgInvalidDepType(ElemValid, Qualifier,
Expected,
1295 case IITDescriptor::VecElement: {
1296 unsigned OIdx =
D.getOverloadIndex();
1297 if (OIdx >= OverloadTys.
size())
1298 return IsDeferredCheck || DeferCheck(Ty);
1299 StringRef Qualifier =
"vector element of";
1302 return PrintMsgInvalidOverloadTy(Qualifier,
"vector", OIdx);
1304 return PrintMsgInvalidDepType(
Expected == Ty, Qualifier,
1307 case IITDescriptor::Subdivide2:
1308 case IITDescriptor::Subdivide4: {
1309 unsigned OIdx =
D.getOverloadIndex();
1311 if (OIdx >= OverloadTys.
size())
1312 return IsDeferredCheck || DeferCheck(Ty);
1314 int SubDivs =
D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
1317 formatv(
"subdivided by {} vector of", SubDivs == 1 ? 2 : 4);
1319 return PrintMsgInvalidOverloadTy(Qualifier,
"vector", OIdx);
1324 return PrintMsgInvalidDepType(
Expected == Ty, Qualifier,
1327 case IITDescriptor::VecOfBitcastsToInt: {
1328 unsigned OIdx =
D.getOverloadIndex();
1329 if (OIdx >= OverloadTys.
size())
1330 return IsDeferredCheck || DeferCheck(Ty);
1332 StringRef Qualifier =
"vector of bitcasts to int of";
1334 return PrintMsgInvalidOverloadTy(Qualifier,
"vector", OIdx);
1336 return PrintMsgInvalidDepType(
Expected == Ty, Qualifier,
1339 case IITDescriptor::VarArg:
1360 unsigned NumArgs,
bool IsVarArg,
1363 unsigned NumMissingTrailingParams) {
1366 assert(!Infos.
empty() &&
"Table consistency error");
1373 DeferredChecks,
false, OS))
1376 unsigned ProvidedArgs = FTy->getNumParams();
1377 if (ProvidedArgs + NumMissingTrailingParams != NumArgs) {
1378 OS <<
"intrinsic has incorrect number of args. Expected " << NumArgs
1379 <<
", but got " << ProvidedArgs;
1391 for (
unsigned I = 0,
E = DeferredChecks.
size();
I !=
E; ++
I) {
1392 auto &[DefTy, DefInfos, DefPosition] = DeferredChecks[
I];
1394 DeferredChecks,
true, OS))
1400 if (NumMissingTrailingParams != 0 &&
1401 (Infos.
size() != NumMissingTrailingParams ||
1403 return D.Kind != Intrinsic::IITDescriptor::Integer;
1405 OS <<
"cannot omit trailing parameters that are overloaded or non-integer";
1408 Infos = Infos.
drop_front(NumMissingTrailingParams);
1410 if (!Infos.
empty()) {
1411 OS <<
"intrinsic has too few arguments!";
1415 if (FTy->isVarArg() != IsVarArg) {
1417 OS <<
"intrinsic was not defined with variable arguments!";
1419 OS <<
"intrinsic was defined with variable arguments!";
1442 unsigned NumMissingTrailingParams,
1450 return ::isSignatureValid(FT,
TableRef, NumArgs, IsVarArg, OverloadTys, OS,
1451 NumMissingTrailingParams);
1464 return std::nullopt;
1468 std::string WantedName =
1470 if (Name == WantedName)
1471 return std::nullopt;
1474 if (
auto *ExistingGV =
F->getParent()->getNamedValue(WantedName)) {
1476 if (ExistingF->getFunctionType() ==
F->getFunctionType())
1483 ExistingGV->setName(WantedName +
".renamed");
1490 "Shouldn't change the signature");
1499 {Intrinsic::vector_interleave2, Intrinsic::vector_deinterleave2},
1500 {Intrinsic::vector_interleave3, Intrinsic::vector_deinterleave3},
1501 {Intrinsic::vector_interleave4, Intrinsic::vector_deinterleave4},
1502 {Intrinsic::vector_interleave5, Intrinsic::vector_deinterleave5},
1503 {Intrinsic::vector_interleave6, Intrinsic::vector_deinterleave6},
1504 {Intrinsic::vector_interleave7, Intrinsic::vector_deinterleave7},
1505 {Intrinsic::vector_interleave8, Intrinsic::vector_deinterleave8},
1509 assert(Factor >= 2 && Factor <= 8 &&
"Unexpected factor");
1514 assert(Factor >= 2 && Factor <= 8 &&
"Unexpected factor");
1521 OS << static_cast<FPClassTest>(Val);
1524#define GET_INTRINSIC_IMMARG_RANGE_SET_CHECKS
1525#include "llvm/IR/IntrinsicImpl.inc"
1527#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1528#include "llvm/IR/IntrinsicImpl.inc"
1532#define GET_INTRINSIC_DEFAULT_ARG_VALUES
1533#include "llvm/IR/IntrinsicImpl.inc"
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
ArrayRef< TableEntry > TableRef
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Module.h This file contains the declarations for the Module class.
static bool matchIntrinsicType(Type *Ty, ArrayRef< Intrinsic::IITDescriptor > &Infos, MatchPosition Position, SmallVectorImpl< Type * > &OverloadTys, SmallVectorImpl< DeferredIntrinsicMatchInfo > &DeferredChecks, bool IsDeferredCheck, raw_ostream &OS)
static InterleaveIntrinsic InterleaveIntrinsics[]
static std::pair< bool, StringRef > checkElementConstraint(Type *Ty, Intrinsic::IITDescriptor::AnyKindElementConstraint EC)
Check whether Ty satisfies the given element constraint EC and return a description of the expected e...
std::tuple< Type *, ArrayRef< Intrinsic::IITDescriptor >, MatchPosition > DeferredIntrinsicMatchInfo
static std::pair< ArrayRef< unsigned >, StringRef > findTargetSubtable(StringRef Name)
Find the segment of IntrinsicNameOffsetTable for intrinsics with the same target as Name,...
static Function * getOrInsertIntrinsicDeclarationImpl(Module *M, Intrinsic::ID id, ArrayRef< Type * > OverloadTys, FunctionType *FT)
static void DecodeIITType(unsigned &NextElt, ArrayRef< unsigned char > Infos, SmallVectorImpl< Intrinsic::IITDescriptor > &OutputTable)
static std::string getIntrinsicNameImpl(Intrinsic::ID Id, ArrayRef< Type * > OverloadTys, Module *M, FunctionType *FT, bool EarlyModuleCheck)
IIT_Info
IIT_Info - These are enumerators that describe the entries returned by the getIntrinsicInfoTableEntri...
static Type * DecodeFixedType(ArrayRef< Intrinsic::IITDescriptor > &Infos, ArrayRef< Type * > OverloadTys, LLVMContext &Context)
static int lookupLLVMIntrinsicByName(ArrayRef< unsigned > NameOffsetTable, StringRef Name, StringRef Target="")
Looks up Name in NameTable via binary search.
static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType)
Returns a stable mangling for the type specified for use in the name mangling scheme used by 'any' ty...
static bool isSignatureValid(FunctionType *FTy, ArrayRef< Intrinsic::IITDescriptor > &Infos, unsigned NumArgs, bool IsVarArg, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS, unsigned NumMissingTrailingParams=0)
Return true if the function type FTy is a valid type signature for the type constraints specified in ...
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
static StringRef getName(Value *V)
static SymbolRef::Type getType(const Symbol *Sym)
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
const T & consume_front()
consume_front() - Returns the first element and drops it from ArrayRef.
This is an important base class in LLVM.
Tagged union holding either a T or a Error.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const Function & getFunction() const
void setCallingConv(CallingConv::ID CC)
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
A Module instance is used to store all the information related to an LLVM module.
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
static constexpr size_t npos
std::string str() const
Get the contents as an std::string.
constexpr size_t size() const
Get the string size.
Class to represent struct types.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI Type * getX86_AMXTy(LLVMContext &C)
LLVM_ABI Type * getTruncatedType() const
Given scalar/vector integer type, returns a type with elements half as wide as in the original type.
static LLVM_ABI Type * getMetadataTy(LLVMContext &C)
static LLVM_ABI Type * getTokenTy(LLVMContext &C)
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
bool isPointerTy() const
True if this is an instance of PointerType.
static LLVM_ABI Type * getPPC_FP128Ty(LLVMContext &C)
static LLVM_ABI Type * getFP128Ty(LLVMContext &C)
@ X86_AMXTyID
AMX vectors (8192 bits, X86 specific)
@ HalfTyID
16-bit floating point type
@ VoidTyID
type with no size
@ FloatTyID
32-bit floating point type
@ IntegerTyID
Arbitrary bit width integers.
@ BFloatTyID
16-bit floating point type (7-bit significand)
@ DoubleTyID
64-bit floating point type
@ X86_FP80TyID
80-bit floating point type (X87)
@ PPC_FP128TyID
128-bit floating point type (two 64-bits, PowerPC)
@ ByteTyID
Arbitrary bit width bytes.
@ FP128TyID
128-bit floating point type (112-bit significand)
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
LLVM_ABI Type * getExtendedType() const
Given scalar/vector integer type, returns a type with elements twice as wide as in the original type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
static VectorType * getOneNthElementsVectorType(VectorType *VTy, unsigned Denominator)
static VectorType * getSubdividedVectorType(VectorType *VTy, int NumSubdivs)
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor)
Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N.
LLVM_ABI void printFPClassMask(raw_ostream &OS, const Constant *ImmArgVal)
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
LLVM_ABI bool hasConstrainedFPRoundingModeOperand(ID QID)
Returns true if the intrinsic ID is for one of the "ConstrainedFloating-Point Intrinsics" that take r...
LLVM_ABI StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI bool isConstrainedFPIntrinsic(ID QID)
Returns true if the intrinsic ID is for one of the "ConstrainedFloating-Point Intrinsics".
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
LLVM_ABI bool hasPrettyPrintedArgs(ID id)
Returns true if the intrinsic has pretty printed immediate arguments.
LLVM_ABI std::tuple< ArrayRef< IITDescriptor >, unsigned, bool > getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Fill the IIT table descriptor for the intrinsic id into an array of IITDescriptors.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
LLVM_ABI Intrinsic::ID getInterleaveIntrinsicID(unsigned Factor)
Returns the corresponding llvm.vector.interleaveN intrinsic for factor N.
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
LLVM_ABI bool isTriviallyScalarizable(ID id)
Returns true if the intrinsic is trivially scalarizable.
LLVM_ABI bool isTargetIntrinsic(ID IID)
isTargetIntrinsic - Returns true if IID is an intrinsic specific to a certain target.
LLVM_ABI StringRef getRequiredTargetFeatures(ID id)
LLVM_ABI std::string getNameNoUnnamedTypes(ID Id, ArrayRef< Type * > OverloadTys)
Return the LLVM name for an intrinsic.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
std::string utostr(uint64_t X, bool isNeg=false)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)
LLVM_ABI raw_ostream & nulls()
This returns a reference to a raw_ostream which simply discards output.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Intrinsic::ID Deinterleave
This is a type descriptor which explains the type requirements of an intrinsic.