Intrinsics.cpp

Go to the documentation of this file.

//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements functions required for supporting intrinsic functions.
//
//===----------------------------------------------------------------------===//
 
#include "llvm/IR/Intrinsics.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringTable.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsBPF.h"
#include "llvm/IR/IntrinsicsHexagon.h"
#include "llvm/IR/IntrinsicsLoongArch.h"
#include "llvm/IR/IntrinsicsMips.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/IntrinsicsPowerPC.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/IntrinsicsS390.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/IR/IntrinsicsVE.h"
#include "llvm/IR/IntrinsicsX86.h"
#include "llvm/IR/IntrinsicsXCore.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
 
using namespace llvm;
 
// Forward declaration of static functions.
static bool isSignatureValid(FunctionType *FTy,
                             ArrayRef<Intrinsic::IITDescriptor> &Infos,
                             unsigned NumArgs, bool IsVarArg,
                             SmallVectorImpl<Type *> &OverloadTys,
                             raw_ostream &OS);
 
/// Table of string intrinsic names indexed by enum value.
#define GET_INTRINSIC_NAME_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
 
/// Table of required target features indexed by enum value.
#define GET_INTRINSIC_TARGET_FEATURES_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
 
StringRef Intrinsic::getBaseName(ID id) {
  assert(id < num_intrinsics && "Invalid intrinsic ID!");
  return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
}
 
StringRef Intrinsic::getRequiredTargetFeatures(ID id) {
  assert(id < num_intrinsics && "invalid intrinsic ID!");
  return IntrinsicTargetFeaturesTable[IntrinsicTargetFeaturesOffsetTable[id]];
}
 
StringRef Intrinsic::getName(ID id) {
  assert(id < num_intrinsics && "Invalid intrinsic ID!");
  assert(!Intrinsic::isOverloaded(id) &&
         "This version of getName does not support overloading");
  return getBaseName(id);
}
 
/// Returns a stable mangling for the type specified for use in the name
/// mangling scheme used by 'any' types in intrinsic signatures.  The mangling
/// of named types is simply their name.  Manglings for unnamed types consist
/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
/// combined with the mangling of their component types.  A vararg function
/// type will have a suffix of 'vararg'.  Since function types can contain
/// other function types, we close a function type mangling with suffix 'f'
/// which can't be confused with it's prefix.  This ensures we don't have
/// collisions between two unrelated function types. Otherwise, you might
/// parse ffXX as f(fXX) or f(fX)X.  (X is a placeholder for any other type.)
/// The HasUnnamedType boolean is set if an unnamed type was encountered,
/// indicating that extra care must be taken to ensure a unique name.
static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
  std::string Result;
  if (PointerType *PTyp = dyn_cast<PointerType>(Ty)) {
    Result += "p" + utostr(PTyp->getAddressSpace());
  } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Ty)) {
    Result += "a" + utostr(ATyp->getNumElements()) +
              getMangledTypeStr(ATyp->getElementType(), HasUnnamedType);
  } else if (StructType *STyp = dyn_cast<StructType>(Ty)) {
    if (!STyp->isLiteral()) {
      Result += "s_";
      if (STyp->hasName())
        Result += STyp->getName();
      else
        HasUnnamedType = true;
    } else {
      Result += "sl_";
      for (auto *Elem : STyp->elements())
        Result += getMangledTypeStr(Elem, HasUnnamedType);
    }
    // Ensure nested structs are distinguishable.
    Result += "s";
  } else if (FunctionType *FT = dyn_cast<FunctionType>(Ty)) {
    Result += "f_" + getMangledTypeStr(FT->getReturnType(), HasUnnamedType);
    for (size_t i = 0; i < FT->getNumParams(); i++)
      Result += getMangledTypeStr(FT->getParamType(i), HasUnnamedType);
    if (FT->isVarArg())
      Result += "vararg";
    // Ensure nested function types are distinguishable.
    Result += "f";
  } else if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
    ElementCount EC = VTy->getElementCount();
    if (EC.isScalable())
      Result += "nx";
    Result += "v" + utostr(EC.getKnownMinValue()) +
              getMangledTypeStr(VTy->getElementType(), HasUnnamedType);
  } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Ty)) {
    Result += "t";
    Result += TETy->getName();
    for (Type *ParamTy : TETy->type_params())
      Result += "_" + getMangledTypeStr(ParamTy, HasUnnamedType);
    for (unsigned IntParam : TETy->int_params())
      Result += "_" + utostr(IntParam);
    // Ensure nested target extension types are distinguishable.
    Result += "t";
  } else if (Ty) {
    switch (Ty->getTypeID()) {
    default:
      llvm_unreachable("Unhandled type");
    case Type::VoidTyID:
      Result += "isVoid";
      break;
    case Type::MetadataTyID:
      Result += "Metadata";
      break;
    case Type::HalfTyID:
      Result += "f16";
      break;
    case Type::BFloatTyID:
      Result += "bf16";
      break;
    case Type::FloatTyID:
      Result += "f32";
      break;
    case Type::DoubleTyID:
      Result += "f64";
      break;
    case Type::X86_FP80TyID:
      Result += "f80";
      break;
    case Type::FP128TyID:
      Result += "f128";
      break;
    case Type::PPC_FP128TyID:
      Result += "ppcf128";
      break;
    case Type::X86_AMXTyID:
      Result += "x86amx";
      break;
    case Type::IntegerTyID:
      Result += "i" + utostr(cast<IntegerType>(Ty)->getBitWidth());
      break;
    case Type::ByteTyID:
      Result += "b" + utostr(cast<ByteType>(Ty)->getBitWidth());
      break;
    }
  }
  return Result;
}
 
static std::string getIntrinsicNameImpl(Intrinsic::ID Id,
                                        ArrayRef<Type *> OverloadTys, Module *M,
                                        FunctionType *FT,
                                        bool EarlyModuleCheck) {
 
  assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
  assert((OverloadTys.empty() || Intrinsic::isOverloaded(Id)) &&
         "This version of getName is for overloaded intrinsics only");
  (void)EarlyModuleCheck;
  assert((!EarlyModuleCheck || M ||
          !any_of(OverloadTys, llvm::IsaPred<PointerType>)) &&
         "Intrinsic overloading on pointer types need to provide a Module");
  bool HasUnnamedType = false;
  std::string Result(Intrinsic::getBaseName(Id));
  for (Type *Ty : OverloadTys)
    Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
  if (HasUnnamedType) {
    assert(M && "unnamed types need a module");
    if (!FT)
      FT = Intrinsic::getType(M->getContext(), Id, OverloadTys);
    else
      assert(FT == Intrinsic::getType(M->getContext(), Id, OverloadTys) &&
             "Provided FunctionType must match arguments");
    return M->getUniqueIntrinsicName(Result, Id, FT);
  }
  return Result;
}
 
std::string Intrinsic::getName(ID Id, ArrayRef<Type *> OverloadTys, Module *M,
                               FunctionType *FT) {
  assert(M && "We need to have a Module");
  return getIntrinsicNameImpl(Id, OverloadTys, M, FT, true);
}
 
std::string Intrinsic::getNameNoUnnamedTypes(ID Id,
                                             ArrayRef<Type *> OverloadTys) {
  return getIntrinsicNameImpl(Id, OverloadTys, nullptr, nullptr, false);
}
 
/// IIT_Info - These are enumerators that describe the entries returned by the
/// getIntrinsicInfoTableEntries function.
///
/// Defined in Intrinsics.td.
enum IIT_Info {
#define GET_INTRINSIC_IITINFO
#include "llvm/IR/IntrinsicImpl.inc"
};
 
static_assert(IIT_Done == 0, "IIT_Done expected to be 0");
 
static void
DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
              SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
  using namespace Intrinsic;
 
  auto IsScalableVector = [&]() {
    IIT_Info NextInfo = IIT_Info(Infos[NextElt]);
    if (NextInfo != IIT_SCALABLE_VEC)
      return false;
    // Eat the IIT_SCALABLE_VEC token.
    ++NextElt;
    return true;
  };
 
  IIT_Info Info = IIT_Info(Infos[NextElt++]);
 
  switch (Info) {
  case IIT_Done:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Void, 0));
    return;
  case IIT_VARARG:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::VarArg, 0));
    return;
  case IIT_MMX:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::MMX, 0));
    return;
  case IIT_AMX:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::AMX, 0));
    return;
  case IIT_TOKEN:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Token, 0));
    return;
  case IIT_METADATA:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Metadata, 0));
    return;
  case IIT_F16:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Half, 0));
    return;
  case IIT_BF16:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::BFloat, 0));
    return;
  case IIT_F32:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Float, 0));
    return;
  case IIT_F64:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Double, 0));
    return;
  case IIT_F128:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Quad, 0));
    return;
  case IIT_PPCF128:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::PPCQuad, 0));
    return;
  case IIT_I1:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 1));
    return;
  case IIT_I2:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 2));
    return;
  case IIT_I4:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 4));
    return;
  case IIT_AARCH64_SVCOUNT:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::AArch64Svcount, 0));
    return;
  case IIT_I8:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 8));
    return;
  case IIT_I16:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 16));
    return;
  case IIT_I32:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 32));
    return;
  case IIT_I64:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 64));
    return;
  case IIT_I128:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 128));
    return;
  case IIT_V1:
    OutputTable.push_back(IITDescriptor::getVector(1, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V2:
    OutputTable.push_back(IITDescriptor::getVector(2, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V3:
    OutputTable.push_back(IITDescriptor::getVector(3, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V4:
    OutputTable.push_back(IITDescriptor::getVector(4, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V6:
    OutputTable.push_back(IITDescriptor::getVector(6, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V8:
    OutputTable.push_back(IITDescriptor::getVector(8, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V10:
    OutputTable.push_back(IITDescriptor::getVector(10, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V16:
    OutputTable.push_back(IITDescriptor::getVector(16, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V32:
    OutputTable.push_back(IITDescriptor::getVector(32, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V64:
    OutputTable.push_back(IITDescriptor::getVector(64, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V128:
    OutputTable.push_back(IITDescriptor::getVector(128, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V256:
    OutputTable.push_back(IITDescriptor::getVector(256, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V512:
    OutputTable.push_back(IITDescriptor::getVector(512, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V1024:
    OutputTable.push_back(IITDescriptor::getVector(1024, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V2048:
    OutputTable.push_back(IITDescriptor::getVector(2048, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_V4096:
    OutputTable.push_back(IITDescriptor::getVector(4096, IsScalableVector()));
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  case IIT_EXTERNREF:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::WasmExternref, 0));
    return;
  case IIT_FUNCREF:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::WasmFuncref, 0));
    return;
  case IIT_PTR:
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 0));
    return;
  case IIT_PTR_AS: // pointer with address space.
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Pointer, Infos[NextElt++]));
    return;
  case IIT_ANY: {
    unsigned OverloadIndex = Infos[NextElt++];
    unsigned ArgKindEnums = Infos[NextElt++];
    unsigned Packed = (ArgKindEnums << 8) | OverloadIndex;
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Overloaded, Packed));
    return;
  }
  case IIT_MATCH: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Match, OverloadIndex));
    return;
  }
  case IIT_EXTEND_ARG: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Extend, OverloadIndex));
    return;
  }
  case IIT_TRUNC_ARG: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Trunc, OverloadIndex));
    return;
  }
  case IIT_ONE_NTH_ELTS_VEC_ARG: {
    unsigned short OverloadIndex = Infos[NextElt++];
    unsigned short N = Infos[NextElt++];
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::OneNthEltsVec,
                                             /*Hi=*/N, /*Lo=*/OverloadIndex));
    return;
  }
  case IIT_SAME_VEC_WIDTH_ARG: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::SameVecWidth, OverloadIndex));
    // IIT_SAME_VEC_WIDTH_ARG entry is followed by the element type.
    DecodeIITType(NextElt, Infos, OutputTable);
    return;
  }
  case IIT_VEC_OF_ANYPTRS_TO_ELT: {
    unsigned short OverloadIndex = Infos[NextElt++];
    unsigned short RefOverloadIndex = Infos[NextElt++];
    OutputTable.push_back(IITDescriptor::get(IITDescriptor::VecOfAnyPtrsToElt,
                                             /*Hi=*/RefOverloadIndex,
                                             /*Lo=*/OverloadIndex));
    return;
  }
  case IIT_STRUCT: {
    unsigned StructElts = Infos[NextElt++] + 2;
 
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Struct, StructElts));
 
    for (unsigned i = 0; i != StructElts; ++i)
      DecodeIITType(NextElt, Infos, OutputTable);
    return;
  }
  case IIT_SUBDIVIDE2_ARG: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Subdivide2, OverloadIndex));
    return;
  }
  case IIT_SUBDIVIDE4_ARG: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::Subdivide4, OverloadIndex));
    return;
  }
  case IIT_VEC_ELEMENT: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::VecElement, OverloadIndex));
    return;
  }
  case IIT_VEC_OF_BITCASTS_TO_INT: {
    unsigned OverloadIndex = Infos[NextElt++];
    OutputTable.push_back(
        IITDescriptor::get(IITDescriptor::VecOfBitcastsToInt, OverloadIndex));
    return;
  }
  case IIT_SCALABLE_VEC:
    break;
  }
  llvm_unreachable("unhandled");
}
 
#define GET_INTRINSIC_GENERATOR_GLOBAL
#include "llvm/IR/IntrinsicImpl.inc"
 
std::tuple<ArrayRef<Intrinsic::IITDescriptor>, unsigned, bool>
Intrinsic::getIntrinsicInfoTableEntries(ID id,
                                        SmallVectorImpl<IITDescriptor> &T) {
  // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
  // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
  // IntrinsicEmitter.cpp.
  constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
  constexpr unsigned MSBPosition = FixedEncodingBits - 1;
  // Mask with all bits 1 except the most significant bit.
  constexpr unsigned Mask = (1U << MSBPosition) - 1;
 
  FixedEncodingTy TableVal = IIT_Table[id - 1];
 
  // Array to hold the inlined fixed encoding values expanded from nibbles to
  // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
  // of nibbles that can fit in `FixedEncodingTy` + 1 (the IIT_Done terminator
  // that is not explicitly encoded). Note that if there are trailing 0 bytes
  // in the encoding (for example, payload following one of the IIT tokens),
  // the inlined encoding does not encode the actual size of the encoding, so
  // we always assume its size of this maximum length possible, followed by the
  // IIT_Done terminator token (whose value is 0).
  unsigned char IITValues[FixedEncodingBits / 4 + 1] = {0};
 
  ArrayRef<unsigned char> IITEntries;
  unsigned NextElt = 0;
  // Check to see if the intrinsic's type was inlined in the fixed encoding
  // table.
  if (TableVal >> MSBPosition) {
    // This is an offset into the IIT_LongEncodingTable.
    IITEntries = IIT_LongEncodingTable;
 
    // Strip sentinel bit.
    NextElt = TableVal & Mask;
  } else {
    // If the entry was encoded into a single word in the table itself, decode
    // it from an array of nibbles to an array of bytes.
    do {
      IITValues[NextElt++] = TableVal & 0xF;
      TableVal >>= 4;
    } while (TableVal);
 
    IITEntries = IITValues;
    NextElt = 0;
  }
 
  // Okay, decode the table into the output vector of IITDescriptors.
  DecodeIITType(NextElt, IITEntries, T);
  unsigned NumArgs = 0;
  while (IITEntries[NextElt] != IIT_Done) {
    DecodeIITType(NextElt, IITEntries, T);
    ++NumArgs;
  }
 
  ArrayRef<IITDescriptor> TableRef = T;
 
  bool IsVarArg = false;
  if (TableRef.back().Kind == Intrinsic::IITDescriptor::VarArg) {
    IsVarArg = true;
    TableRef.consume_back();
    --NumArgs;
  }
  return {TableRef, NumArgs, IsVarArg};
}
 
static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
                             ArrayRef<Type *> OverloadTys,
                             LLVMContext &Context) {
  using namespace Intrinsic;
 
  IITDescriptor D = Infos.consume_front();
 
  switch (D.Kind) {
  case IITDescriptor::Void:
    return Type::getVoidTy(Context);
  case IITDescriptor::MMX:
    return llvm::FixedVectorType::get(llvm::IntegerType::get(Context, 64), 1);
  case IITDescriptor::AMX:
    return Type::getX86_AMXTy(Context);
  case IITDescriptor::Token:
    return Type::getTokenTy(Context);
  case IITDescriptor::Metadata:
    return Type::getMetadataTy(Context);
  case IITDescriptor::Half:
    return Type::getHalfTy(Context);
  case IITDescriptor::BFloat:
    return Type::getBFloatTy(Context);
  case IITDescriptor::Float:
    return Type::getFloatTy(Context);
  case IITDescriptor::Double:
    return Type::getDoubleTy(Context);
  case IITDescriptor::Quad:
    return Type::getFP128Ty(Context);
  case IITDescriptor::PPCQuad:
    return Type::getPPC_FP128Ty(Context);
  case IITDescriptor::AArch64Svcount:
    return TargetExtType::get(Context, "aarch64.svcount");
  case IITDescriptor::WasmExternref:
    return TargetExtType::get(Context, "wasm.externref");
  case IITDescriptor::WasmFuncref:
    return TargetExtType::get(Context, "wasm.funcref");
  case IITDescriptor::Integer:
    return IntegerType::get(Context, D.IntegerWidth);
  case IITDescriptor::Vector:
    return VectorType::get(DecodeFixedType(Infos, OverloadTys, Context),
                           D.VectorWidth);
  case IITDescriptor::Pointer:
    return PointerType::get(Context, D.PointerAddressSpace);
  case IITDescriptor::Struct: {
    SmallVector<Type *, 8> Elts;
    for (unsigned i = 0, e = D.StructNumElements; i != e; ++i)
      Elts.push_back(DecodeFixedType(Infos, OverloadTys, Context));
    return StructType::get(Context, Elts);
  }
  // For any overload type or partially dependent type, substitute it with the
  // corresponding concrete type from OverloadTys. Additionally, do the same
  // for the fully dependent type that matches an overload type.
  case IITDescriptor::Overloaded:
  case IITDescriptor::VecOfAnyPtrsToElt:
  case IITDescriptor::Match:
    return OverloadTys[D.getOverloadIndex()];
  case IITDescriptor::Extend:
    return OverloadTys[D.getOverloadIndex()]->getExtendedType();
  case IITDescriptor::Trunc:
    return OverloadTys[D.getOverloadIndex()]->getTruncatedType();
  case IITDescriptor::Subdivide2:
  case IITDescriptor::Subdivide4: {
    Type *Ty = OverloadTys[D.getOverloadIndex()];
    VectorType *VTy = dyn_cast<VectorType>(Ty);
    assert(VTy && "Expected overload type to be a Vector Type");
    int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
    return VectorType::getSubdividedVectorType(VTy, SubDivs);
  }
  case IITDescriptor::OneNthEltsVec:
    return VectorType::getOneNthElementsVectorType(
        cast<VectorType>(OverloadTys[D.getOverloadIndex()]),
        D.getVectorDivisor());
  case IITDescriptor::SameVecWidth: {
    Type *EltTy = DecodeFixedType(Infos, OverloadTys, Context);
    Type *Ty = OverloadTys[D.getOverloadIndex()];
    if (auto *VTy = dyn_cast<VectorType>(Ty))
      return VectorType::get(EltTy, VTy->getElementCount());
    return EltTy;
  }
  case IITDescriptor::VecElement: {
    Type *Ty = OverloadTys[D.getOverloadIndex()];
    if (VectorType *VTy = dyn_cast<VectorType>(Ty))
      return VTy->getElementType();
    llvm_unreachable("Expected overload type to be a Vector Type");
  }
  case IITDescriptor::VecOfBitcastsToInt: {
    Type *Ty = OverloadTys[D.getOverloadIndex()];
    VectorType *VTy = dyn_cast<VectorType>(Ty);
    assert(VTy && "Expected overload type to be a Vector Type");
    return VectorType::getInteger(VTy);
  }
  case IITDescriptor::VarArg:
    // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
    // should never see it here.
    llvm_unreachable("IITDescriptor::VarArg not expected");
  }
  llvm_unreachable("unhandled");
}
 
FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
                                 ArrayRef<Type *> OverloadTys) {
  SmallVector<IITDescriptor, 8> Table;
  auto [TableRef, _, IsVarArg] = getIntrinsicInfoTableEntries(id, Table);
 
  Type *ResultTy = DecodeFixedType(TableRef, OverloadTys, Context);
 
  SmallVector<Type *, 8> ArgTys;
  while (!TableRef.empty())
    ArgTys.push_back(DecodeFixedType(TableRef, OverloadTys, Context));
  return FunctionType::get(ResultTy, ArgTys, IsVarArg);
}
 
bool Intrinsic::isOverloaded(ID id) {
#define GET_INTRINSIC_OVERLOAD_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
}
 
bool Intrinsic::isTriviallyScalarizable(ID id) {
#define GET_INTRINSIC_SCALARIZABLE_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
}
 
bool Intrinsic::hasPrettyPrintedArgs(ID id){
#define GET_INTRINSIC_PRETTY_PRINT_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
}
 
/// Table of per-target intrinsic name tables.
#define GET_INTRINSIC_TARGET_DATA
#include "llvm/IR/IntrinsicImpl.inc"
 
bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
  return IID > TargetInfos[0].Count;
}
 
/// Looks up Name in NameTable via binary search. NameTable must be sorted
/// and all entries must start with "llvm.".  If NameTable contains an exact
/// match for Name or a prefix of Name followed by a dot, its index in
/// NameTable is returned. Otherwise, -1 is returned.
static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
                                     StringRef Name, StringRef Target = "") {
  assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
  assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
 
  // Do successive binary searches of the dotted name components. For
  // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
  // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
  // "llvm.gc.experimental.statepoint", and then we will stop as the range is
  // size 1. During the search, we can skip the prefix that we already know is
  // identical. By using strncmp we consider names with differing suffixes to
  // be part of the equal range.
  size_t CmpEnd = 4; // Skip the "llvm" component.
  if (!Target.empty())
    CmpEnd += 1 + Target.size(); // skip the .target component.
 
  const unsigned *Low = NameOffsetTable.begin();
  const unsigned *High = NameOffsetTable.end();
  const unsigned *LastLow = Low;
  while (CmpEnd < Name.size() && High - Low > 0) {
    size_t CmpStart = CmpEnd;
    CmpEnd = Name.find('.', CmpStart + 1);
    CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
    auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
      // `equal_range` requires the comparison to work with either side being an
      // offset or the value. Detect which kind each side is to set up the
      // compared strings.
      const char *LHSStr;
      if constexpr (std::is_integral_v<decltype(LHS)>)
        LHSStr = IntrinsicNameTable.getCString(LHS);
      else
        LHSStr = LHS;
 
      const char *RHSStr;
      if constexpr (std::is_integral_v<decltype(RHS)>)
        RHSStr = IntrinsicNameTable.getCString(RHS);
      else
        RHSStr = RHS;
 
      return strncmp(LHSStr + CmpStart, RHSStr + CmpStart, CmpEnd - CmpStart) <
             0;
    };
    LastLow = Low;
    std::tie(Low, High) = std::equal_range(Low, High, Name.data(), Cmp);
  }
  if (High - Low > 0)
    LastLow = Low;
 
  if (LastLow == NameOffsetTable.end())
    return -1;
  StringRef NameFound = IntrinsicNameTable[*LastLow];
  if (Name == NameFound ||
      (Name.starts_with(NameFound) && Name[NameFound.size()] == '.'))
    return LastLow - NameOffsetTable.begin();
  return -1;
}
 
/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
/// target as \c Name, or the generic table if \c Name is not target specific.
///
/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
/// name.
static std::pair<ArrayRef<unsigned>, StringRef>
findTargetSubtable(StringRef Name) {
  assert(Name.starts_with("llvm."));
 
  ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
  // Drop "llvm." and take the first dotted component. That will be the target
  // if this is target specific.
  StringRef Target = Name.drop_front(5).split('.').first;
  auto It = partition_point(
      Targets, [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
  // We've either found the target or just fall back to the generic set, which
  // is always first.
  const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
  return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
          TI.Name};
}
 
/// This does the actual lookup of an intrinsic ID which matches the given
/// function name.
Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
  auto [NameOffsetTable, Target] = findTargetSubtable(Name);
  int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
  if (Idx == -1)
    return Intrinsic::not_intrinsic;
 
  // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
  // an index into a sub-table.
  int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
  Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
 
  // If the intrinsic is not overloaded, require an exact match. If it is
  // overloaded, require either exact or prefix match.
  const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
  assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
  bool IsExactMatch = Name.size() == MatchSize;
  return IsExactMatch || Intrinsic::isOverloaded(ID) ? ID
                                                     : Intrinsic::not_intrinsic;
}
 
/// This defines the "Intrinsic::getAttributes(ID id)" method.
#define GET_INTRINSIC_ATTRIBUTES
#include "llvm/IR/IntrinsicImpl.inc"
 
static Function *
getOrInsertIntrinsicDeclarationImpl(Module *M, Intrinsic::ID id,
                                    ArrayRef<Type *> OverloadTys,
                                    FunctionType *FT) {
  std::string Name = OverloadTys.empty()
                         ? Intrinsic::getName(id).str()
                         : Intrinsic::getName(id, OverloadTys, M, FT);
  Function *F = cast<Function>(M->getOrInsertFunction(Name, FT).getCallee());
  if (F->getFunctionType() == FT)
    return F;
 
  // It's possible that a declaration for this intrinsic already exists with an
  // incorrect signature, if the signature has changed, but this particular
  // declaration has not been auto-upgraded yet. In that case, rename the
  // invalid declaration and insert a new one with the correct signature. The
  // invalid declaration will get upgraded later.
  F->setName(F->getName() + ".invalid");
  return cast<Function>(M->getOrInsertFunction(Name, FT).getCallee());
}
 
Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
                                            ArrayRef<Type *> OverloadTys) {
  // There can never be multiple globals with the same name of different types,
  // because intrinsics must be a specific type.
  FunctionType *FT = getType(M->getContext(), id, OverloadTys);
  return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT);
}
 
Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
                                            ArrayRef<Type *> ArgTys) {
  // If the intrinsic is not overloaded, use the non-overloaded version.
  if (!Intrinsic::isOverloaded(id))
    return getOrInsertDeclaration(M, id);
 
  // Get the intrinsic signature metadata.
  SmallVector<Intrinsic::IITDescriptor, 8> Table;
  auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id, Table);
  FunctionType *FTy = FunctionType::get(RetTy, ArgTys, IsVarArg);
 
  // Automatically determine the overloaded types.
  SmallVector<Type *, 4> OverloadTys;
  [[maybe_unused]] bool IsValid = ::isSignatureValid(
      FTy, TableRef, NumArgs, IsVarArg, OverloadTys, nulls());
  assert(IsValid && "intrinsic signature mismatch");
  return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FTy);
}
 
Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
  return M->getFunction(getName(id));
}
 
Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
                                            ArrayRef<Type *> OverloadTys,
                                            FunctionType *FT) {
  return M->getFunction(getName(id, OverloadTys, M, FT));
}
 
// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
#include "llvm/IR/IntrinsicImpl.inc"
 
// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
#include "llvm/IR/IntrinsicImpl.inc"
 
bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
  switch (QID) {
#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
  case Intrinsic::INTRINSIC:
#include "llvm/IR/ConstrainedOps.def"
#undef INSTRUCTION
    return true;
  default:
    return false;
  }
}
 
bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
  switch (QID) {
#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
  case Intrinsic::INTRINSIC:                                                   \
    return ROUND_MODE == 1;
#include "llvm/IR/ConstrainedOps.def"
#undef INSTRUCTION
  default:
    return false;
  }
}
 
// This class represents a position in the intrinsic's type signature and is
// used to generate error messages in `matchIntrinsicType`. The printed position
// can be of the following forms:
//
//   return
//   return struct element 3
//   return vector element
//   return struct element 3 vector element
//   argument 3
//   argument 3 vector element
//
// To support deferred checks also being able to generate these error messages
// we need to encode the position compactly so that it can be stashed into
// DeferredIntrinsicMatchInfo below (without materializing it into a string).
// The class below serves that purpose.
//
namespace {
struct MatchPosition {
  uint16_t IsRet : 1;
  uint16_t Num : 15; // Argument number (when IsRet = false).
  struct Index {
    uint16_t IsStruct : 1; // If true, this is a struct element with element
                           // index `Num`, else its a vector element.
    uint16_t Num : 15;     // Struct element index.
  };
  // We expect this to be just 2 levels deep, since nested structs are not
  // supported.
  static constexpr unsigned INDEX_TABLE_SIZE = 2;
  Index Indices[INDEX_TABLE_SIZE];
  uint16_t NumIndices = 0;
 
  void pop_index() {
    assert(NumIndices > 0 && "cannot pop from empty indices");
    --NumIndices;
  }
 
  void push_struct_element(unsigned ElementNum) {
    assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
    assert(isInt<15>(ElementNum) && "Element index overflow");
    Indices[NumIndices].IsStruct = true;
    Indices[NumIndices++].Num = ElementNum;
  }
 
  void push_vector_element() {
    assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
    Indices[NumIndices].IsStruct = false;
    Indices[NumIndices++].Num = 0;
  }
};
} // namespace
 
static raw_ostream &operator<<(raw_ostream &OS, const MatchPosition &Pos) {
  OS << "intrinsic ";
 
  if (Pos.IsRet)
    OS << "return";
  else
    OS << "argument " << Pos.Num;
 
  for (const MatchPosition::Index &Idx :
       ArrayRef(Pos.Indices).take_front(Pos.NumIndices)) {
    if (Idx.IsStruct)
      OS << " struct element " << Idx.Num;
    else
      OS << " vector element";
  }
  return OS;
}
 
using DeferredIntrinsicMatchInfo =
    std::tuple<Type *, ArrayRef<Intrinsic::IITDescriptor>, MatchPosition>;
 
static bool
matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
                   MatchPosition Position, SmallVectorImpl<Type *> &OverloadTys,
                   SmallVectorImpl<DeferredIntrinsicMatchInfo> &DeferredChecks,
                   bool IsDeferredCheck, raw_ostream &OS) {
  using namespace Intrinsic;
 
  // If we ran out of descriptors, there are too many arguments or returns.
  if (Infos.empty()) {
    OS << Position << " too many "
       << (Position.IsRet ? "returns" : "arguments");
    return true;
  }
 
  // Do this before slicing off the 'front' part
  auto InfosRef = Infos;
  auto DeferCheck = [&DeferredChecks, &InfosRef, &Position](Type *T) {
    DeferredChecks.emplace_back(T, InfosRef, Position);
    return false;
  };
 
  IITDescriptor D = Infos.consume_front();
 
  // Print error message when the (non-dependent) type for current position is
  // invalid.
  auto PrintMsg = [&OS, &Position,
                   Ty](bool IsValid, const Twine &Expected,
                       std::optional<unsigned> OIdx = std::nullopt) -> bool {
    if (IsValid)
      return false;
    OS << Position << " type";
    if (OIdx)
      OS << " (overload type " << *OIdx << ")";
    OS << " expected " << Expected << ", but got " << *Ty;
    return true;
  };
 
  // Print message when an overload type is invalid as a result of its use in
  // current dependent type. DependentQualifier describes the "function" applied
  // to the overload type to get the dependent type.
  auto PrintMsgInvalidOverloadTy =
      [&OS, &Position, &OverloadTys](const Twine &DependentQualifier,
                                     const Twine &Expected,
                                     unsigned OIdx) -> bool {
    OS << Position << " is " << DependentQualifier << " overload type " << OIdx
       << ", so overload type " << OIdx << " expected " << Expected
       << ", but got " << *OverloadTys[OIdx];
    return true;
  };
 
  // Print message when a dependent type is invalid.
  auto PrintMsgInvalidDepType =
      [&OS, &Position, &OverloadTys,
       Ty](bool IsValid, const Twine &DependentQualifier, const Twine &Expected,
           unsigned OIdx) -> bool {
    if (IsValid)
      return false;
    bool IsMatching = DependentQualifier.isSingleStringRef() &&
                      DependentQualifier.getSingleStringRef() == "matching";
    OS << Position << " type (" << DependentQualifier << " overload type "
       << OIdx << ") expected " << Expected;
    if (!IsMatching)
      OS << " (overload type " << OIdx << " is " << *OverloadTys[OIdx] << ")";
    OS << ", but got " << *Ty;
    return true;
  };
 
  switch (D.Kind) {
  case IITDescriptor::Void:
    assert(Position.IsRet && Position.NumIndices == 0 &&
           "void descriptor expected only for return type");
    return PrintMsg(Ty->isVoidTy(), "void");
  case IITDescriptor::MMX: {
    FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty);
    return PrintMsg(VT && VT->getNumElements() == 1 &&
                        VT->getElementType()->isIntegerTy(64),
                    "x86_mmx (<1 x i64>)");
  }
  case IITDescriptor::AMX:
    return PrintMsg(Ty->isX86_AMXTy(), "x86_amx");
  case IITDescriptor::Token:
    return PrintMsg(Ty->isTokenTy(), "token");
  case IITDescriptor::Metadata:
    return PrintMsg(Ty->isMetadataTy(), "metadata");
  case IITDescriptor::Half:
    return PrintMsg(Ty->isHalfTy(), "half");
  case IITDescriptor::BFloat:
    return PrintMsg(Ty->isBFloatTy(), "bfloat");
  case IITDescriptor::Float:
    return PrintMsg(Ty->isFloatTy(), "float");
  case IITDescriptor::Double:
    return PrintMsg(Ty->isDoubleTy(), "double");
  case IITDescriptor::Quad:
    return PrintMsg(Ty->isFP128Ty(), "fp128");
  case IITDescriptor::PPCQuad:
    return PrintMsg(Ty->isPPC_FP128Ty(), "ppc_fp128");
  case IITDescriptor::Integer:
    return PrintMsg(Ty->isIntegerTy(D.IntegerWidth),
                    "i" + Twine(D.IntegerWidth));
  case IITDescriptor::AArch64Svcount:
    return PrintMsg(isa<TargetExtType>(Ty) &&
                        cast<TargetExtType>(Ty)->getName() == "aarch64.svcount",
                    "aarch64.svcount");
  case IITDescriptor::WasmExternref:
    return PrintMsg(isa<TargetExtType>(Ty) &&
                        cast<TargetExtType>(Ty)->getName() == "wasm.externref",
                    "wasm.externref");
  case IITDescriptor::WasmFuncref:
    return PrintMsg(isa<TargetExtType>(Ty) &&
                        cast<TargetExtType>(Ty)->getName() == "wasm.funcref",
                    "wasm.funcref");
  case IITDescriptor::Vector: {
    VectorType *VT = dyn_cast<VectorType>(Ty);
    StringRef Scalable = D.VectorWidth.isScalable() ? "vscale " : "";
    bool HasError =
        PrintMsg(VT && VT->getElementCount() == D.VectorWidth,
                 Twine(Scalable) + "vector with " +
                     Twine(D.VectorWidth.getKnownMinValue()) + " elements");
    if (HasError)
      return true;
    Position.push_vector_element();
    return matchIntrinsicType(VT->getElementType(), Infos, Position,
                              OverloadTys, DeferredChecks, IsDeferredCheck, OS);
  }
  case IITDescriptor::Pointer: {
    PointerType *PT = dyn_cast<PointerType>(Ty);
    unsigned AS = D.PointerAddressSpace;
    bool IsValid = PT && PT->getAddressSpace() == AS;
    if (AS == 0)
      return PrintMsg(IsValid, "ptr");
    return PrintMsg(IsValid, "ptr addrspace(" + Twine(AS) + ")");
  }
 
  case IITDescriptor::Struct: {
    StructType *ST = dyn_cast<StructType>(Ty);
    unsigned EC = D.StructNumElements;
    bool HasError = PrintMsg(
        ST && ST->isLiteral() && !ST->isPacked() && ST->getNumElements() == EC,
        "literal non-packed struct with " + Twine(EC) + " elements");
    if (HasError)
      return true;
 
    for (const auto &[Idx, ETy] : llvm::enumerate(ST->elements())) {
      Position.push_struct_element(Idx);
      if (matchIntrinsicType(ETy, Infos, Position, OverloadTys, DeferredChecks,
                             IsDeferredCheck, OS))
        return true;
      Position.pop_index();
    }
    return false;
  }
 
  case IITDescriptor::Overloaded: {
    unsigned OIdx = D.getOverloadIndex();
    assert(OIdx == OverloadTys.size() && !IsDeferredCheck &&
           "Table consistency error");
    OverloadTys.push_back(Ty);
 
    IITDescriptor::AnyKindVectorConstraint VC;
    IITDescriptor::AnyKindElementConstraint EC;
    std::tie(VC, EC) = D.getOverloadConstraints();
 
    bool IsValid = [&]() {
      switch (VC) {
      case IITDescriptor::VC_None:
        return true;
      case IITDescriptor::VC_Vector:
        return isa<VectorType>(Ty);
      case IITDescriptor::VC_Scalar:
        return !isa<VectorType>(Ty);
      }
      llvm_unreachable("invalid vector constraint");
    }();
 
    IsValid &= [&]() {
      Type *ETy = Ty->getScalarType();
      switch (EC) {
      case IITDescriptor::EC_None:
        return true;
      case IITDescriptor::EC_Integer:
        return ETy->isIntegerTy();
      case IITDescriptor::EC_Float:
        return ETy->isFloatingPointTy();
      case IITDescriptor::EC_Pointer:
        return ETy->isPointerTy();
      }
      llvm_unreachable("invalid element constraint");
    }();
 
    if (IsValid)
      return false;
 
    static constexpr StringLiteral VectorKinds[] = {
        "",
        "vector",
        "scalar",
    };
    static constexpr StringLiteral ElementKinds[] = {
        "",
        "integer",
        "fp",
        "pointer",
    };
 
    if (EC == IITDescriptor::EC_None) {
      // No constraint on element type.
      // Expected = any {vector | scalar} type.
      StringLiteral VK = ArrayRef(VectorKinds)[VC];
      return PrintMsg(false, formatv("any {} type", VK), OIdx);
    }
 
    StringLiteral EK = ArrayRef(ElementKinds)[EC];
    switch (VC) {
    case IITDescriptor::VC_None:
      // Expected = any EK or EK vector.
      return PrintMsg(false, formatv("any {0} or {0} vector", EK), OIdx);
    case IITDescriptor::VC_Vector:
      return PrintMsg(false, formatv("any {} vector", EK), OIdx);
    case IITDescriptor::VC_Scalar:
      return PrintMsg(false, formatv("any {} type", EK), OIdx);
    }
    llvm_unreachable("invalid vector constraint");
  }
 
  case IITDescriptor::Match: {
    unsigned OIdx = D.getOverloadIndex();
    if (OIdx >= OverloadTys.size())
      return IsDeferredCheck || DeferCheck(Ty);
    return PrintMsgInvalidDepType(Ty == OverloadTys[OIdx], "matching",
                                  formatv("{}", *OverloadTys[OIdx]), OIdx);
  }
 
  case IITDescriptor::Extend:
  case IITDescriptor::Trunc: {
    unsigned OIdx = D.getOverloadIndex();
    // If this is a forward reference, defer the check for later.
    if (OIdx >= OverloadTys.size())
      return IsDeferredCheck || DeferCheck(Ty);
 
    Type *OTy = OverloadTys[OIdx];
    bool IsExtend = D.Kind == IITDescriptor::Extend;
    StringRef Qualifier = IsExtend ? "extended" : "truncated";
    if (!OTy->isIntOrIntVectorTy())
      return PrintMsgInvalidOverloadTy(Qualifier, "int or vector of int", OIdx);
 
    Type *NewTy = IsExtend ? OTy->getExtendedType() : OTy->getTruncatedType();
    return PrintMsgInvalidDepType(Ty == NewTy, Qualifier, formatv("{}", *NewTy),
                                  OIdx);
  }
  case IITDescriptor::OneNthEltsVec: {
    unsigned OIdx = D.getOverloadIndex();
    unsigned Divisor = D.getVectorDivisor();
    // If this is a forward reference, defer the check for later.
    if (OIdx >= OverloadTys.size())
      return IsDeferredCheck || DeferCheck(Ty);
    Type *OTy = OverloadTys[OIdx];
    auto *OVecTy = dyn_cast<VectorType>(OTy);
    auto Qualifier = formatv("1/nth (n={}) elements vector of", Divisor);
    if (!OVecTy)
      return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
    if (!OVecTy->getElementCount().isKnownMultipleOf(Divisor))
      return PrintMsgInvalidOverloadTy(
          Qualifier, formatv("vector with multiple of {} elements", Divisor),
          OIdx);
    Type *Expected = VectorType::getOneNthElementsVectorType(OVecTy, Divisor);
    return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
                                  formatv("{}", *Expected), OIdx);
  }
  case IITDescriptor::SameVecWidth: {
    unsigned OIdx = D.getOverloadIndex();
    if (OIdx >= OverloadTys.size()) {
      // Defer check and subsequent check for the vector element type.
      Infos.consume_front();
      return IsDeferredCheck || DeferCheck(Ty);
    }
    auto *OVecTy = dyn_cast<VectorType>(OverloadTys[OIdx]);
    auto *ThisArgVecType = dyn_cast<VectorType>(Ty);
    // Both must be vectors of the same number of elements or neither.
    StringRef Qualifier = "same vector width of";
    if (OVecTy && !ThisArgVecType)
      return PrintMsgInvalidDepType(false, Qualifier, "vector", OIdx);
    if (!OVecTy && ThisArgVecType)
      return PrintMsgInvalidDepType(false, Qualifier, "scalar", OIdx);
    Type *EltTy = Ty;
    if (ThisArgVecType) {
      ElementCount Expected = OVecTy->getElementCount();
      if (Expected != ThisArgVecType->getElementCount())
        return PrintMsgInvalidDepType(
            false, Qualifier, formatv("vector with {} elements", Expected),
            OIdx);
      EltTy = ThisArgVecType->getElementType();
      Position.push_vector_element();
    }
    return matchIntrinsicType(EltTy, Infos, Position, OverloadTys,
                              DeferredChecks, IsDeferredCheck, OS);
  }
  case IITDescriptor::VecOfAnyPtrsToElt: {
    unsigned RefOverloadIndex = D.getRefOverloadIndex();
    if (RefOverloadIndex >= OverloadTys.size()) {
      if (IsDeferredCheck)
        return true;
      // If forward referencing, already add the pointer-vector type and
      // defer the checks for later.
      assert(D.getOverloadIndex() == OverloadTys.size() &&
             "Table consistency error");
      OverloadTys.push_back(Ty);
      return DeferCheck(Ty);
    }
 
    if (!IsDeferredCheck) {
      assert(D.getOverloadIndex() == OverloadTys.size() &&
             "Table consistency error");
      OverloadTys.push_back(Ty);
    }
 
    // Verify the overloaded type "matches" the Ref type.
    // i.e. Ty is a vector with the same width as Ref and composed of pointers.
 
    StringRef Qualifier = "vector of pointers to elements of";
    auto *ReferenceType = dyn_cast<VectorType>(OverloadTys[RefOverloadIndex]);
    if (!ReferenceType)
      return PrintMsgInvalidOverloadTy(Qualifier, "vector", RefOverloadIndex);
 
    auto *ThisArgVecTy = dyn_cast<VectorType>(Ty);
    if (!ThisArgVecTy)
      return PrintMsgInvalidDepType(false, Qualifier, "vector",
                                    RefOverloadIndex);
 
    auto ExpectedCount = ReferenceType->getElementCount();
    auto Expected =
        formatv("vector of pointers with {} elements", ExpectedCount);
    bool IsValid = ThisArgVecTy->getElementCount() == ExpectedCount &&
                   ThisArgVecTy->getElementType()->isPointerTy();
    return PrintMsgInvalidDepType(IsValid, Qualifier, Expected,
                                  RefOverloadIndex);
  }
  case IITDescriptor::VecElement: {
    unsigned OIdx = D.getOverloadIndex();
    if (OIdx >= OverloadTys.size())
      return IsDeferredCheck || DeferCheck(Ty);
    StringRef Qualifier = "vector element of";
    auto *OVecTy = dyn_cast<VectorType>(OverloadTys[OIdx]);
    if (!OVecTy)
      return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
    Type *Expected = OVecTy->getElementType();
    return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
                                  formatv("{}", *Expected), OIdx);
  }
  case IITDescriptor::Subdivide2:
  case IITDescriptor::Subdivide4: {
    unsigned OIdx = D.getOverloadIndex();
    // If this is a forward reference, defer the check for later.
    if (OIdx >= OverloadTys.size())
      return IsDeferredCheck || DeferCheck(Ty);
 
    int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
    auto *OVecTy = dyn_cast<VectorType>(OverloadTys[OIdx]);
    auto Qualifier =
        formatv("subdivided by {} vector of", SubDivs == 1 ? 2 : 4);
    if (!OVecTy)
      return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
 
    // TODO: Verify that the element type of the overload type is subdivisible
    // by 2 or 4.
    Type *Expected = VectorType::getSubdividedVectorType(OVecTy, SubDivs);
    return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
                                  formatv("{}", *Expected), OIdx);
  }
  case IITDescriptor::VecOfBitcastsToInt: {
    unsigned OIdx = D.getOverloadIndex();
    if (OIdx >= OverloadTys.size())
      return IsDeferredCheck || DeferCheck(Ty);
    auto *OVecTy = dyn_cast<VectorType>(OverloadTys[OIdx]);
    StringRef Qualifier = "vector of bitcasts to int of";
    if (!OVecTy)
      return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
    Type *Expected = VectorType::getInteger(OVecTy);
    return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
                                  formatv("{}", *Expected), OIdx);
  }
  case IITDescriptor::VarArg:
    // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
    // should never see it here.
    llvm_unreachable("IITDescriptor::VarArg not expected");
  }
  llvm_unreachable("unhandled");
}
 
/// Return true if the function type \p FTy is a valid type signature for the
/// type constraints specified in the .td file, represented by \p Infos and
/// \p IsVarArg. The overloaded types for the intrinsic are pushed to the
/// \p OverloadTys vector.
///
/// If the type is not valid, returns false and prints an error message to
/// \p OS.
static bool isSignatureValid(FunctionType *FTy,
                             ArrayRef<Intrinsic::IITDescriptor> &Infos,
                             unsigned NumArgs, bool IsVarArg,
                             SmallVectorImpl<Type *> &OverloadTys,
                             raw_ostream &OS) {
  SmallVector<DeferredIntrinsicMatchInfo, 2> DeferredChecks;
 
  assert(!Infos.empty() && "Table consistency error");
 
  MatchPosition Pos;
  Pos.IsRet = true;
  Pos.Num = 0;
 
  if (matchIntrinsicType(FTy->getReturnType(), Infos, Pos, OverloadTys,
                         DeferredChecks, false, OS))
    return false;
 
  if (FTy->getNumParams() != NumArgs) {
    OS << "intrinsic has incorrect number of args. Expected " << NumArgs
       << ", but got " << FTy->getNumParams();
    return false;
  }
 
  Pos.IsRet = false;
  for (const auto &[Idx, Ty] : llvm::enumerate(FTy->params())) {
    Pos.Num = Idx;
    if (matchIntrinsicType(Ty, Infos, Pos, OverloadTys, DeferredChecks, false,
                           OS))
      return false;
  }
 
  for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
    auto &[DefTy, DefInfos, DefPosition] = DeferredChecks[I];
    if (matchIntrinsicType(DefTy, DefInfos, DefPosition, OverloadTys,
                           DeferredChecks, true, OS))
      return false;
  }
 
  if (!Infos.empty()) {
    OS << "intrinsic has too few arguments!";
    return false;
  }
 
  if (FTy->isVarArg() != IsVarArg) {
    if (IsVarArg)
      OS << "intrinsic was not defined with variable arguments!";
    else
      OS << "intrinsic was defined with variable arguments!";
    return false;
  }
 
  return true;
}
 
bool Intrinsic::hasStructReturnType(ID id) {
  using namespace Intrinsic;
  SmallVector<IITDescriptor> Table;
  getIntrinsicInfoTableEntries(id, Table);
  return !Table.empty() && Table[0].Kind == IITDescriptor::Struct;
}
 
bool Intrinsic::isSignatureValid(Intrinsic::ID ID, FunctionType *FT,
                                 SmallVectorImpl<Type *> &OverloadTys,
                                 raw_ostream &OS) {
  if (!ID)
    return false;
 
  SmallVector<Intrinsic::IITDescriptor, 8> Table;
  auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(ID, Table);
 
  return ::isSignatureValid(FT, TableRef, NumArgs, IsVarArg, OverloadTys, OS);
}
 
bool Intrinsic::isSignatureValid(Function *F,
                                 SmallVectorImpl<Type *> &OverloadTys,
                                 raw_ostream &OS) {
  return isSignatureValid(F->getIntrinsicID(), F->getFunctionType(),
                          OverloadTys, OS);
}
 
std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
  SmallVector<Type *, 4> OverloadTys;
  if (!isSignatureValid(F, OverloadTys))
    return std::nullopt;
 
  Intrinsic::ID ID = F->getIntrinsicID();
  StringRef Name = F->getName();
  std::string WantedName =
      Intrinsic::getName(ID, OverloadTys, F->getParent(), F->getFunctionType());
  if (Name == WantedName)
    return std::nullopt;
 
  Function *NewDecl = [&] {
    if (auto *ExistingGV = F->getParent()->getNamedValue(WantedName)) {
      if (auto *ExistingF = dyn_cast<Function>(ExistingGV))
        if (ExistingF->getFunctionType() == F->getFunctionType())
          return ExistingF;
 
      // The name already exists, but is not a function or has the wrong
      // prototype. Make place for the new one by renaming the old version.
      // Either this old version will be removed later on or the module is
      // invalid and we'll get an error.
      ExistingGV->setName(WantedName + ".renamed");
    }
    return Intrinsic::getOrInsertDeclaration(F->getParent(), ID, OverloadTys);
  }();
 
  NewDecl->setCallingConv(F->getCallingConv());
  assert(NewDecl->getFunctionType() == F->getFunctionType() &&
         "Shouldn't change the signature");
  return NewDecl;
}
 
struct InterleaveIntrinsic {
  Intrinsic::ID Interleave, Deinterleave;
};
 
static InterleaveIntrinsic InterleaveIntrinsics[] = {
    {Intrinsic::vector_interleave2, Intrinsic::vector_deinterleave2},
    {Intrinsic::vector_interleave3, Intrinsic::vector_deinterleave3},
    {Intrinsic::vector_interleave4, Intrinsic::vector_deinterleave4},
    {Intrinsic::vector_interleave5, Intrinsic::vector_deinterleave5},
    {Intrinsic::vector_interleave6, Intrinsic::vector_deinterleave6},
    {Intrinsic::vector_interleave7, Intrinsic::vector_deinterleave7},
    {Intrinsic::vector_interleave8, Intrinsic::vector_deinterleave8},
};
 
Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
  assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
  return InterleaveIntrinsics[Factor - 2].Interleave;
}
 
Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
  assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
  return InterleaveIntrinsics[Factor - 2].Deinterleave;
}
 
#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
#include "llvm/IR/IntrinsicImpl.inc"
 
// Emit the default-argument values table and lookup function
// (Intrinsic::getAllDefaultArgValues).
#define GET_INTRINSIC_DEFAULT_ARG_VALUES
#include "llvm/IR/IntrinsicImpl.inc"