SPIRVCallLowering.cpp

Go to the documentation of this file.

//===--- SPIRVCallLowering.cpp - Call lowering ------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the lowering of LLVM calls to machine code calls for
// GlobalISel.
//
//===----------------------------------------------------------------------===//
 
#include "SPIRVCallLowering.h"
#include "MCTargetDesc/SPIRVBaseInfo.h"
#include "SPIRV.h"
#include "SPIRVBuiltins.h"
#include "SPIRVGlobalRegistry.h"
#include "SPIRVISelLowering.h"
#include "SPIRVMetadata.h"
#include "SPIRVRegisterInfo.h"
#include "SPIRVSubtarget.h"
#include "SPIRVUtils.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/Support/ModRef.h"
 
using namespace llvm;
 
SPIRVCallLowering::SPIRVCallLowering(const SPIRVTargetLowering &TLI,
                                     SPIRVGlobalRegistry *GR)
    : CallLowering(&TLI), GR(GR) {}
 
bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
                                    const Value *Val, ArrayRef<Register> VRegs,
                                    FunctionLoweringInfo &FLI,
                                    Register SwiftErrorVReg) const {
  // Ignore if called from the internal service function
  if (MIRBuilder.getMF()
          .getFunction()
          .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
          .isValid())
    return true;
 
  // Maybe run postponed production of types for function pointers
  if (IndirectCalls.size() > 0) {
    produceIndirectPtrTypes(MIRBuilder);
    IndirectCalls.clear();
  }
 
  // Currently all return types should use a single register.
  // TODO: handle the case of multiple registers.
  if (VRegs.size() > 1)
    return false;
  if (Val) {
    const auto &STI = MIRBuilder.getMF().getSubtarget();
    return MIRBuilder.buildInstr(SPIRV::OpReturnValue)
        .addUse(VRegs[0])
        .constrainAllUses(MIRBuilder.getTII(), *STI.getRegisterInfo(),
                          *STI.getRegBankInfo());
  }
  MIRBuilder.buildInstr(SPIRV::OpReturn);
  return true;
}
 
// Based on the LLVM function attributes, get a SPIR-V FunctionControl.
static uint32_t getFunctionControl(const Function &F,
                                   const SPIRVSubtarget *ST) {
  MemoryEffects MemEffects = F.getMemoryEffects();
 
  uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None);
 
  if (F.hasFnAttribute(Attribute::AttrKind::NoInline))
    FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::DontInline);
  else if (F.hasFnAttribute(Attribute::AttrKind::AlwaysInline))
    FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Inline);
 
  if (MemEffects.doesNotAccessMemory())
    FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Pure);
  else if (MemEffects.onlyReadsMemory())
    FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const);
 
  if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_optnone) ||
      ST->canUseExtension(SPIRV::Extension::SPV_EXT_optnone))
    if (F.hasFnAttribute(Attribute::OptimizeNone))
      FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::OptNoneEXT);
 
  return FuncControl;
}
 
static ConstantInt *getConstInt(MDNode *MD, unsigned NumOp) {
  if (MD->getNumOperands() > NumOp) {
    auto *CMeta = dyn_cast<ConstantAsMetadata>(MD->getOperand(NumOp));
    if (CMeta)
      return dyn_cast<ConstantInt>(CMeta->getValue());
  }
  return nullptr;
}
 
// If the function has pointer arguments, we are forced to re-create this
// function type from the very beginning, changing PointerType by
// TypedPointerType for each pointer argument. Otherwise, the same `Type*`
// potentially corresponds to different SPIR-V function type, effectively
// invalidating logic behind global registry and duplicates tracker.
static FunctionType *
fixFunctionTypeIfPtrArgs(SPIRVGlobalRegistry *GR, const Function &F,
                         FunctionType *FTy, const SPIRVType *SRetTy,
                         const SmallVector<SPIRVType *, 4> &SArgTys) {
  bool hasArgPtrs = false;
  for (auto &Arg : F.args()) {
    // check if it's an instance of a non-typed PointerType
    if (Arg.getType()->isPointerTy()) {
      hasArgPtrs = true;
      break;
    }
  }
  if (!hasArgPtrs) {
    Type *RetTy = FTy->getReturnType();
    // check if it's an instance of a non-typed PointerType
    if (!RetTy->isPointerTy())
      return FTy;
  }
 
  // re-create function type, using TypedPointerType instead of PointerType to
  // properly trace argument types
  const Type *RetTy = GR->getTypeForSPIRVType(SRetTy);
  SmallVector<Type *, 4> ArgTys;
  for (auto SArgTy : SArgTys)
    ArgTys.push_back(const_cast<Type *>(GR->getTypeForSPIRVType(SArgTy)));
  return FunctionType::get(const_cast<Type *>(RetTy), ArgTys, false);
}
 
// This code restores function args/retvalue types for composite cases
// because the final types should still be aggregate whereas they're i32
// during the translation to cope with aggregate flattening etc.
static FunctionType *getOriginalFunctionType(const Function &F) {
  auto *NamedMD = F.getParent()->getNamedMetadata("spv.cloned_funcs");
  if (NamedMD == nullptr)
    return F.getFunctionType();
 
  Type *RetTy = F.getFunctionType()->getReturnType();
  SmallVector<Type *, 4> ArgTypes;
  for (auto &Arg : F.args())
    ArgTypes.push_back(Arg.getType());
 
  auto ThisFuncMDIt =
      std::find_if(NamedMD->op_begin(), NamedMD->op_end(), [&F](MDNode *N) {
        return isa<MDString>(N->getOperand(0)) &&
               cast<MDString>(N->getOperand(0))->getString() == F.getName();
      });
  // TODO: probably one function can have numerous type mutations,
  // so we should support this.
  if (ThisFuncMDIt != NamedMD->op_end()) {
    auto *ThisFuncMD = *ThisFuncMDIt;
    MDNode *MD = dyn_cast<MDNode>(ThisFuncMD->getOperand(1));
    assert(MD && "MDNode operand is expected");
    ConstantInt *Const = getConstInt(MD, 0);
    if (Const) {
      auto *CMeta = dyn_cast<ConstantAsMetadata>(MD->getOperand(1));
      assert(CMeta && "ConstantAsMetadata operand is expected");
      assert(Const->getSExtValue() >= -1);
      // Currently -1 indicates return value, greater values mean
      // argument numbers.
      if (Const->getSExtValue() == -1)
        RetTy = CMeta->getType();
      else
        ArgTypes[Const->getSExtValue()] = CMeta->getType();
    }
  }
 
  return FunctionType::get(RetTy, ArgTypes, F.isVarArg());
}
 
static SPIRV::AccessQualifier::AccessQualifier
getArgAccessQual(const Function &F, unsigned ArgIdx) {
  if (F.getCallingConv() != CallingConv::SPIR_KERNEL)
    return SPIRV::AccessQualifier::ReadWrite;
 
  MDString *ArgAttribute = getOCLKernelArgAccessQual(F, ArgIdx);
  if (!ArgAttribute)
    return SPIRV::AccessQualifier::ReadWrite;
 
  if (ArgAttribute->getString() == "read_only")
    return SPIRV::AccessQualifier::ReadOnly;
  if (ArgAttribute->getString() == "write_only")
    return SPIRV::AccessQualifier::WriteOnly;
  return SPIRV::AccessQualifier::ReadWrite;
}
 
static std::vector<SPIRV::Decoration::Decoration>
getKernelArgTypeQual(const Function &F, unsigned ArgIdx) {
  MDString *ArgAttribute = getOCLKernelArgTypeQual(F, ArgIdx);
  if (ArgAttribute && ArgAttribute->getString() == "volatile")
    return {SPIRV::Decoration::Volatile};
  return {};
}
 
static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx,
                                  SPIRVGlobalRegistry *GR,
                                  MachineIRBuilder &MIRBuilder,
                                  const SPIRVSubtarget &ST) {
  // Read argument's access qualifier from metadata or default.
  SPIRV::AccessQualifier::AccessQualifier ArgAccessQual =
      getArgAccessQual(F, ArgIdx);
 
  Type *OriginalArgType = getOriginalFunctionType(F)->getParamType(ArgIdx);
 
  // If OriginalArgType is non-pointer, use the OriginalArgType (the type cannot
  // be legally reassigned later).
  if (!isPointerTy(OriginalArgType))
    return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual,
                                    true);
 
  Argument *Arg = F.getArg(ArgIdx);
  Type *ArgType = Arg->getType();
  if (isTypedPointerTy(ArgType)) {
    return GR->getOrCreateSPIRVPointerType(
        cast<TypedPointerType>(ArgType)->getElementType(), MIRBuilder,
        addressSpaceToStorageClass(getPointerAddressSpace(ArgType), ST));
  }
 
  // In case OriginalArgType is of untyped pointer type, there are three
  // possibilities:
  // 1) This is a pointer of an LLVM IR element type, passed byval/byref.
  // 2) This is an OpenCL/SPIR-V builtin type if there is spv_assign_type
  //    intrinsic assigning a TargetExtType.
  // 3) This is a pointer, try to retrieve pointer element type from a
  // spv_assign_ptr_type intrinsic or otherwise use default pointer element
  // type.
  if (hasPointeeTypeAttr(Arg)) {
    return GR->getOrCreateSPIRVPointerType(
        getPointeeTypeByAttr(Arg), MIRBuilder,
        addressSpaceToStorageClass(getPointerAddressSpace(ArgType), ST));
  }
 
  for (auto User : Arg->users()) {
    auto *II = dyn_cast<IntrinsicInst>(User);
    // Check if this is spv_assign_type assigning OpenCL/SPIR-V builtin type.
    if (II && II->getIntrinsicID() == Intrinsic::spv_assign_type) {
      MetadataAsValue *VMD = cast<MetadataAsValue>(II->getOperand(1));
      Type *BuiltinType =
          cast<ConstantAsMetadata>(VMD->getMetadata())->getType();
      assert(BuiltinType->isTargetExtTy() && "Expected TargetExtType");
      return GR->getOrCreateSPIRVType(BuiltinType, MIRBuilder, ArgAccessQual,
                                      true);
    }
 
    // Check if this is spv_assign_ptr_type assigning pointer element type.
    if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type)
      continue;
 
    MetadataAsValue *VMD = cast<MetadataAsValue>(II->getOperand(1));
    Type *ElementTy =
        toTypedPointer(cast<ConstantAsMetadata>(VMD->getMetadata())->getType());
    return GR->getOrCreateSPIRVPointerType(
        ElementTy, MIRBuilder,
        addressSpaceToStorageClass(
            cast<ConstantInt>(II->getOperand(2))->getZExtValue(), ST));
  }
 
  // Replace PointerType with TypedPointerType to be able to map SPIR-V types to
  // LLVM types in a consistent manner
  return GR->getOrCreateSPIRVType(toTypedPointer(OriginalArgType), MIRBuilder,
                                  ArgAccessQual, true);
}
 
static SPIRV::ExecutionModel::ExecutionModel
getExecutionModel(const SPIRVSubtarget &STI, const Function &F) {
  if (STI.isKernel())
    return SPIRV::ExecutionModel::Kernel;
 
  if (STI.isShader()) {
    auto attribute = F.getFnAttribute("hlsl.shader");
    if (!attribute.isValid()) {
      report_fatal_error(
          "This entry point lacks mandatory hlsl.shader attribute.");
    }
 
    const auto value = attribute.getValueAsString();
    if (value == "compute")
      return SPIRV::ExecutionModel::GLCompute;
    if (value == "vertex")
      return SPIRV::ExecutionModel::Vertex;
    if (value == "pixel")
      return SPIRV::ExecutionModel::Fragment;
 
    report_fatal_error(
        "This HLSL entry point is not supported by this backend.");
  }
 
  assert(STI.getEnv() == SPIRVSubtarget::Unknown);
  // "hlsl.shader" attribute is mandatory for Vulkan, so we can set Env to
  // Shader whenever we find it, and to Kernel otherwise.
 
  // We will now change the Env based on the attribute, so we need to strip
  // `const` out of the ref to STI.
  SPIRVSubtarget *NonConstSTI = const_cast<SPIRVSubtarget *>(&STI);
  auto attribute = F.getFnAttribute("hlsl.shader");
  if (!attribute.isValid()) {
    NonConstSTI->setEnv(SPIRVSubtarget::Kernel);
    return SPIRV::ExecutionModel::Kernel;
  }
  NonConstSTI->setEnv(SPIRVSubtarget::Shader);
 
  const auto value = attribute.getValueAsString();
  if (value == "compute")
    return SPIRV::ExecutionModel::GLCompute;
  if (value == "vertex")
    return SPIRV::ExecutionModel::Vertex;
  if (value == "pixel")
    return SPIRV::ExecutionModel::Fragment;
 
  report_fatal_error("This HLSL entry point is not supported by this backend.");
}
 
bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
                                             const Function &F,
                                             ArrayRef<ArrayRef<Register>> VRegs,
                                             FunctionLoweringInfo &FLI) const {
  // Discard the internal service function
  if (F.getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME).isValid())
    return true;
 
  assert(GR && "Must initialize the SPIRV type registry before lowering args.");
  GR->setCurrentFunc(MIRBuilder.getMF());
 
  // Get access to information about available extensions
  const SPIRVSubtarget *ST =
      static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
 
  // Assign types and names to all args, and store their types for later.
  SmallVector<SPIRVType *, 4> ArgTypeVRegs;
  if (VRegs.size() > 0) {
    unsigned i = 0;
    for (const auto &Arg : F.args()) {
      // Currently formal args should use single registers.
      // TODO: handle the case of multiple registers.
      if (VRegs[i].size() > 1)
        return false;
      auto *SpirvTy = getArgSPIRVType(F, i, GR, MIRBuilder, *ST);
      GR->assignSPIRVTypeToVReg(SpirvTy, VRegs[i][0], MIRBuilder.getMF());
      ArgTypeVRegs.push_back(SpirvTy);
 
      if (Arg.hasName())
        buildOpName(VRegs[i][0], Arg.getName(), MIRBuilder);
      if (isPointerTyOrWrapper(Arg.getType())) {
        auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes());
        if (DerefBytes != 0)
          buildOpDecorate(VRegs[i][0], MIRBuilder,
                          SPIRV::Decoration::MaxByteOffset, {DerefBytes});
      }
      if (Arg.hasAttribute(Attribute::Alignment) && !ST->isShader()) {
        auto Alignment = static_cast<unsigned>(
            Arg.getAttribute(Attribute::Alignment).getValueAsInt());
        buildOpDecorate(VRegs[i][0], MIRBuilder, SPIRV::Decoration::Alignment,
                        {Alignment});
      }
      if (Arg.hasAttribute(Attribute::ReadOnly)) {
        auto Attr =
            static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoWrite);
        buildOpDecorate(VRegs[i][0], MIRBuilder,
                        SPIRV::Decoration::FuncParamAttr, {Attr});
      }
      if (Arg.hasAttribute(Attribute::ZExt)) {
        auto Attr =
            static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext);
        buildOpDecorate(VRegs[i][0], MIRBuilder,
                        SPIRV::Decoration::FuncParamAttr, {Attr});
      }
      if (Arg.hasAttribute(Attribute::NoAlias)) {
        auto Attr =
            static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoAlias);
        buildOpDecorate(VRegs[i][0], MIRBuilder,
                        SPIRV::Decoration::FuncParamAttr, {Attr});
      }
      if (Arg.hasAttribute(Attribute::ByVal)) {
        auto Attr =
            static_cast<unsigned>(SPIRV::FunctionParameterAttribute::ByVal);
        buildOpDecorate(VRegs[i][0], MIRBuilder,
                        SPIRV::Decoration::FuncParamAttr, {Attr});
      }
      if (Arg.hasAttribute(Attribute::StructRet)) {
        auto Attr =
            static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sret);
        buildOpDecorate(VRegs[i][0], MIRBuilder,
                        SPIRV::Decoration::FuncParamAttr, {Attr});
      }
 
      if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
        std::vector<SPIRV::Decoration::Decoration> ArgTypeQualDecs =
            getKernelArgTypeQual(F, i);
        for (SPIRV::Decoration::Decoration Decoration : ArgTypeQualDecs)
          buildOpDecorate(VRegs[i][0], MIRBuilder, Decoration, {});
      }
 
      MDNode *Node = F.getMetadata("spirv.ParameterDecorations");
      if (Node && i < Node->getNumOperands() &&
          isa<MDNode>(Node->getOperand(i))) {
        MDNode *MD = cast<MDNode>(Node->getOperand(i));
        for (const MDOperand &MDOp : MD->operands()) {
          MDNode *MD2 = dyn_cast<MDNode>(MDOp);
          assert(MD2 && "Metadata operand is expected");
          ConstantInt *Const = getConstInt(MD2, 0);
          assert(Const && "MDOperand should be ConstantInt");
          auto Dec =
              static_cast<SPIRV::Decoration::Decoration>(Const->getZExtValue());
          std::vector<uint32_t> DecVec;
          for (unsigned j = 1; j < MD2->getNumOperands(); j++) {
            ConstantInt *Const = getConstInt(MD2, j);
            assert(Const && "MDOperand should be ConstantInt");
            DecVec.push_back(static_cast<uint32_t>(Const->getZExtValue()));
          }
          buildOpDecorate(VRegs[i][0], MIRBuilder, Dec, DecVec);
        }
      }
      ++i;
    }
  }
 
  auto MRI = MIRBuilder.getMRI();
  Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
  MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass);
  FunctionType *FTy = getOriginalFunctionType(F);
  Type *FRetTy = FTy->getReturnType();
  if (isUntypedPointerTy(FRetTy)) {
    if (Type *FRetElemTy = GR->findDeducedElementType(&F)) {
      TypedPointerType *DerivedTy = TypedPointerType::get(
          toTypedPointer(FRetElemTy), getPointerAddressSpace(FRetTy));
      GR->addReturnType(&F, DerivedTy);
      FRetTy = DerivedTy;
    }
  }
  SPIRVType *RetTy = GR->getOrCreateSPIRVType(
      FRetTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
  FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, RetTy, ArgTypeVRegs);
  SPIRVType *FuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
      FTy, RetTy, ArgTypeVRegs, MIRBuilder);
  uint32_t FuncControl = getFunctionControl(F, ST);
 
  // Add OpFunction instruction
  MachineInstrBuilder MB = MIRBuilder.buildInstr(SPIRV::OpFunction)
                               .addDef(FuncVReg)
                               .addUse(GR->getSPIRVTypeID(RetTy))
                               .addImm(FuncControl)
                               .addUse(GR->getSPIRVTypeID(FuncTy));
  GR->recordFunctionDefinition(&F, &MB.getInstr()->getOperand(0));
  GR->addGlobalObject(&F, &MIRBuilder.getMF(), FuncVReg);
  if (F.isDeclaration())
    GR->add(&F, MB);
 
  // Add OpFunctionParameter instructions
  int i = 0;
  for (const auto &Arg : F.args()) {
    assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs");
    Register ArgReg = VRegs[i][0];
    MRI->setRegClass(ArgReg, GR->getRegClass(ArgTypeVRegs[i]));
    MRI->setType(ArgReg, GR->getRegType(ArgTypeVRegs[i]));
    auto MIB = MIRBuilder.buildInstr(SPIRV::OpFunctionParameter)
                   .addDef(ArgReg)
                   .addUse(GR->getSPIRVTypeID(ArgTypeVRegs[i]));
    if (F.isDeclaration())
      GR->add(&Arg, MIB);
    GR->addGlobalObject(&Arg, &MIRBuilder.getMF(), ArgReg);
    i++;
  }
  // Name the function.
  if (F.hasName())
    buildOpName(FuncVReg, F.getName(), MIRBuilder);
 
  // Handle entry points and function linkage.
  if (isEntryPoint(F)) {
    // EntryPoints can help us to determine the environment we're working on.
    // Therefore, we need a non-const pointer to SPIRVSubtarget to update the
    // environment if we need to.
    const SPIRVSubtarget *ST =
        static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
    auto MIB = MIRBuilder.buildInstr(SPIRV::OpEntryPoint)
                   .addImm(static_cast<uint32_t>(getExecutionModel(*ST, F)))
                   .addUse(FuncVReg);
    addStringImm(F.getName(), MIB);
  } else if (const auto LnkTy = getSpirvLinkageTypeFor(*ST, F)) {
    buildOpDecorate(FuncVReg, MIRBuilder, SPIRV::Decoration::LinkageAttributes,
                    {static_cast<uint32_t>(*LnkTy)}, F.getName());
  }
 
  // Handle function pointers decoration
  bool hasFunctionPointers =
      ST->canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers);
  if (hasFunctionPointers) {
    if (F.hasFnAttribute("referenced-indirectly")) {
      assert((F.getCallingConv() != CallingConv::SPIR_KERNEL) &&
             "Unexpected 'referenced-indirectly' attribute of the kernel "
             "function");
      buildOpDecorate(FuncVReg, MIRBuilder,
                      SPIRV::Decoration::ReferencedIndirectlyINTEL, {});
    }
  }
 
  return true;
}
 
// Used to postpone producing of indirect function pointer types after all
// indirect calls info is collected
// TODO:
// - add a topological sort of IndirectCalls to ensure the best types knowledge
// - we may need to fix function formal parameter types if they are opaque
//   pointers used as function pointers in these indirect calls
void SPIRVCallLowering::produceIndirectPtrTypes(
    MachineIRBuilder &MIRBuilder) const {
  // Create indirect call data types if any
  MachineFunction &MF = MIRBuilder.getMF();
  for (auto const &IC : IndirectCalls) {
    SPIRVType *SpirvRetTy = GR->getOrCreateSPIRVType(
        IC.RetTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
    SmallVector<SPIRVType *, 4> SpirvArgTypes;
    for (size_t i = 0; i < IC.ArgTys.size(); ++i) {
      SPIRVType *SPIRVTy = GR->getOrCreateSPIRVType(
          IC.ArgTys[i], MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
      SpirvArgTypes.push_back(SPIRVTy);
      if (!GR->getSPIRVTypeForVReg(IC.ArgRegs[i]))
        GR->assignSPIRVTypeToVReg(SPIRVTy, IC.ArgRegs[i], MF);
    }
    // SPIR-V function type:
    FunctionType *FTy =
        FunctionType::get(const_cast<Type *>(IC.RetTy), IC.ArgTys, false);
    SPIRVType *SpirvFuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
        FTy, SpirvRetTy, SpirvArgTypes, MIRBuilder);
    // SPIR-V pointer to function type:
    SPIRVType *IndirectFuncPtrTy = GR->getOrCreateSPIRVPointerType(
        SpirvFuncTy, MIRBuilder, SPIRV::StorageClass::Function);
    // Correct the Callee type
    GR->assignSPIRVTypeToVReg(IndirectFuncPtrTy, IC.Callee, MF);
  }
}
 
bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
                                  CallLoweringInfo &Info) const {
  // Currently call returns should have single vregs.
  // TODO: handle the case of multiple registers.
  if (Info.OrigRet.Regs.size() > 1)
    return false;
  MachineFunction &MF = MIRBuilder.getMF();
  GR->setCurrentFunc(MF);
  const Function *CF = nullptr;
  std::string DemangledName;
  const Type *OrigRetTy = Info.OrigRet.Ty;
 
  // Emit a regular OpFunctionCall. If it's an externally declared function,
  // be sure to emit its type and function declaration here. It will be hoisted
  // globally later.
  if (Info.Callee.isGlobal()) {
    std::string FuncName = Info.Callee.getGlobal()->getName().str();
    DemangledName = getOclOrSpirvBuiltinDemangledName(FuncName);
    CF = dyn_cast_or_null<const Function>(Info.Callee.getGlobal());
    // TODO: support constexpr casts and indirect calls.
    if (CF == nullptr)
      return false;
    if (FunctionType *FTy = getOriginalFunctionType(*CF)) {
      OrigRetTy = FTy->getReturnType();
      if (isUntypedPointerTy(OrigRetTy)) {
        if (auto *DerivedRetTy = GR->findReturnType(CF))
          OrigRetTy = DerivedRetTy;
      }
    }
  }
 
  MachineRegisterInfo *MRI = MIRBuilder.getMRI();
  Register ResVReg =
      Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0];
  const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
 
  bool isFunctionDecl = CF && CF->isDeclaration();
  if (isFunctionDecl && !DemangledName.empty()) {
    if (ResVReg.isValid()) {
      if (!GR->getSPIRVTypeForVReg(ResVReg)) {
        const Type *RetTy = OrigRetTy;
        if (auto *PtrRetTy = dyn_cast<PointerType>(OrigRetTy)) {
          const Value *OrigValue = Info.OrigRet.OrigValue;
          if (!OrigValue)
            OrigValue = Info.CB;
          if (OrigValue)
            if (Type *ElemTy = GR->findDeducedElementType(OrigValue))
              RetTy =
                  TypedPointerType::get(ElemTy, PtrRetTy->getAddressSpace());
        }
        setRegClassType(ResVReg, RetTy, GR, MIRBuilder,
                        SPIRV::AccessQualifier::ReadWrite, true);
      }
    } else {
      ResVReg = createVirtualRegister(OrigRetTy, GR, MIRBuilder,
                                      SPIRV::AccessQualifier::ReadWrite, true);
    }
    SmallVector<Register, 8> ArgVRegs;
    for (auto Arg : Info.OrigArgs) {
      assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
      Register ArgReg = Arg.Regs[0];
      ArgVRegs.push_back(ArgReg);
      SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg);
      if (!SpvType) {
        Type *ArgTy = nullptr;
        if (auto *PtrArgTy = dyn_cast<PointerType>(Arg.Ty)) {
          // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we
          // don't have access to original value in LLVM IR or info about
          // deduced pointee type, then we should wait with setting the type for
          // the virtual register until pre-legalizer step when we access
          // @llvm.spv.assign.ptr.type.p...(...)'s info.
          if (Arg.OrigValue)
            if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue))
              ArgTy =
                  TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
        } else {
          ArgTy = Arg.Ty;
        }
        if (ArgTy) {
          SpvType = GR->getOrCreateSPIRVType(
              ArgTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
          GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
        }
      }
      if (!MRI->getRegClassOrNull(ArgReg)) {
        // Either we have SpvType created, or Arg.Ty is an untyped pointer and
        // we know its virtual register's class and type even if we don't know
        // pointee type.
        MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
                                         : &SPIRV::pIDRegClass);
        MRI->setType(
            ArgReg,
            SpvType ? GR->getRegType(SpvType)
                    : LLT::pointer(cast<PointerType>(Arg.Ty)->getAddressSpace(),
                                   GR->getPointerSize()));
      }
    }
    if (auto Res = SPIRV::lowerBuiltin(
            DemangledName, ST->getPreferredInstructionSet(), MIRBuilder,
            ResVReg, OrigRetTy, ArgVRegs, GR, *Info.CB))
      return *Res;
  }
 
  if (isFunctionDecl && !GR->find(CF, &MF).isValid()) {
    // Emit the type info and forward function declaration to the first MBB
    // to ensure VReg definition dependencies are valid across all MBBs.
    MachineIRBuilder FirstBlockBuilder;
    FirstBlockBuilder.setMF(MF);
    FirstBlockBuilder.setMBB(*MF.getBlockNumbered(0));
 
    SmallVector<ArrayRef<Register>, 8> VRegArgs;
    SmallVector<SmallVector<Register, 1>, 8> ToInsert;
    for (const Argument &Arg : CF->args()) {
      if (MIRBuilder.getDataLayout().getTypeStoreSize(Arg.getType()).isZero())
        continue; // Don't handle zero sized types.
      Register Reg = MRI->createGenericVirtualRegister(LLT::scalar(64));
      MRI->setRegClass(Reg, &SPIRV::iIDRegClass);
      ToInsert.push_back({Reg});
      VRegArgs.push_back(ToInsert.back());
    }
    // TODO: Reuse FunctionLoweringInfo
    FunctionLoweringInfo FuncInfo;
    lowerFormalArguments(FirstBlockBuilder, *CF, VRegArgs, FuncInfo);
  }
 
  // Ignore the call if it's called from the internal service function
  if (MIRBuilder.getMF()
          .getFunction()
          .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
          .isValid()) {
    // insert a no-op
    MIRBuilder.buildTrap();
    return true;
  }
 
  unsigned CallOp;
  if (Info.CB->isIndirectCall()) {
    if (!ST->canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers))
      report_fatal_error("An indirect call is encountered but SPIR-V without "
                         "extensions does not support it",
                         false);
    // Set instruction operation according to SPV_INTEL_function_pointers
    CallOp = SPIRV::OpFunctionPointerCallINTEL;
    // Collect information about the indirect call to support possible
    // specification of opaque ptr types of parent function's parameters
    Register CalleeReg = Info.Callee.getReg();
    if (CalleeReg.isValid()) {
      SPIRVCallLowering::SPIRVIndirectCall IndirectCall;
      IndirectCall.Callee = CalleeReg;
      IndirectCall.RetTy = OrigRetTy;
      for (const auto &Arg : Info.OrigArgs) {
        assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
        IndirectCall.ArgTys.push_back(Arg.Ty);
        IndirectCall.ArgRegs.push_back(Arg.Regs[0]);
      }
      IndirectCalls.push_back(IndirectCall);
    }
  } else {
    // Emit a regular OpFunctionCall
    CallOp = SPIRV::OpFunctionCall;
  }
 
  // Make sure there's a valid return reg, even for functions returning void.
  if (!ResVReg.isValid())
    ResVReg = MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
  SPIRVType *RetType = GR->assignTypeToVReg(
      OrigRetTy, ResVReg, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
 
  // Emit the call instruction and its args.
  auto MIB = MIRBuilder.buildInstr(CallOp)
                 .addDef(ResVReg)
                 .addUse(GR->getSPIRVTypeID(RetType))
                 .add(Info.Callee);
 
  for (const auto &Arg : Info.OrigArgs) {
    // Currently call args should have single vregs.
    if (Arg.Regs.size() > 1)
      return false;
    MIB.addUse(Arg.Regs[0]);
  }
 
  if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
    // Process aliasing metadata.
    const CallBase *CI = Info.CB;
    if (CI && CI->hasMetadata()) {
      if (MDNode *MD = CI->getMetadata(LLVMContext::MD_alias_scope))
        GR->buildMemAliasingOpDecorate(ResVReg, MIRBuilder,
                                       SPIRV::Decoration::AliasScopeINTEL, MD);
      if (MDNode *MD = CI->getMetadata(LLVMContext::MD_noalias))
        GR->buildMemAliasingOpDecorate(ResVReg, MIRBuilder,
                                       SPIRV::Decoration::NoAliasINTEL, MD);
    }
  }
 
  return MIB.constrainAllUses(MIRBuilder.getTII(), *ST->getRegisterInfo(),
                              *ST->getRegBankInfo());
}