SIISelLowering.cpp

Go to the documentation of this file.

//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Custom DAG lowering for SI
//
//===----------------------------------------------------------------------===//
 
#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
 
using namespace llvm;
 
#define DEBUG_TYPE "si-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
 
static cl::opt<bool> DisableLoopAlignment(
  "amdgpu-disable-loop-alignment",
  cl::desc("Do not align and prefetch loops"),
  cl::init(false));
 
static cl::opt<bool> UseDivergentRegisterIndexing(
  "amdgpu-use-divergent-register-indexing",
  cl::Hidden,
  cl::desc("Use indirect register addressing for divergent indexes"),
  cl::init(false));
 
static bool hasFP32Denormals(const MachineFunction &MF) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  return Info->getMode().allFP32Denormals();
}
 
static bool hasFP64FP16Denormals(const MachineFunction &MF) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  return Info->getMode().allFP64FP16Denormals();
}
 
static unsigned findFirstFreeSGPR(CCState &CCInfo) {
  unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
  for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
    if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
      return AMDGPU::SGPR0 + Reg;
    }
  }
  llvm_unreachable("Cannot allocate sgpr");
}
 
SITargetLowering::SITargetLowering(const TargetMachine &TM,
                                   const GCNSubtarget &STI)
    : AMDGPUTargetLowering(TM, STI),
      Subtarget(&STI) {
  addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
  addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
 
  addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
  addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass);
 
  addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
 
  const SIRegisterInfo *TRI = STI.getRegisterInfo();
  const TargetRegisterClass *V64RegClass = TRI->getVGPR64Class();
 
  addRegisterClass(MVT::f64, V64RegClass);
  addRegisterClass(MVT::v2f32, V64RegClass);
 
  addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
  addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96));
 
  addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass);
  addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass);
 
  addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass);
  addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128));
 
  addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
  addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160));
 
  addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass);
  addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192));
 
  addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass);
  addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192));
 
  addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass);
  addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224));
 
  addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass);
  addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256));
 
  addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass);
  addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256));
 
  addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass);
  addRegisterClass(MVT::v9f32, TRI->getVGPRClassForBitWidth(288));
 
  addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass);
  addRegisterClass(MVT::v10f32, TRI->getVGPRClassForBitWidth(320));
 
  addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass);
  addRegisterClass(MVT::v11f32, TRI->getVGPRClassForBitWidth(352));
 
  addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass);
  addRegisterClass(MVT::v12f32, TRI->getVGPRClassForBitWidth(384));
 
  addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass);
  addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512));
 
  addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass);
  addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512));
 
  addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass);
  addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024));
 
  if (Subtarget->has16BitInsts()) {
    addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass);
 
    // Unless there are also VOP3P operations, not operations are really legal.
    addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass);
  }
 
  addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
  addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024));
 
  computeRegisterProperties(Subtarget->getRegisterInfo());
 
  // The boolean content concept here is too inflexible. Compares only ever
  // really produce a 1-bit result. Any copy/extend from these will turn into a
  // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
  // it's what most targets use.
  setBooleanContents(ZeroOrOneBooleanContent);
  setBooleanVectorContents(ZeroOrOneBooleanContent);
 
  // We need to custom lower vector stores from local memory
  setOperationAction(ISD::LOAD,
                     {MVT::v2i32,  MVT::v3i32,  MVT::v4i32,  MVT::v5i32,
                      MVT::v6i32,  MVT::v7i32,  MVT::v8i32,  MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
                      MVT::i1,     MVT::v32i32},
                     Custom);
 
  setOperationAction(ISD::STORE,
                     {MVT::v2i32,  MVT::v3i32,  MVT::v4i32,  MVT::v5i32,
                      MVT::v6i32,  MVT::v7i32,  MVT::v8i32,  MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
                      MVT::i1,     MVT::v32i32},
                     Custom);
 
  setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
  setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand);
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
  setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
  setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
  setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand);
  setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
  setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand);
  setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand);
  setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand);
 
  setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
  setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
  setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand);
  setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
 
  setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
 
  setOperationAction(ISD::SELECT, MVT::i1, Promote);
  setOperationAction(ISD::SELECT, MVT::i64, Custom);
  setOperationAction(ISD::SELECT, MVT::f64, Promote);
  AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
 
  setOperationAction(ISD::SELECT_CC,
                     {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand);
 
  setOperationAction(ISD::SETCC, MVT::i1, Promote);
  setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand);
  AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
 
  setOperationAction(ISD::TRUNCATE,
                     {MVT::v2i32,  MVT::v3i32,  MVT::v4i32,  MVT::v5i32,
                      MVT::v6i32,  MVT::v7i32,  MVT::v8i32,  MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
                     Expand);
  setOperationAction(ISD::FP_ROUND,
                     {MVT::v2f32,  MVT::v3f32,  MVT::v4f32,  MVT::v5f32,
                      MVT::v6f32,  MVT::v7f32,  MVT::v8f32,  MVT::v9f32,
                      MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
                     Expand);
 
  setOperationAction(ISD::SIGN_EXTEND_INREG,
                     {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
                      MVT::v3i16, MVT::v4i16, MVT::Other},
                     Custom);
 
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::BR_CC,
                     {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand);
 
  setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Legal);
 
  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i32, Legal);
 
  setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64,
                     Expand);
 
#if 0
  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal);
#endif
 
  // We only support LOAD/STORE and vector manipulation ops for vectors
  // with > 4 elements.
  for (MVT VT :
       {MVT::v8i32,  MVT::v8f32,  MVT::v9i32,   MVT::v9f32,  MVT::v10i32,
        MVT::v10f32, MVT::v11i32, MVT::v11f32,  MVT::v12i32, MVT::v12f32,
        MVT::v16i32, MVT::v16f32, MVT::v2i64,   MVT::v2f64,  MVT::v4i16,
        MVT::v4f16,  MVT::v3i64,  MVT::v3f64,   MVT::v6i32,  MVT::v6f32,
        MVT::v4i64,  MVT::v4f64,  MVT::v8i64,   MVT::v8f64,  MVT::v8i16,
        MVT::v8f16,  MVT::v16i16, MVT::v16f16,  MVT::v16i64, MVT::v16f64,
        MVT::v32i32, MVT::v32f32}) {
    for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
      switch (Op) {
      case ISD::LOAD:
      case ISD::STORE:
      case ISD::BUILD_VECTOR:
      case ISD::BITCAST:
      case ISD::UNDEF:
      case ISD::EXTRACT_VECTOR_ELT:
      case ISD::INSERT_VECTOR_ELT:
      case ISD::EXTRACT_SUBVECTOR:
      case ISD::SCALAR_TO_VECTOR:
      case ISD::IS_FPCLASS:
        break;
      case ISD::INSERT_SUBVECTOR:
      case ISD::CONCAT_VECTORS:
        setOperationAction(Op, VT, Custom);
        break;
      default:
        setOperationAction(Op, VT, Expand);
        break;
      }
    }
  }
 
  setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand);
 
  // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
  // is expanded to avoid having two separate loops in case the index is a VGPR.
 
  // Most operations are naturally 32-bit vector operations. We only support
  // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
  for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32);
  }
 
  for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32);
  }
 
  for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32);
  }
 
  for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32);
  }
 
  for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32);
  }
 
  setOperationAction(ISD::VECTOR_SHUFFLE,
                     {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32},
                     Expand);
 
  setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16}, Custom);
 
  // Avoid stack access for these.
  // TODO: Generalize to more vector types.
  setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
                     {MVT::v2i16, MVT::v2f16, MVT::v2i8, MVT::v4i8, MVT::v8i8,
                      MVT::v4i16, MVT::v4f16},
                     Custom);
 
  // Deal with vec3 vector operations when widened to vec4.
  setOperationAction(ISD::INSERT_SUBVECTOR,
                     {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom);
 
  // Deal with vec5/6/7 vector operations when widened to vec8.
  setOperationAction(ISD::INSERT_SUBVECTOR,
                     {MVT::v5i32,  MVT::v5f32,  MVT::v6i32,  MVT::v6f32,
                      MVT::v7i32,  MVT::v7f32,  MVT::v8i32,  MVT::v8f32,
                      MVT::v9i32,  MVT::v9f32,  MVT::v10i32, MVT::v10f32,
                      MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
                     Custom);
 
  // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
  // and output demarshalling
  setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom);
 
  // We can't return success/failure, only the old value,
  // let LLVM add the comparison
  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, {MVT::i32, MVT::i64},
                     Expand);
 
  setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom);
 
  setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal);
 
  // FIXME: This should be narrowed to i32, but that only happens if i64 is
  // illegal.
  // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
  setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal);
 
  // On SI this is s_memtime and s_memrealtime on VI.
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
  setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction({ISD::FPOW, ISD::FPOWI}, MVT::f16, Promote);
    setOperationAction({ISD::FLOG, ISD::FEXP, ISD::FLOG10}, MVT::f16, Custom);
  }
 
  if (Subtarget->hasMadMacF32Insts())
    setOperationAction(ISD::FMAD, MVT::f32, Legal);
 
  if (!Subtarget->hasBFI())
    // fcopysign can be done in a single instruction with BFI.
    setOperationAction(ISD::FCOPYSIGN, {MVT::f32, MVT::f64}, Expand);
 
  if (!Subtarget->hasBCNT(32))
    setOperationAction(ISD::CTPOP, MVT::i32, Expand);
 
  if (!Subtarget->hasBCNT(64))
    setOperationAction(ISD::CTPOP, MVT::i64, Expand);
 
  if (Subtarget->hasFFBH())
    setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom);
 
  if (Subtarget->hasFFBL())
    setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);
 
  // We only really have 32-bit BFE instructions (and 16-bit on VI).
  //
  // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
  // effort to match them now. We want this to be false for i64 cases when the
  // extraction isn't restricted to the upper or lower half. Ideally we would
  // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
  // span the midpoint are probably relatively rare, so don't worry about them
  // for now.
  if (Subtarget->hasBFE())
    setHasExtractBitsInsn(true);
 
  // Clamp modifier on add/sub
  if (Subtarget->hasIntClamp())
    setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Legal);
 
  if (Subtarget->hasAddNoCarry())
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32},
                       Legal);
 
  setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, {MVT::f32, MVT::f64},
                     Custom);
 
  // These are really only legal for ieee_mode functions. We should be avoiding
  // them for functions that don't have ieee_mode enabled, so just say they are
  // legal.
  setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
                     {MVT::f32, MVT::f64}, Legal);
 
  if (Subtarget->haveRoundOpsF64())
    setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FRINT}, MVT::f64, Legal);
  else
    setOperationAction({ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FFLOOR},
                       MVT::f64, Custom);
 
  setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
 
  setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FDIV}, MVT::f32, Custom);
  setOperationAction(ISD::FDIV, MVT::f64, Custom);
 
  setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand);
  setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction({ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
                        ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i16, Legal);
 
    AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32);
 
    setOperationAction({ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
                       MVT::i16, Expand);
 
    setOperationAction({ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
                        ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
                        ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
                        ISD::CTPOP},
                       MVT::i16, Promote);
 
    setOperationAction(ISD::LOAD, MVT::i16, Custom);
 
    setTruncStoreAction(MVT::i64, MVT::i16, Expand);
 
    setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote);
    AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32);
    setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote);
    AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32);
 
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom);
 
    // F16 - Constant Actions.
    setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
 
    // F16 - Load/Store Actions.
    setOperationAction(ISD::LOAD, MVT::f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16);
    setOperationAction(ISD::STORE, MVT::f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16);
 
    // F16 - VOP1 Actions.
    setOperationAction(
        {ISD::FP_ROUND, ISD::FCOS, ISD::FSIN, ISD::FROUND, ISD::FPTRUNC_ROUND},
        MVT::f16, Custom);
 
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
 
    setOperationAction(
        {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP},
        MVT::f16, Promote);
 
    // F16 - VOP2 Actions.
    setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, MVT::f16, Expand);
 
    setOperationAction(ISD::FDIV, MVT::f16, Custom);
 
    // F16 - VOP3 Actions.
    setOperationAction(ISD::FMA, MVT::f16, Legal);
    if (STI.hasMadF16())
      setOperationAction(ISD::FMAD, MVT::f16, Legal);
 
    for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16, MVT::v8i16,
                   MVT::v8f16, MVT::v16i16, MVT::v16f16}) {
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
        switch (Op) {
        case ISD::LOAD:
        case ISD::STORE:
        case ISD::BUILD_VECTOR:
        case ISD::BITCAST:
        case ISD::UNDEF:
        case ISD::EXTRACT_VECTOR_ELT:
        case ISD::INSERT_VECTOR_ELT:
        case ISD::INSERT_SUBVECTOR:
        case ISD::EXTRACT_SUBVECTOR:
        case ISD::SCALAR_TO_VECTOR:
        case ISD::IS_FPCLASS:
          break;
        case ISD::CONCAT_VECTORS:
          setOperationAction(Op, VT, Custom);
          break;
        default:
          setOperationAction(Op, VT, Expand);
          break;
        }
      }
    }
 
    // v_perm_b32 can handle either of these.
    setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal);
    setOperationAction(ISD::BSWAP, MVT::v4i16, Custom);
 
    // XXX - Do these do anything? Vector constants turn into build_vector.
    setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
 
    setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16}, Legal);
 
    setOperationAction(ISD::STORE, MVT::v2i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::STORE, MVT::v2f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::LOAD, MVT::v2i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::AND, MVT::v2i16, Promote);
    AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::OR, MVT::v2i16, Promote);
    AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::XOR, MVT::v2i16, Promote);
    AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32);
 
    setOperationAction(ISD::LOAD, MVT::v4i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::LOAD, MVT::v4f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32);
 
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
 
    setOperationAction(ISD::LOAD, MVT::v8i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32);
    setOperationAction(ISD::LOAD, MVT::v8f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32);
 
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
 
    setOperationAction(ISD::STORE, MVT::v8i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32);
    setOperationAction(ISD::STORE, MVT::v8f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32);
 
    setOperationAction(ISD::LOAD, MVT::v16i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32);
    setOperationAction(ISD::LOAD, MVT::v16f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32);
 
    setOperationAction(ISD::STORE, MVT::v16i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32);
    setOperationAction(ISD::STORE, MVT::v16f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v2i32, Expand);
    setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v4i32, Expand);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v8i32, Expand);
 
    if (!Subtarget->hasVOP3PInsts())
      setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16}, Custom);
 
    setOperationAction(ISD::FNEG, MVT::v2f16, Legal);
    // This isn't really legal, but this avoids the legalizer unrolling it (and
    // allows matching fneg (fabs x) patterns)
    setOperationAction(ISD::FABS, MVT::v2f16, Legal);
 
    setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, MVT::f16, Custom);
    setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal);
 
    setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16}, Custom);
 
    setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16}, Expand);
 
    for (MVT Vec16 : {MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16}) {
      setOperationAction(
          {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
          Vec16, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, Vec16, Expand);
    }
  }
 
  if (Subtarget->hasVOP3PInsts()) {
    setOperationAction({ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
                        ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
                        ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
                       MVT::v2i16, Legal);
 
    setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMINNUM_IEEE,
                        ISD::FMAXNUM_IEEE, ISD::FCANONICALIZE},
                       MVT::v2f16, Legal);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, {MVT::v2i16, MVT::v2f16},
                       Custom);
 
    setOperationAction(ISD::VECTOR_SHUFFLE,
                       {MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16,
                        MVT::v16f16, MVT::v16i16},
                       Custom);
 
    for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16})
      // Split vector operations.
      setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
                          ISD::MUL, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
                          ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
                          ISD::SSUBSAT},
                         VT, Custom);
 
    for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16})
      // Split vector operations.
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE},
                         VT, Custom);
 
    setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, {MVT::v2f16, MVT::v4f16},
                       Custom);
 
    setOperationAction(ISD::FEXP, MVT::v2f16, Custom);
    setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16}, Custom);
 
    if (Subtarget->hasPackedFP32Ops()) {
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
                         MVT::v2f32, Legal);
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA},
                         {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
                         Custom);
    }
  }
 
  setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v4f16, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction(ISD::SELECT, MVT::v2i16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
  } else {
    // Legalization hack.
    setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
 
    setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v2f16, Custom);
  }
 
  setOperationAction(ISD::SELECT,
                     {MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8,
                      MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16},
                     Custom);
 
  setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom);
 
  if (Subtarget->hasMad64_32())
    setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom);
 
  setOperationAction(ISD::INTRINSIC_WO_CHAIN,
                     {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
                      MVT::v2i16, MVT::v2f16},
                     Custom);
 
  setOperationAction(ISD::INTRINSIC_W_CHAIN,
                     {MVT::v2f16, MVT::v2i16, MVT::v3f16, MVT::v3i16,
                      MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::Other, MVT::f16,
                      MVT::i16, MVT::i8},
                     Custom);
 
  setOperationAction(ISD::INTRINSIC_VOID,
                     {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v3i16,
                      MVT::v3f16, MVT::v4f16, MVT::v4i16, MVT::f16, MVT::i16,
                      MVT::i8},
                     Custom);
 
  setTargetDAGCombine({ISD::ADD,
                       ISD::UADDO_CARRY,
                       ISD::SUB,
                       ISD::USUBO_CARRY,
                       ISD::FADD,
                       ISD::FSUB,
                       ISD::FMINNUM,
                       ISD::FMAXNUM,
                       ISD::FMINNUM_IEEE,
                       ISD::FMAXNUM_IEEE,
                       ISD::FMA,
                       ISD::SMIN,
                       ISD::SMAX,
                       ISD::UMIN,
                       ISD::UMAX,
                       ISD::SETCC,
                       ISD::AND,
                       ISD::OR,
                       ISD::XOR,
                       ISD::SINT_TO_FP,
                       ISD::UINT_TO_FP,
                       ISD::FCANONICALIZE,
                       ISD::SCALAR_TO_VECTOR,
                       ISD::ZERO_EXTEND,
                       ISD::SIGN_EXTEND_INREG,
                       ISD::EXTRACT_VECTOR_ELT,
                       ISD::INSERT_VECTOR_ELT,
                       ISD::FCOPYSIGN});
 
  if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
    setTargetDAGCombine(ISD::FP_ROUND);
 
  // All memory operations. Some folding on the pointer operand is done to help
  // matching the constant offsets in the addressing modes.
  setTargetDAGCombine({ISD::LOAD,
                       ISD::STORE,
                       ISD::ATOMIC_LOAD,
                       ISD::ATOMIC_STORE,
                       ISD::ATOMIC_CMP_SWAP,
                       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
                       ISD::ATOMIC_SWAP,
                       ISD::ATOMIC_LOAD_ADD,
                       ISD::ATOMIC_LOAD_SUB,
                       ISD::ATOMIC_LOAD_AND,
                       ISD::ATOMIC_LOAD_OR,
                       ISD::ATOMIC_LOAD_XOR,
                       ISD::ATOMIC_LOAD_NAND,
                       ISD::ATOMIC_LOAD_MIN,
                       ISD::ATOMIC_LOAD_MAX,
                       ISD::ATOMIC_LOAD_UMIN,
                       ISD::ATOMIC_LOAD_UMAX,
                       ISD::ATOMIC_LOAD_FADD,
                       ISD::ATOMIC_LOAD_UINC_WRAP,
                       ISD::ATOMIC_LOAD_UDEC_WRAP,
                       ISD::INTRINSIC_VOID,
                       ISD::INTRINSIC_W_CHAIN});
 
  // FIXME: In other contexts we pretend this is a per-function property.
  setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
 
  setSchedulingPreference(Sched::RegPressure);
}
 
const GCNSubtarget *SITargetLowering::getSubtarget() const {
  return Subtarget;
}
 
//===----------------------------------------------------------------------===//
// TargetLowering queries
//===----------------------------------------------------------------------===//
 
// v_mad_mix* support a conversion from f16 to f32.
//
// There is only one special case when denormals are enabled we don't currently,
// where this is OK to use.
bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
                                       EVT DestVT, EVT SrcVT) const {
  return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
          (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
    DestVT.getScalarType() == MVT::f32 &&
    SrcVT.getScalarType() == MVT::f16 &&
    // TODO: This probably only requires no input flushing?
    !hasFP32Denormals(DAG.getMachineFunction());
}
 
bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
                                       LLT DestTy, LLT SrcTy) const {
  return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
          (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
         DestTy.getScalarSizeInBits() == 32 &&
         SrcTy.getScalarSizeInBits() == 16 &&
         // TODO: This probably only requires no input flushing?
         !hasFP32Denormals(*MI.getMF());
}
 
bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
  // SI has some legal vector types, but no legal vector operations. Say no
  // shuffles are legal in order to prefer scalarizing some vector operations.
  return false;
}
 
MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                    CallingConv::ID CC,
                                                    EVT VT) const {
  if (CC == CallingConv::AMDGPU_KERNEL)
    return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
 
  if (VT.isVector()) {
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
    if (Size == 16) {
      if (Subtarget->has16BitInsts()) {
        if (VT.isInteger())
          return MVT::v2i16;
        return (ScalarVT == MVT::bf16 ? MVT::i32 : MVT::v2f16);
      }
      return VT.isInteger() ? MVT::i32 : MVT::f32;
    }
 
    if (Size < 16)
      return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
    return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
  }
 
  if (VT.getSizeInBits() > 32)
    return MVT::i32;
 
  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
 
unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                         CallingConv::ID CC,
                                                         EVT VT) const {
  if (CC == CallingConv::AMDGPU_KERNEL)
    return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
 
  if (VT.isVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
 
    // FIXME: Should probably promote 8-bit vectors to i16.
    if (Size == 16 && Subtarget->has16BitInsts())
      return (NumElts + 1) / 2;
 
    if (Size <= 32)
      return NumElts;
 
    if (Size > 32)
      return NumElts * ((Size + 31) / 32);
  } else if (VT.getSizeInBits() > 32)
    return (VT.getSizeInBits() + 31) / 32;
 
  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
 
unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
  LLVMContext &Context, CallingConv::ID CC,
  EVT VT, EVT &IntermediateVT,
  unsigned &NumIntermediates, MVT &RegisterVT) const {
  if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
    // FIXME: We should fix the ABI to be the same on targets without 16-bit
    // support, but unless we can properly handle 3-vectors, it will be still be
    // inconsistent.
    if (Size == 16 && Subtarget->has16BitInsts()) {
      if (ScalarVT == MVT::bf16) {
        RegisterVT = MVT::i32;
        IntermediateVT = MVT::v2bf16;
      } else {
        RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
        IntermediateVT = RegisterVT;
      }
      NumIntermediates = (NumElts + 1) / 2;
      return NumIntermediates;
    }
 
    if (Size == 32) {
      RegisterVT = ScalarVT.getSimpleVT();
      IntermediateVT = RegisterVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size < 16 && Subtarget->has16BitInsts()) {
      // FIXME: Should probably form v2i16 pieces
      RegisterVT = MVT::i16;
      IntermediateVT = ScalarVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
 
    if (Size != 16 && Size <= 32) {
      RegisterVT = MVT::i32;
      IntermediateVT = ScalarVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size > 32) {
      RegisterVT = MVT::i32;
      IntermediateVT = RegisterVT;
      NumIntermediates = NumElts * ((Size + 31) / 32);
      return NumIntermediates;
    }
  }
 
  return TargetLowering::getVectorTypeBreakdownForCallingConv(
    Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
}
 
static EVT memVTFromLoadIntrData(Type *Ty, unsigned MaxNumLanes) {
  assert(MaxNumLanes != 0);
 
  if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
    unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
    return EVT::getVectorVT(Ty->getContext(),
                            EVT::getEVT(VT->getElementType()),
                            NumElts);
  }
 
  return EVT::getEVT(Ty);
}
 
// Peek through TFE struct returns to only use the data size.
static EVT memVTFromLoadIntrReturn(Type *Ty, unsigned MaxNumLanes) {
  auto *ST = dyn_cast<StructType>(Ty);
  if (!ST)
    return memVTFromLoadIntrData(Ty, MaxNumLanes);
 
  // TFE intrinsics return an aggregate type.
  assert(ST->getNumContainedTypes() == 2 &&
         ST->getContainedType(1)->isIntegerTy(32));
  return memVTFromLoadIntrData(ST->getContainedType(0), MaxNumLanes);
}
 
/// Map address space 7 to MVT::v5i32 because that's its in-memory
/// representation. This return value is vector-typed because there is no
/// MVT::i160 and it is not clear if one can be added. While this could
/// cause issues during codegen, these address space 7 pointers will be
/// rewritten away by then. Therefore, we can return MVT::v5i32 in order
/// to allow pre-codegen passes that query TargetTransformInfo, often for cost
/// modeling, to work.
MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
  if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
    return MVT::v5i32;
  return AMDGPUTargetLowering::getPointerTy(DL, AS);
}
/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
/// v8i32 when padding is added.
MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
  if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
    return MVT::v8i32;
  return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
}
 
bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                          const CallInst &CI,
                                          MachineFunction &MF,
                                          unsigned IntrID) const {
  Info.flags = MachineMemOperand::MONone;
  if (CI.hasMetadata(LLVMContext::MD_invariant_load))
    Info.flags |= MachineMemOperand::MOInvariant;
 
  if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
          AMDGPU::lookupRsrcIntrinsic(IntrID)) {
    AttributeList Attr = Intrinsic::getAttributes(CI.getContext(),
                                                  (Intrinsic::ID)IntrID);
    MemoryEffects ME = Attr.getMemoryEffects();
    if (ME.doesNotAccessMemory())
      return false;
 
    // TODO: Should images get their own address space?
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
 
    if (RsrcIntr->IsImage)
      Info.align.reset();
 
    Info.flags |= MachineMemOperand::MODereferenceable;
    if (ME.onlyReadsMemory()) {
      unsigned MaxNumLanes = 4;
 
      if (RsrcIntr->IsImage) {
        const AMDGPU::ImageDimIntrinsicInfo *Intr
          = AMDGPU::getImageDimIntrinsicInfo(IntrID);
        const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
          AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
 
        if (!BaseOpcode->Gather4) {
          // If this isn't a gather, we may have excess loaded elements in the
          // IR type. Check the dmask for the real number of elements loaded.
          unsigned DMask
            = cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
          MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
        }
      }
 
      Info.memVT = memVTFromLoadIntrReturn(CI.getType(), MaxNumLanes);
 
      // FIXME: What does alignment mean for an image?
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.flags |= MachineMemOperand::MOLoad;
    } else if (ME.onlyWritesMemory()) {
      Info.opc = ISD::INTRINSIC_VOID;
 
      Type *DataTy = CI.getArgOperand(0)->getType();
      if (RsrcIntr->IsImage) {
        unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
        unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
        Info.memVT = memVTFromLoadIntrData(DataTy, DMaskLanes);
      } else
        Info.memVT = EVT::getEVT(DataTy);
 
      Info.flags |= MachineMemOperand::MOStore;
    } else {
      // Atomic
      Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID :
                                            ISD::INTRINSIC_W_CHAIN;
      Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
      Info.flags |= MachineMemOperand::MOLoad |
                    MachineMemOperand::MOStore |
                    MachineMemOperand::MODereferenceable;
 
      // XXX - Should this be volatile without known ordering?
      Info.flags |= MachineMemOperand::MOVolatile;
 
      switch (IntrID) {
      default:
        break;
      case Intrinsic::amdgcn_raw_buffer_load_lds:
      case Intrinsic::amdgcn_struct_buffer_load_lds: {
        unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
        Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
        return true;
      }
      }
    }
    return true;
  }
 
  switch (IntrID) {
  case Intrinsic::amdgcn_atomic_inc:
  case Intrinsic::amdgcn_atomic_dec:
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap:
  case Intrinsic::amdgcn_ds_fadd:
  case Intrinsic::amdgcn_ds_fmin:
  case Intrinsic::amdgcn_ds_fmax: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
    if (!Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    return true;
  }
  case Intrinsic::amdgcn_buffer_atomic_fadd: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4));
    if (!Vol || !Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    return true;
  }
  case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
  case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
    Info.ptrVal = nullptr;
    Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    return true;
  }
  case Intrinsic::amdgcn_ds_append:
  case Intrinsic::amdgcn_ds_consume: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
    if (!Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    return true;
  }
  case Intrinsic::amdgcn_global_atomic_csub: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad |
                  MachineMemOperand::MOStore |
                  MachineMemOperand::MOVolatile;
    return true;
  }
  case Intrinsic::amdgcn_image_bvh_intersect_ray: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT?
 
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad |
                  MachineMemOperand::MODereferenceable;
    return true;
  }
  case Intrinsic::amdgcn_global_atomic_fadd:
  case Intrinsic::amdgcn_global_atomic_fmin:
  case Intrinsic::amdgcn_global_atomic_fmax:
  case Intrinsic::amdgcn_flat_atomic_fadd:
  case Intrinsic::amdgcn_flat_atomic_fmin:
  case Intrinsic::amdgcn_flat_atomic_fmax:
  case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
  case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad |
                  MachineMemOperand::MOStore |
                  MachineMemOperand::MODereferenceable |
                  MachineMemOperand::MOVolatile;
    return true;
  }
  case Intrinsic::amdgcn_ds_gws_init:
  case Intrinsic::amdgcn_ds_gws_barrier:
  case Intrinsic::amdgcn_ds_gws_sema_v:
  case Intrinsic::amdgcn_ds_gws_sema_br:
  case Intrinsic::amdgcn_ds_gws_sema_p:
  case Intrinsic::amdgcn_ds_gws_sema_release_all: {
    Info.opc = ISD::INTRINSIC_VOID;
 
    const GCNTargetMachine &TM =
        static_cast<const GCNTargetMachine &>(getTargetMachine());
 
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    Info.ptrVal = MFI->getGWSPSV(TM);
 
    // This is an abstract access, but we need to specify a type and size.
    Info.memVT = MVT::i32;
    Info.size = 4;
    Info.align = Align(4);
 
    if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
      Info.flags |= MachineMemOperand::MOLoad;
    else
      Info.flags |= MachineMemOperand::MOStore;
    return true;
  }
  case Intrinsic::amdgcn_global_load_lds: {
    Info.opc = ISD::INTRINSIC_VOID;
    unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
    Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                  MachineMemOperand::MOVolatile;
    return true;
  }
  case Intrinsic::amdgcn_ds_bvh_stack_rtn: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
 
    const GCNTargetMachine &TM =
        static_cast<const GCNTargetMachine &>(getTargetMachine());
 
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    Info.ptrVal = MFI->getGWSPSV(TM);
 
    // This is an abstract access, but we need to specify a type and size.
    Info.memVT = MVT::i32;
    Info.size = 4;
    Info.align = Align(4);
 
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    return true;
  }
  default:
    return false;
  }
}
 
bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
                                            SmallVectorImpl<Value*> &Ops,
                                            Type *&AccessTy) const {
  switch (II->getIntrinsicID()) {
  case Intrinsic::amdgcn_atomic_inc:
  case Intrinsic::amdgcn_atomic_dec:
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap:
  case Intrinsic::amdgcn_ds_append:
  case Intrinsic::amdgcn_ds_consume:
  case Intrinsic::amdgcn_ds_fadd:
  case Intrinsic::amdgcn_ds_fmin:
  case Intrinsic::amdgcn_ds_fmax:
  case Intrinsic::amdgcn_global_atomic_fadd:
  case Intrinsic::amdgcn_flat_atomic_fadd:
  case Intrinsic::amdgcn_flat_atomic_fmin:
  case Intrinsic::amdgcn_flat_atomic_fmax:
  case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
  case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16:
  case Intrinsic::amdgcn_global_atomic_csub: {
    Value *Ptr = II->getArgOperand(0);
    AccessTy = II->getType();
    Ops.push_back(Ptr);
    return true;
  }
  default:
    return false;
  }
}
 
bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const {
  if (!Subtarget->hasFlatInstOffsets()) {
    // Flat instructions do not have offsets, and only have the register
    // address.
    return AM.BaseOffs == 0 && AM.Scale == 0;
  }
 
  return AM.Scale == 0 &&
         (AM.BaseOffs == 0 ||
          Subtarget->getInstrInfo()->isLegalFLATOffset(
              AM.BaseOffs, AMDGPUAS::FLAT_ADDRESS, SIInstrFlags::FLAT));
}
 
bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
  if (Subtarget->hasFlatGlobalInsts())
    return AM.Scale == 0 &&
           (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
                                    AM.BaseOffs, AMDGPUAS::GLOBAL_ADDRESS,
                                    SIInstrFlags::FlatGlobal));
 
  if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
      // Assume the we will use FLAT for all global memory accesses
      // on VI.
      // FIXME: This assumption is currently wrong.  On VI we still use
      // MUBUF instructions for the r + i addressing mode.  As currently
      // implemented, the MUBUF instructions only work on buffer < 4GB.
      // It may be possible to support > 4GB buffers with MUBUF instructions,
      // by setting the stride value in the resource descriptor which would
      // increase the size limit to (stride * 4GB).  However, this is risky,
      // because it has never been validated.
    return isLegalFlatAddressingMode(AM);
  }
 
  return isLegalMUBUFAddressingMode(AM);
}
 
bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
  // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
  // additionally can do r + r + i with addr64. 32-bit has more addressing
  // mode options. Depending on the resource constant, it can also do
  // (i64 r0) + (i32 r1) * (i14 i).
  //
  // Private arrays end up using a scratch buffer most of the time, so also
  // assume those use MUBUF instructions. Scratch loads / stores are currently
  // implemented as mubuf instructions with offen bit set, so slightly
  // different than the normal addr64.
  if (!SIInstrInfo::isLegalMUBUFImmOffset(AM.BaseOffs))
    return false;
 
  // FIXME: Since we can split immediate into soffset and immediate offset,
  // would it make sense to allow any immediate?
 
  switch (AM.Scale) {
  case 0: // r + i or just i, depending on HasBaseReg.
    return true;
  case 1:
    return true; // We have r + r or r + i.
  case 2:
    if (AM.HasBaseReg) {
      // Reject 2 * r + r.
      return false;
    }
 
    // Allow 2 * r as r + r
    // Or  2 * r + i is allowed as r + r + i.
    return true;
  default: // Don't allow n * r
    return false;
  }
}
 
bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                             const AddrMode &AM, Type *Ty,
                                             unsigned AS, Instruction *I) const {
  // No global is ever allowed as a base.
  if (AM.BaseGV)
    return false;
 
  if (AS == AMDGPUAS::GLOBAL_ADDRESS)
    return isLegalGlobalAddressingMode(AM);
 
  if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
      AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
      AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE) {
    // If the offset isn't a multiple of 4, it probably isn't going to be
    // correctly aligned.
    // FIXME: Can we get the real alignment here?
    if (AM.BaseOffs % 4 != 0)
      return isLegalMUBUFAddressingMode(AM);
 
    // There are no SMRD extloads, so if we have to do a small type access we
    // will use a MUBUF load.
    // FIXME?: We also need to do this if unaligned, but we don't know the
    // alignment here.
    if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
      return isLegalGlobalAddressingMode(AM);
 
    if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
      // SMRD instructions have an 8-bit, dword offset on SI.
      if (!isUInt<8>(AM.BaseOffs / 4))
        return false;
    } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
      // On CI+, this can also be a 32-bit literal constant offset. If it fits
      // in 8-bits, it can use a smaller encoding.
      if (!isUInt<32>(AM.BaseOffs / 4))
        return false;
    } else if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
      // On VI, these use the SMEM format and the offset is 20-bit in bytes.
      if (!isUInt<20>(AM.BaseOffs))
        return false;
    } else
      llvm_unreachable("unhandled generation");
 
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
      return true;
 
    if (AM.Scale == 1 && AM.HasBaseReg)
      return true;
 
    return false;
  }
 
  if (AS == AMDGPUAS::PRIVATE_ADDRESS)
    return isLegalMUBUFAddressingMode(AM);
 
  if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
    // Basic, single offset DS instructions allow a 16-bit unsigned immediate
    // field.
    // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
    // an 8-bit dword offset but we don't know the alignment here.
    if (!isUInt<16>(AM.BaseOffs))
      return false;
 
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
      return true;
 
    if (AM.Scale == 1 && AM.HasBaseReg)
      return true;
 
    return false;
  }
 
  if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
    // For an unknown address space, this usually means that this is for some
    // reason being used for pure arithmetic, and not based on some addressing
    // computation. We don't have instructions that compute pointers with any
    // addressing modes, so treat them as having no offset like flat
    // instructions.
    return isLegalFlatAddressingMode(AM);
  }
 
  // Assume a user alias of global for unknown address spaces.
  return isLegalGlobalAddressingMode(AM);
}
 
bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
                                        const MachineFunction &MF) const {
  if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
    return (MemVT.getSizeInBits() <= 4 * 32);
  } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
    unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
    return (MemVT.getSizeInBits() <= MaxPrivateBits);
  } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
    return (MemVT.getSizeInBits() <= 2 * 32);
  }
  return true;
}
 
bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
    unsigned Size, unsigned AddrSpace, Align Alignment,
    MachineMemOperand::Flags Flags, unsigned *IsFast) const {
  if (IsFast)
    *IsFast = 0;
 
  if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
      AddrSpace == AMDGPUAS::REGION_ADDRESS) {
    // Check if alignment requirements for ds_read/write instructions are
    // disabled.
    if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
      return false;
 
    Align RequiredAlignment(PowerOf2Ceil(Size/8)); // Natural alignment.
    if (Subtarget->hasLDSMisalignedBug() && Size > 32 &&
        Alignment < RequiredAlignment)
      return false;
 
    // Either, the alignment requirements are "enabled", or there is an
    // unaligned LDS access related hardware bug though alignment requirements
    // are "disabled". In either case, we need to check for proper alignment
    // requirements.
    //
    switch (Size) {
    case 64:
      // SI has a hardware bug in the LDS / GDS bounds checking: if the base
      // address is negative, then the instruction is incorrectly treated as
      // out-of-bounds even if base + offsets is in bounds. Split vectorized
      // loads here to avoid emitting ds_read2_b32. We may re-combine the
      // load later in the SILoadStoreOptimizer.
      if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
        return false;
 
      // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
      // can do a 4 byte aligned, 8 byte access in a single operation using
      // ds_read2/write2_b32 with adjacent offsets.
      RequiredAlignment = Align(4);
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
        // ds_write2_b32 depending on the alignment. In either case with either
        // alignment there is no faster way of doing this.
 
        // The numbers returned here and below are not additive, it is a 'speed
        // rank'. They are just meant to be compared to decide if a certain way
        // of lowering an operation is faster than another. For that purpose
        // naturally aligned operation gets it bitsize to indicate that "it
        // operates with a speed comparable to N-bit wide load". With the full
        // alignment ds128 is slower than ds96 for example. If underaligned it
        // is comparable to a speed of a single dword access, which would then
        // mean 32 < 128 and it is faster to issue a wide load regardless.
        // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
        // wider load which will not be aligned anymore the latter is slower.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 64
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    case 96:
      if (!Subtarget->hasDS96AndDS128())
        return false;
 
      // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
      // gfx8 and older.
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // Naturally aligned access is fastest. However, also report it is Fast
        // if memory is aligned less than DWORD. A narrow load or store will be
        // be equally slow as a single ds_read_b96/ds_write_b96, but there will
        // be more of them, so overall we will pay less penalty issuing a single
        // instruction.
 
        // See comment on the values above.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 96
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    case 128:
      if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
        return false;
 
      // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
      // gfx8 and older, but  we can do a 8 byte aligned, 16 byte access in a
      // single operation using ds_read2/write2_b64.
      RequiredAlignment = Align(8);
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // Naturally aligned access is fastest. However, also report it is Fast
        // if memory is aligned less than DWORD. A narrow load or store will be
        // be equally slow as a single ds_read_b128/ds_write_b128, but there
        // will be more of them, so overall we will pay less penalty issuing a
        // single instruction.
 
        // See comment on the values above.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 128
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    default:
      if (Size > 32)
        return false;
 
      break;
    }
 
    // See comment on the values above.
    // Note that we have a single-dword or sub-dword here, so if underaligned
    // it is a slowest possible access, hence returned value is 0.
    if (IsFast)
      *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
 
    return Alignment >= RequiredAlignment ||
           Subtarget->hasUnalignedDSAccessEnabled();
  }
 
  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
    bool AlignedBy4 = Alignment >= Align(4);
    if (IsFast)
      *IsFast = AlignedBy4;
 
    return AlignedBy4 ||
           Subtarget->enableFlatScratch() ||
           Subtarget->hasUnalignedScratchAccess();
  }
 
  // FIXME: We have to be conservative here and assume that flat operations
  // will access scratch.  If we had access to the IR function, then we
  // could determine if any private memory was used in the function.
  if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
      !Subtarget->hasUnalignedScratchAccess()) {
    bool AlignedBy4 = Alignment >= Align(4);
    if (IsFast)
      *IsFast = AlignedBy4;
 
    return AlignedBy4;
  }
 
  // So long as they are correct, wide global memory operations perform better
  // than multiple smaller memory ops -- even when misaligned
  if (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace)) {
    if (IsFast)
      *IsFast = Size;
 
    return Alignment >= Align(4) ||
           Subtarget->hasUnalignedBufferAccessEnabled();
  }
 
  // Smaller than dword value must be aligned.
  if (Size < 32)
    return false;
 
  // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
  // byte-address are ignored, thus forcing Dword alignment.
  // This applies to private, global, and constant memory.
  if (IsFast)
    *IsFast = 1;
 
  return Size >= 32 && Alignment >= Align(4);
}
 
bool SITargetLowering::allowsMisalignedMemoryAccesses(
    EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    unsigned *IsFast) const {
  return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace,
                                            Alignment, Flags, IsFast);
}
 
EVT SITargetLowering::getOptimalMemOpType(
    const MemOp &Op, const AttributeList &FuncAttributes) const {
  // FIXME: Should account for address space here.
 
  // The default fallback uses the private pointer size as a guess for a type to
  // use. Make sure we switch these to 64-bit accesses.
 
  if (Op.size() >= 16 &&
      Op.isDstAligned(Align(4))) // XXX: Should only do for global
    return MVT::v4i32;
 
  if (Op.size() >= 8 && Op.isDstAligned(Align(4)))
    return MVT::v2i32;
 
  // Use the default.
  return MVT::Other;
}
 
bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
  const MemSDNode *MemNode = cast<MemSDNode>(N);
  return MemNode->getMemOperand()->getFlags() & MONoClobber;
}
 
bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
  return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
         AS == AMDGPUAS::PRIVATE_ADDRESS;
}
 
bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
                                           unsigned DestAS) const {
  // Flat -> private/local is a simple truncate.
  // Flat -> global is no-op
  if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
    return true;
 
  const GCNTargetMachine &TM =
      static_cast<const GCNTargetMachine &>(getTargetMachine());
  return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
}
 
bool SITargetLowering::isMemOpUniform(const SDNode *N) const {
  const MemSDNode *MemNode = cast<MemSDNode>(N);
 
  return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand());
}
 
TargetLoweringBase::LegalizeTypeAction
SITargetLowering::getPreferredVectorAction(MVT VT) const {
  if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
      VT.getScalarType().bitsLE(MVT::i16))
    return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                                         Type *Ty) const {
  // FIXME: Could be smarter if called for vector constants.
  return true;
}
 
bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
                                               unsigned Index) const {
  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
    return false;
 
  // TODO: Add more cases that are cheap.
  return Index == 0;
}
 
bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
  if (Subtarget->has16BitInsts() && VT == MVT::i16) {
    switch (Op) {
    case ISD::LOAD:
    case ISD::STORE:
 
    // These operations are done with 32-bit instructions anyway.
    case ISD::AND:
    case ISD::OR:
    case ISD::XOR:
    case ISD::SELECT:
      // TODO: Extensions?
      return true;
    default:
      return false;
    }
  }
 
  // SimplifySetCC uses this function to determine whether or not it should
  // create setcc with i1 operands.  We don't have instructions for i1 setcc.
  if (VT == MVT::i1 && Op == ISD::SETCC)
    return false;
 
  return TargetLowering::isTypeDesirableForOp(Op, VT);
}
 
SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
                                                   const SDLoc &SL,
                                                   SDValue Chain,
                                                   uint64_t Offset) const {
  const DataLayout &DL = DAG.getDataLayout();
  MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  const ArgDescriptor *InputPtrReg;
  const TargetRegisterClass *RC;
  LLT ArgTy;
  MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 
  std::tie(InputPtrReg, RC, ArgTy) =
      Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
 
  // We may not have the kernarg segment argument if we have no kernel
  // arguments.
  if (!InputPtrReg)
    return DAG.getConstant(0, SL, PtrVT);
 
  MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
  SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
    MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
 
  return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Offset));
}
 
SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
                                            const SDLoc &SL) const {
  uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(),
                                               FIRST_IMPLICIT);
  return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
}
 
SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
                                         const SDLoc &SL) const {
 
  Function &F = DAG.getMachineFunction().getFunction();
  std::optional<uint32_t> KnownSize =
      AMDGPUMachineFunction::getLDSKernelIdMetadata(F);
  if (KnownSize.has_value())
    return DAG.getConstant(*KnownSize, SL, MVT::i32);
  return SDValue();
}
 
SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
                                         const SDLoc &SL, SDValue Val,
                                         bool Signed,
                                         const ISD::InputArg *Arg) const {
  // First, if it is a widened vector, narrow it.
  if (VT.isVector() &&
      VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
    EVT NarrowedVT =
        EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(),
                         VT.getVectorNumElements());
    Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
                      DAG.getConstant(0, SL, MVT::i32));
  }
 
  // Then convert the vector elements or scalar value.
  if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) &&
      VT.bitsLT(MemVT)) {
    unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
    Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
  }
 
  if (MemVT.isFloatingPoint())
    Val = getFPExtOrFPRound(DAG, Val, SL, VT);
  else if (Signed)
    Val = DAG.getSExtOrTrunc(Val, SL, VT);
  else
    Val = DAG.getZExtOrTrunc(Val, SL, VT);
 
  return Val;
}
 
SDValue SITargetLowering::lowerKernargMemParameter(
    SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
    uint64_t Offset, Align Alignment, bool Signed,
    const ISD::InputArg *Arg) const {
  MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
 
  // Try to avoid using an extload by loading earlier than the argument address,
  // and extracting the relevant bits. The load should hopefully be merged with
  // the previous argument.
  if (MemVT.getStoreSize() < 4 && Alignment < 4) {
    // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
    int64_t AlignDownOffset = alignDown(Offset, 4);
    int64_t OffsetDiff = Offset - AlignDownOffset;
 
    EVT IntVT = MemVT.changeTypeToInteger();
 
    // TODO: If we passed in the base kernel offset we could have a better
    // alignment than 4, but we don't really need it.
    SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
    SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4),
                               MachineMemOperand::MODereferenceable |
                                   MachineMemOperand::MOInvariant);
 
    SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
    SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
 
    SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
    ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
    ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
 
 
    return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
  }
 
  SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
  SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
                             MachineMemOperand::MODereferenceable |
                                 MachineMemOperand::MOInvariant);
 
  SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
  return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
}
 
SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
                                              const SDLoc &SL, SDValue Chain,
                                              const ISD::InputArg &Arg) const {
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  if (Arg.Flags.isByVal()) {
    unsigned Size = Arg.Flags.getByValSize();
    int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
    return DAG.getFrameIndex(FrameIdx, MVT::i32);
  }
 
  unsigned ArgOffset = VA.getLocMemOffset();
  unsigned ArgSize = VA.getValVT().getStoreSize();
 
  int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
 
  // Create load nodes to retrieve arguments from the stack.
  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
  SDValue ArgValue;
 
  // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
  ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
  MVT MemVT = VA.getValVT();
 
  switch (VA.getLocInfo()) {
  default:
    break;
  case CCValAssign::BCvt:
    MemVT = VA.getLocVT();
    break;
  case CCValAssign::SExt:
    ExtType = ISD::SEXTLOAD;
    break;
  case CCValAssign::ZExt:
    ExtType = ISD::ZEXTLOAD;
    break;
  case CCValAssign::AExt:
    ExtType = ISD::EXTLOAD;
    break;
  }
 
  ArgValue = DAG.getExtLoad(
    ExtType, SL, VA.getLocVT(), Chain, FIN,
    MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
    MemVT);
  return ArgValue;
}
 
SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
  const SIMachineFunctionInfo &MFI,
  EVT VT,
  AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
  const ArgDescriptor *Reg;
  const TargetRegisterClass *RC;
  LLT Ty;
 
  std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID);
  if (!Reg) {
    if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
      // It's possible for a kernarg intrinsic call to appear in a kernel with
      // no allocated segment, in which case we do not add the user sgpr
      // argument, so just return null.
      return DAG.getConstant(0, SDLoc(), VT);
    }
 
    // It's undefined behavior if a function marked with the amdgpu-no-*
    // attributes uses the corresponding intrinsic.
    return DAG.getUNDEF(VT);
  }
 
  return loadInputValue(DAG, RC, VT, SDLoc(DAG.getEntryNode()), *Reg);
}
 
static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
                               CallingConv::ID CallConv,
                               ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
                               FunctionType *FType,
                               SIMachineFunctionInfo *Info) {
  for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
    const ISD::InputArg *Arg = &Ins[I];
 
    assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
           "vector type argument should have been split");
 
    // First check if it's a PS input addr.
    if (CallConv == CallingConv::AMDGPU_PS &&
        !Arg->Flags.isInReg() && PSInputNum <= 15) {
      bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum);
 
      // Inconveniently only the first part of the split is marked as isSplit,
      // so skip to the end. We only want to increment PSInputNum once for the
      // entire split argument.
      if (Arg->Flags.isSplit()) {
        while (!Arg->Flags.isSplitEnd()) {
          assert((!Arg->VT.isVector() ||
                  Arg->VT.getScalarSizeInBits() == 16) &&
                 "unexpected vector split in ps argument type");
          if (!SkipArg)
            Splits.push_back(*Arg);
          Arg = &Ins[++I];
        }
      }
 
      if (SkipArg) {
        // We can safely skip PS inputs.
        Skipped.set(Arg->getOrigArgIndex());
        ++PSInputNum;
        continue;
      }
 
      Info->markPSInputAllocated(PSInputNum);
      if (Arg->Used)
        Info->markPSInputEnabled(PSInputNum);
 
      ++PSInputNum;
    }
 
    Splits.push_back(*Arg);
  }
}
 
// Allocate special inputs passed in VGPRs.
void SITargetLowering::allocateSpecialEntryInputVGPRs(CCState &CCInfo,
                                                      MachineFunction &MF,
                                                      const SIRegisterInfo &TRI,
                                                      SIMachineFunctionInfo &Info) const {
  const LLT S32 = LLT::scalar(32);
  MachineRegisterInfo &MRI = MF.getRegInfo();
 
  if (Info.hasWorkItemIDX()) {
    Register Reg = AMDGPU::VGPR0;
    MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
    CCInfo.AllocateReg(Reg);
    unsigned Mask = (Subtarget->hasPackedTID() &&
                     Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
    Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
  }
 
  if (Info.hasWorkItemIDY()) {
    assert(Info.hasWorkItemIDX());
    if (Subtarget->hasPackedTID()) {
      Info.setWorkItemIDY(ArgDescriptor::createRegister(AMDGPU::VGPR0,
                                                        0x3ff << 10));
    } else {
      unsigned Reg = AMDGPU::VGPR1;
      MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
      CCInfo.AllocateReg(Reg);
      Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
    }
  }
 
  if (Info.hasWorkItemIDZ()) {
    assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
    if (Subtarget->hasPackedTID()) {
      Info.setWorkItemIDZ(ArgDescriptor::createRegister(AMDGPU::VGPR0,
                                                        0x3ff << 20));
    } else {
      unsigned Reg = AMDGPU::VGPR2;
      MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
      CCInfo.AllocateReg(Reg);
      Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
    }
  }
}
 
// Try to allocate a VGPR at the end of the argument list, or if no argument
// VGPRs are left allocating a stack slot.
// If \p Mask is is given it indicates bitfield position in the register.
// If \p Arg is given use it with new ]p Mask instead of allocating new.
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
                                         ArgDescriptor Arg = ArgDescriptor()) {
  if (Arg.isSet())
    return ArgDescriptor::createArg(Arg, Mask);
 
  ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
  if (RegIdx == ArgVGPRs.size()) {
    // Spill to stack required.
    int64_t Offset = CCInfo.AllocateStack(4, Align(4));
 
    return ArgDescriptor::createStack(Offset, Mask);
  }
 
  unsigned Reg = ArgVGPRs[RegIdx];
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
 
  MachineFunction &MF = CCInfo.getMachineFunction();
  Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
  MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32));
  return ArgDescriptor::createRegister(Reg, Mask);
}
 
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
                                             const TargetRegisterClass *RC,
                                             unsigned NumArgRegs) {
  ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
  if (RegIdx == ArgSGPRs.size())
    report_fatal_error("ran out of SGPRs for arguments");
 
  unsigned Reg = ArgSGPRs[RegIdx];
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
 
  MachineFunction &MF = CCInfo.getMachineFunction();
  MF.addLiveIn(Reg, RC);
  return ArgDescriptor::createRegister(Reg);
}
 
// If this has a fixed position, we still should allocate the register in the
// CCInfo state. Technically we could get away with this for values passed
// outside of the normal argument range.
static void allocateFixedSGPRInputImpl(CCState &CCInfo,
                                       const TargetRegisterClass *RC,
                                       MCRegister Reg) {
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
  MachineFunction &MF = CCInfo.getMachineFunction();
  MF.addLiveIn(Reg, RC);
}
 
static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
  if (Arg) {
    allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass,
                               Arg.getRegister());
  } else
    Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
}
 
static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
  if (Arg) {
    allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass,
                               Arg.getRegister());
  } else
    Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
}
 
/// Allocate implicit function VGPR arguments at the end of allocated user
/// arguments.
void SITargetLowering::allocateSpecialInputVGPRs(
  CCState &CCInfo, MachineFunction &MF,
  const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
  const unsigned Mask = 0x3ff;
  ArgDescriptor Arg;
 
  if (Info.hasWorkItemIDX()) {
    Arg = allocateVGPR32Input(CCInfo, Mask);
    Info.setWorkItemIDX(Arg);
  }
 
  if (Info.hasWorkItemIDY()) {
    Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
    Info.setWorkItemIDY(Arg);
  }
 
  if (Info.hasWorkItemIDZ())
    Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
}
 
/// Allocate implicit function VGPR arguments in fixed registers.
void SITargetLowering::allocateSpecialInputVGPRsFixed(
  CCState &CCInfo, MachineFunction &MF,
  const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
  Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31);
  if (!Reg)
    report_fatal_error("failed to allocated VGPR for implicit arguments");
 
  const unsigned Mask = 0x3ff;
  Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
  Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10));
  Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20));
}
 
void SITargetLowering::allocateSpecialInputSGPRs(
  CCState &CCInfo,
  MachineFunction &MF,
  const SIRegisterInfo &TRI,
  SIMachineFunctionInfo &Info) const {
  auto &ArgInfo = Info.getArgInfo();
 
  // TODO: Unify handling with private memory pointers.
  if (Info.hasDispatchPtr())
    allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr);
 
  const Module *M = MF.getFunction().getParent();
  if (Info.hasQueuePtr() &&
      AMDGPU::getCodeObjectVersion(*M) < AMDGPU::AMDHSA_COV5)
    allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr);
 
  // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
  // constant offset from the kernarg segment.
  if (Info.hasImplicitArgPtr())
    allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr);
 
  if (Info.hasDispatchID())
    allocateSGPR64Input(CCInfo, ArgInfo.DispatchID);
 
  // flat_scratch_init is not applicable for non-kernel functions.
 
  if (Info.hasWorkGroupIDX())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX);
 
  if (Info.hasWorkGroupIDY())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY);
 
  if (Info.hasWorkGroupIDZ())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ);
 
  if (Info.hasLDSKernelId())
    allocateSGPR32Input(CCInfo, ArgInfo.LDSKernelId);
}
 
// Allocate special inputs passed in user SGPRs.
void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
                                            MachineFunction &MF,
                                            const SIRegisterInfo &TRI,
                                            SIMachineFunctionInfo &Info) const {
  if (Info.hasImplicitBufferPtr()) {
    Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
    MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(ImplicitBufferPtrReg);
  }
 
  // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
  if (Info.hasPrivateSegmentBuffer()) {
    Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
    MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
    CCInfo.AllocateReg(PrivateSegmentBufferReg);
  }
 
  if (Info.hasDispatchPtr()) {
    Register DispatchPtrReg = Info.addDispatchPtr(TRI);
    MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(DispatchPtrReg);
  }
 
  const Module *M = MF.getFunction().getParent();
  if (Info.hasQueuePtr() &&
      AMDGPU::getCodeObjectVersion(*M) < AMDGPU::AMDHSA_COV5) {
    Register QueuePtrReg = Info.addQueuePtr(TRI);
    MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(QueuePtrReg);
  }
 
  if (Info.hasKernargSegmentPtr()) {
    MachineRegisterInfo &MRI = MF.getRegInfo();
    Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
    CCInfo.AllocateReg(InputPtrReg);
 
    Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
    MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
  }
 
  if (Info.hasDispatchID()) {
    Register DispatchIDReg = Info.addDispatchID(TRI);
    MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(DispatchIDReg);
  }
 
  if (Info.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
    Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
    MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(FlatScratchInitReg);
  }
 
  if (Info.hasLDSKernelId()) {
    Register Reg = Info.addLDSKernelId();
    MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(Reg);
  }
 
  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
  // these from the dispatch pointer.
}
 
// Allocate special input registers that are initialized per-wave.
void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo,
                                           MachineFunction &MF,
                                           SIMachineFunctionInfo &Info,
                                           CallingConv::ID CallConv,
                                           bool IsShader) const {
  bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
  if (Subtarget->hasUserSGPRInit16Bug() && !IsShader) {
    // Note: user SGPRs are handled by the front-end for graphics shaders
    // Pad up the used user SGPRs with dead inputs.
 
    // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
    // before enabling architected SGPRs for workgroup IDs.
    assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
 
    unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
    // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
    // rely on it to reach 16 since if we end up having no stack usage, it will
    // not really be added.
    unsigned NumRequiredSystemSGPRs = Info.hasWorkGroupIDX() +
                                      Info.hasWorkGroupIDY() +
                                      Info.hasWorkGroupIDZ() +
                                      Info.hasWorkGroupInfo();
    for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
      Register Reg = Info.addReservedUserSGPR();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
  }
 
  if (Info.hasWorkGroupIDX()) {
    Register Reg = Info.addWorkGroupIDX(HasArchitectedSGPRs);
    if (!HasArchitectedSGPRs)
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
 
    CCInfo.AllocateReg(Reg);
  }
 
  if (Info.hasWorkGroupIDY()) {
    Register Reg = Info.addWorkGroupIDY(HasArchitectedSGPRs);
    if (!HasArchitectedSGPRs)
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
 
    CCInfo.AllocateReg(Reg);
  }
 
  if (Info.hasWorkGroupIDZ()) {
    Register Reg = Info.addWorkGroupIDZ(HasArchitectedSGPRs);
    if (!HasArchitectedSGPRs)
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
 
    CCInfo.AllocateReg(Reg);
  }
 
  if (Info.hasWorkGroupInfo()) {
    Register Reg = Info.addWorkGroupInfo();
    MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(Reg);
  }
 
  if (Info.hasPrivateSegmentWaveByteOffset()) {
    // Scratch wave offset passed in system SGPR.
    unsigned PrivateSegmentWaveByteOffsetReg;
 
    if (IsShader) {
      PrivateSegmentWaveByteOffsetReg =
        Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
 
      // This is true if the scratch wave byte offset doesn't have a fixed
      // location.
      if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
        PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
        Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
      }
    } else
      PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
 
    MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
  }
 
  assert(!Subtarget->hasUserSGPRInit16Bug() || IsShader ||
         Info.getNumPreloadedSGPRs() >= 16);
}
 
static void reservePrivateMemoryRegs(const TargetMachine &TM,
                                     MachineFunction &MF,
                                     const SIRegisterInfo &TRI,
                                     SIMachineFunctionInfo &Info) {
  // Now that we've figured out where the scratch register inputs are, see if
  // should reserve the arguments and use them directly.
  MachineFrameInfo &MFI = MF.getFrameInfo();
  bool HasStackObjects = MFI.hasStackObjects();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
  // Record that we know we have non-spill stack objects so we don't need to
  // check all stack objects later.
  if (HasStackObjects)
    Info.setHasNonSpillStackObjects(true);
 
  // Everything live out of a block is spilled with fast regalloc, so it's
  // almost certain that spilling will be required.
  if (TM.getOptLevel() == CodeGenOpt::None)
    HasStackObjects = true;
 
  // For now assume stack access is needed in any callee functions, so we need
  // the scratch registers to pass in.
  bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
 
  if (!ST.enableFlatScratch()) {
    if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
      // If we have stack objects, we unquestionably need the private buffer
      // resource. For the Code Object V2 ABI, this will be the first 4 user
      // SGPR inputs. We can reserve those and use them directly.
 
      Register PrivateSegmentBufferReg =
          Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
      Info.setScratchRSrcReg(PrivateSegmentBufferReg);
    } else {
      unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
      // We tentatively reserve the last registers (skipping the last registers
      // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
      // we'll replace these with the ones immediately after those which were
      // really allocated. In the prologue copies will be inserted from the
      // argument to these reserved registers.
 
      // Without HSA, relocations are used for the scratch pointer and the
      // buffer resource setup is always inserted in the prologue. Scratch wave
      // offset is still in an input SGPR.
      Info.setScratchRSrcReg(ReservedBufferReg);
    }
  }
 
  MachineRegisterInfo &MRI = MF.getRegInfo();
 
  // For entry functions we have to set up the stack pointer if we use it,
  // whereas non-entry functions get this "for free". This means there is no
  // intrinsic advantage to using S32 over S34 in cases where we do not have
  // calls but do need a frame pointer (i.e. if we are requested to have one
  // because frame pointer elimination is disabled). To keep things simple we
  // only ever use S32 as the call ABI stack pointer, and so using it does not
  // imply we need a separate frame pointer.
  //
  // Try to use s32 as the SP, but move it if it would interfere with input
  // arguments. This won't work with calls though.
  //
  // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
  // registers.
  if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
    Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
  } else {
    assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
 
    if (MFI.hasCalls())
      report_fatal_error("call in graphics shader with too many input SGPRs");
 
    for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
      if (!MRI.isLiveIn(Reg)) {
        Info.setStackPtrOffsetReg(Reg);
        break;
      }
    }
 
    if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
      report_fatal_error("failed to find register for SP");
  }
 
  // hasFP should be accurate for entry functions even before the frame is
  // finalized, because it does not rely on the known stack size, only
  // properties like whether variable sized objects are present.
  if (ST.getFrameLowering()->hasFP(MF)) {
    Info.setFrameOffsetReg(AMDGPU::SGPR33);
  }
}
 
bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
  const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
  return !Info->isEntryFunction();
}
 
void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
 
}
 
void SITargetLowering::insertCopiesSplitCSR(
  MachineBasicBlock *Entry,
  const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
  if (!IStart)
    return;
 
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
  MachineBasicBlock::iterator MBBI = Entry->begin();
  for (const MCPhysReg *I = IStart; *I; ++I) {
    const TargetRegisterClass *RC = nullptr;
    if (AMDGPU::SReg_64RegClass.contains(*I))
      RC = &AMDGPU::SGPR_64RegClass;
    else if (AMDGPU::SReg_32RegClass.contains(*I))
      RC = &AMDGPU::SGPR_32RegClass;
    else
      llvm_unreachable("Unexpected register class in CSRsViaCopy!");
 
    Register NewVR = MRI->createVirtualRegister(RC);
    // Create copy from CSR to a virtual register.
    Entry->addLiveIn(*I);
    BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
      .addReg(*I);
 
    // Insert the copy-back instructions right before the terminator.
    for (auto *Exit : Exits)
      BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
              TII->get(TargetOpcode::COPY), *I)
        .addReg(NewVR);
  }
}
 
SDValue SITargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &Fn = MF.getFunction();
  FunctionType *FType = MF.getFunction().getFunctionType();
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) {
    DiagnosticInfoUnsupported NoGraphicsHSA(
        Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc());
    DAG.getContext()->diagnose(NoGraphicsHSA);
    return DAG.getEntryNode();
  }
 
  Info->allocateKnownAddressLDSGlobal(Fn);
 
  SmallVector<ISD::InputArg, 16> Splits;
  SmallVector<CCValAssign, 16> ArgLocs;
  BitVector Skipped(Ins.size());
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                 *DAG.getContext());
 
  bool IsGraphics = AMDGPU::isGraphics(CallConv);
  bool IsKernel = AMDGPU::isKernel(CallConv);
  bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
 
  if (IsGraphics) {
    assert(!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() &&
           !Info->hasWorkGroupInfo() && !Info->hasLDSKernelId() &&
           !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() &&
           !Info->hasWorkItemIDZ());
    if (!Subtarget->enableFlatScratch())
      assert(!Info->hasFlatScratchInit());
    if (CallConv != CallingConv::AMDGPU_CS || !Subtarget->hasArchitectedSGPRs())
      assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
             !Info->hasWorkGroupIDZ());
  }
 
  if (CallConv == CallingConv::AMDGPU_PS) {
    processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
 
    // At least one interpolation mode must be enabled or else the GPU will
    // hang.
    //
    // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
    // set PSInputAddr, the user wants to enable some bits after the compilation
    // based on run-time states. Since we can't know what the final PSInputEna
    // will look like, so we shouldn't do anything here and the user should take
    // responsibility for the correct programming.
    //
    // Otherwise, the following restrictions apply:
    // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
    // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
    //   enabled too.
    if ((Info->getPSInputAddr() & 0x7F) == 0 ||
        ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) {
      CCInfo.AllocateReg(AMDGPU::VGPR0);
      CCInfo.AllocateReg(AMDGPU::VGPR1);
      Info->markPSInputAllocated(0);
      Info->markPSInputEnabled(0);
    }
    if (Subtarget->isAmdPalOS()) {
      // For isAmdPalOS, the user does not enable some bits after compilation
      // based on run-time states; the register values being generated here are
      // the final ones set in hardware. Therefore we need to apply the
      // workaround to PSInputAddr and PSInputEnable together.  (The case where
      // a bit is set in PSInputAddr but not PSInputEnable is where the
      // frontend set up an input arg for a particular interpolation mode, but
      // nothing uses that input arg. Really we should have an earlier pass
      // that removes such an arg.)
      unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
      if ((PsInputBits & 0x7F) == 0 ||
          ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
        Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));
    }
  } else if (IsKernel) {
    assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
  } else {
    Splits.append(Ins.begin(), Ins.end());
  }
 
  if (IsEntryFunc) {
    allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
    allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
  } else if (!IsGraphics) {
    // For the fixed ABI, pass workitem IDs in the last argument register.
    allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
  }
 
  if (IsKernel) {
    analyzeFormalArgumentsCompute(CCInfo, Ins);
  } else {
    CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
    CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
  }
 
  SmallVector<SDValue, 16> Chains;
 
  // FIXME: This is the minimum kernel argument alignment. We should improve
  // this to the maximum alignment of the arguments.
  //
  // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
  // kern arg offset.
  const Align KernelArgBaseAlign = Align(16);
 
  for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
    const ISD::InputArg &Arg = Ins[i];
    if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) {
      InVals.push_back(DAG.getUNDEF(Arg.VT));
      continue;
    }
 
    CCValAssign &VA = ArgLocs[ArgIdx++];
    MVT VT = VA.getLocVT();
 
    if (IsEntryFunc && VA.isMemLoc()) {
      VT = Ins[i].VT;
      EVT MemVT = VA.getLocVT();
 
      const uint64_t Offset = VA.getLocMemOffset();
      Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
 
      if (Arg.Flags.isByRef()) {
        SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
 
        const GCNTargetMachine &TM =
            static_cast<const GCNTargetMachine &>(getTargetMachine());
        if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
                                    Arg.Flags.getPointerAddrSpace())) {
          Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
                                     Arg.Flags.getPointerAddrSpace());
        }
 
        InVals.push_back(Ptr);
        continue;
      }
 
      SDValue Arg = lowerKernargMemParameter(
        DAG, VT, MemVT, DL, Chain, Offset, Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
      Chains.push_back(Arg.getValue(1));
 
      auto *ParamTy =
        dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
      if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
          ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
                      ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
        // On SI local pointers are just offsets into LDS, so they are always
        // less than 16-bits.  On CI and newer they could potentially be
        // real pointers, so we can't guarantee their size.
        Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg,
                          DAG.getValueType(MVT::i16));
      }
 
      InVals.push_back(Arg);
      continue;
    } else if (!IsEntryFunc && VA.isMemLoc()) {
      SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
      InVals.push_back(Val);
      if (!Arg.Flags.isByVal())
        Chains.push_back(Val.getValue(1));
      continue;
    }
 
    assert(VA.isRegLoc() && "Parameter must be in a register!");
 
    Register Reg = VA.getLocReg();
    const TargetRegisterClass *RC = nullptr;
    if (AMDGPU::VGPR_32RegClass.contains(Reg))
      RC = &AMDGPU::VGPR_32RegClass;
    else if (AMDGPU::SGPR_32RegClass.contains(Reg))
      RC = &AMDGPU::SGPR_32RegClass;
    else
      llvm_unreachable("Unexpected register class in LowerFormalArguments!");
    EVT ValVT = VA.getValVT();
 
    Reg = MF.addLiveIn(Reg, RC);
    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
 
    if (Arg.Flags.isSRet()) {
      // The return object should be reasonably addressable.
 
      // FIXME: This helps when the return is a real sret. If it is a
      // automatically inserted sret (i.e. CanLowerReturn returns false), an
      // extra copy is inserted in SelectionDAGBuilder which obscures this.
      unsigned NumBits
        = 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
      Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
        DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
    }
 
    // If this is an 8 or 16-bit value, it is really passed promoted
    // to 32 bits. Insert an assert[sz]ext to capture this, then
    // truncate to the right size.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val);
      break;
    case CCValAssign::SExt:
      Val = DAG.getNode(ISD::AssertSext, DL, VT, Val,
                        DAG.getValueType(ValVT));
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
      break;
    case CCValAssign::ZExt:
      Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
                        DAG.getValueType(ValVT));
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
      break;
    case CCValAssign::AExt:
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    InVals.push_back(Val);
  }
 
  // Start adding system SGPRs.
  if (IsEntryFunc) {
    allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics);
  } else {
    CCInfo.AllocateReg(Info->getScratchRSrcReg());
    if (!IsGraphics)
      allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
  }
 
  auto &ArgUsageInfo =
    DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
  ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo());
 
  unsigned StackArgSize = CCInfo.getStackSize();
  Info->setBytesInStackArgArea(StackArgSize);
 
  return Chains.empty() ? Chain :
    DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
}
 
// TODO: If return values can't fit in registers, we should return as many as
// possible in registers before passing on stack.
bool SITargetLowering::CanLowerReturn(
  CallingConv::ID CallConv,
  MachineFunction &MF, bool IsVarArg,
  const SmallVectorImpl<ISD::OutputArg> &Outs,
  LLVMContext &Context) const {
  // Replacing returns with sret/stack usage doesn't make sense for shaders.
  // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
  // for shaders. Vector types should be explicitly handled by CC.
  if (AMDGPU::isEntryFunctionCC(CallConv))
    return true;
 
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
  return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg));
}
 
SDValue
SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                              bool isVarArg,
                              const SmallVectorImpl<ISD::OutputArg> &Outs,
                              const SmallVectorImpl<SDValue> &OutVals,
                              const SDLoc &DL, SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  if (AMDGPU::isKernel(CallConv)) {
    return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
                                             OutVals, DL, DAG);
  }
 
  bool IsShader = AMDGPU::isShader(CallConv);
 
  Info->setIfReturnsVoid(Outs.empty());
  bool IsWaveEnd = Info->returnsVoid() && IsShader;
 
  // CCValAssign - represent the assignment of the return value to a location.
  SmallVector<CCValAssign, 48> RVLocs;
  SmallVector<ISD::OutputArg, 48> Splits;
 
  // CCState - Info about the registers and stack slots.
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
 
  // Analyze outgoing return values.
  CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
 
  SDValue Glue;
  SmallVector<SDValue, 48> RetOps;
  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
 
  // Copy the result values into the output registers.
  for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
       ++I, ++RealRVLocIdx) {
    CCValAssign &VA = RVLocs[I];
    assert(VA.isRegLoc() && "Can only return in registers!");
    // TODO: Partially return in registers if return values don't fit.
    SDValue Arg = OutVals[RealRVLocIdx];
 
    // Copied from other backends.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue);
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
  }
 
  // FIXME: Does sret work properly?
  if (!Info->isEntryFunction()) {
    const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
    const MCPhysReg *I =
      TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
    if (I) {
      for (; *I; ++I) {
        if (AMDGPU::SReg_64RegClass.contains(*I))
          RetOps.push_back(DAG.getRegister(*I, MVT::i64));
        else if (AMDGPU::SReg_32RegClass.contains(*I))
          RetOps.push_back(DAG.getRegister(*I, MVT::i32));
        else
          llvm_unreachable("Unexpected register class in CSRsViaCopy!");
      }
    }
  }
 
  // Update chain and glue.
  RetOps[0] = Chain;
  if (Glue.getNode())
    RetOps.push_back(Glue);
 
  unsigned Opc = AMDGPUISD::ENDPGM;
  if (!IsWaveEnd)
    Opc = IsShader ? AMDGPUISD::RETURN_TO_EPILOG : AMDGPUISD::RET_GLUE;
  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
}
 
SDValue SITargetLowering::LowerCallResult(
    SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
    SDValue ThisVal) const {
  CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
 
  // Assign locations to each value returned by this call.
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeCallResult(Ins, RetCC);
 
  // Copy all of the result registers out of their specified physreg.
  for (unsigned i = 0; i != RVLocs.size(); ++i) {
    CCValAssign VA = RVLocs[i];
    SDValue Val;
 
    if (VA.isRegLoc()) {
      Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InGlue);
      Chain = Val.getValue(1);
      InGlue = Val.getValue(2);
    } else if (VA.isMemLoc()) {
      report_fatal_error("TODO: return values in memory");
    } else
      llvm_unreachable("unknown argument location type");
 
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::ZExt:
      Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::SExt:
      Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::AExt:
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    InVals.push_back(Val);
  }
 
  return Chain;
}
 
// Add code to pass special inputs required depending on used features separate
// from the explicit user arguments present in the IR.
void SITargetLowering::passSpecialInputs(
    CallLoweringInfo &CLI,
    CCState &CCInfo,
    const SIMachineFunctionInfo &Info,
    SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
    SmallVectorImpl<SDValue> &MemOpChains,
    SDValue Chain) const {
  // If we don't have a call site, this was a call inserted by
  // legalization. These can never use special inputs.
  if (!CLI.CB)
    return;
 
  SelectionDAG &DAG = CLI.DAG;
  const SDLoc &DL = CLI.DL;
  const Function &F = DAG.getMachineFunction().getFunction();
 
  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
  const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
 
  const AMDGPUFunctionArgInfo *CalleeArgInfo
    = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
  if (const Function *CalleeFunc = CLI.CB->getCalledFunction()) {
    auto &ArgUsageInfo =
      DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
    CalleeArgInfo = &ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc);
  }
 
  // TODO: Unify with private memory register handling. This is complicated by
  // the fact that at least in kernels, the input argument is not necessarily
  // in the same location as the input.
  static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
                             StringLiteral> ImplicitAttrs[] = {
    {AMDGPUFunctionArgInfo::DISPATCH_PTR, "amdgpu-no-dispatch-ptr"},
    {AMDGPUFunctionArgInfo::QUEUE_PTR, "amdgpu-no-queue-ptr" },
    {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, "amdgpu-no-implicitarg-ptr"},
    {AMDGPUFunctionArgInfo::DISPATCH_ID, "amdgpu-no-dispatch-id"},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, "amdgpu-no-workgroup-id-x"},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,"amdgpu-no-workgroup-id-y"},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,"amdgpu-no-workgroup-id-z"},
    {AMDGPUFunctionArgInfo::LDS_KERNEL_ID,"amdgpu-no-lds-kernel-id"},
  };
 
  for (auto Attr : ImplicitAttrs) {
    const ArgDescriptor *OutgoingArg;
    const TargetRegisterClass *ArgRC;
    LLT ArgTy;
 
    AMDGPUFunctionArgInfo::PreloadedValue InputID = Attr.first;
 
    // If the callee does not use the attribute value, skip copying the value.
    if (CLI.CB->hasFnAttr(Attr.second))
      continue;
 
    std::tie(OutgoingArg, ArgRC, ArgTy) =
        CalleeArgInfo->getPreloadedValue(InputID);
    if (!OutgoingArg)
      continue;
 
    const ArgDescriptor *IncomingArg;
    const TargetRegisterClass *IncomingArgRC;
    LLT Ty;
    std::tie(IncomingArg, IncomingArgRC, Ty) =
        CallerArgInfo.getPreloadedValue(InputID);
    assert(IncomingArgRC == ArgRC);
 
    // All special arguments are ints for now.
    EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
    SDValue InputReg;
 
    if (IncomingArg) {
      InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
    } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
      // The implicit arg ptr is special because it doesn't have a corresponding
      // input for kernels, and is computed from the kernarg segment pointer.
      InputReg = getImplicitArgPtr(DAG, DL);
    } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
      std::optional<uint32_t> Id =
          AMDGPUMachineFunction::getLDSKernelIdMetadata(F);
      if (Id.has_value()) {
        InputReg = DAG.getConstant(*Id, DL, ArgVT);
      } else {
        InputReg = DAG.getUNDEF(ArgVT);
      }
    } else {
      // We may have proven the input wasn't needed, although the ABI is
      // requiring it. We just need to allocate the register appropriately.
      InputReg = DAG.getUNDEF(ArgVT);
    }
 
    if (OutgoingArg->isRegister()) {
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
      if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
        report_fatal_error("failed to allocate implicit input argument");
    } else {
      unsigned SpecialArgOffset =
          CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4));
      SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
                                              SpecialArgOffset);
      MemOpChains.push_back(ArgStore);
    }
  }
 
  // Pack workitem IDs into a single register or pass it as is if already
  // packed.
  const ArgDescriptor *OutgoingArg;
  const TargetRegisterClass *ArgRC;
  LLT Ty;
 
  std::tie(OutgoingArg, ArgRC, Ty) =
      CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
  if (!OutgoingArg)
    std::tie(OutgoingArg, ArgRC, Ty) =
        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
  if (!OutgoingArg)
    std::tie(OutgoingArg, ArgRC, Ty) =
        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
  if (!OutgoingArg)
    return;
 
  const ArgDescriptor *IncomingArgX = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
  const ArgDescriptor *IncomingArgY = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
  const ArgDescriptor *IncomingArgZ = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
 
  SDValue InputReg;
  SDLoc SL;
 
  const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-x");
  const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-y");
  const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-z");
 
  // If incoming ids are not packed we need to pack them.
  if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX &&
      NeedWorkItemIDX) {
    if (Subtarget->getMaxWorkitemID(F, 0) != 0) {
      InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
    } else {
      InputReg = DAG.getConstant(0, DL, MVT::i32);
    }
  }
 
  if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY &&
      NeedWorkItemIDY && Subtarget->getMaxWorkitemID(F, 1) != 0) {
    SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
    Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
                    DAG.getShiftAmountConstant(10, MVT::i32, SL));
    InputReg = InputReg.getNode() ?
                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y;
  }
 
  if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ &&
      NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(F, 2) != 0) {
    SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
    Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
                    DAG.getShiftAmountConstant(20, MVT::i32, SL));
    InputReg = InputReg.getNode() ?
                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z;
  }
 
  if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
    if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
      // We're in a situation where the outgoing function requires the workitem
      // ID, but the calling function does not have it (e.g a graphics function
      // calling a C calling convention function). This is illegal, but we need
      // to produce something.
      InputReg = DAG.getUNDEF(MVT::i32);
    } else {
      // Workitem ids are already packed, any of present incoming arguments
      // will carry all required fields.
      ArgDescriptor IncomingArg = ArgDescriptor::createArg(
        IncomingArgX ? *IncomingArgX :
        IncomingArgY ? *IncomingArgY :
        *IncomingArgZ, ~0u);
      InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
    }
  }
 
  if (OutgoingArg->isRegister()) {
    if (InputReg)
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
 
    CCInfo.AllocateReg(OutgoingArg->getRegister());
  } else {
    unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4));
    if (InputReg) {
      SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
                                              SpecialArgOffset);
      MemOpChains.push_back(ArgStore);
    }
  }
}
 
static bool canGuaranteeTCO(CallingConv::ID CC) {
  return CC == CallingConv::Fast;
}
 
/// Return true if we might ever do TCO for calls with this calling convention.
static bool mayTailCallThisCC(CallingConv::ID CC) {
  switch (CC) {
  case CallingConv::C:
  case CallingConv::AMDGPU_Gfx:
    return true;
  default:
    return canGuaranteeTCO(CC);
  }
}
 
bool SITargetLowering::isEligibleForTailCallOptimization(
    SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals,
    const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
  if (!mayTailCallThisCC(CalleeCC))
    return false;
 
  // For a divergent call target, we need to do a waterfall loop over the
  // possible callees which precludes us from using a simple jump.
  if (Callee->isDivergent())
    return false;
 
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &CallerF = MF.getFunction();
  CallingConv::ID CallerCC = CallerF.getCallingConv();
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
 
  // Kernels aren't callable, and don't have a live in return address so it
  // doesn't make sense to do a tail call with entry functions.
  if (!CallerPreserved)
    return false;
 
  bool CCMatch = CallerCC == CalleeCC;
 
  if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
    if (canGuaranteeTCO(CalleeCC) && CCMatch)
      return true;
    return false;
  }
 
  // TODO: Can we handle var args?
  if (IsVarArg)
    return false;
 
  for (const Argument &Arg : CallerF.args()) {
    if (Arg.hasByValAttr())
      return false;
  }
 
  LLVMContext &Ctx = *DAG.getContext();
 
  // Check that the call results are passed in the same way.
  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
                                  CCAssignFnForCall(CalleeCC, IsVarArg),
                                  CCAssignFnForCall(CallerCC, IsVarArg)))
    return false;
 
  // The callee has to preserve all registers the caller needs to preserve.
  if (!CCMatch) {
    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
      return false;
  }
 
  // Nothing more to check if the callee is taking no arguments.
  if (Outs.empty())
    return true;
 
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
 
  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
 
  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
  // If the stack arguments for this call do not fit into our own save area then
  // the call cannot be made tail.
  // TODO: Is this really necessary?
  if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
    return false;
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
  return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
}
 
bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
  if (!CI->isTailCall())
    return false;
 
  const Function *ParentFn = CI->getParent()->getParent();
  if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv()))
    return false;
  return true;
}
 
// The wave scratch offset register is used as the global base pointer.
SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
                                    SmallVectorImpl<SDValue> &InVals) const {
  SelectionDAG &DAG = CLI.DAG;
  const SDLoc &DL = CLI.DL;
  SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
  SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
  SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
  SDValue Chain = CLI.Chain;
  SDValue Callee = CLI.Callee;
  bool &IsTailCall = CLI.IsTailCall;
  CallingConv::ID CallConv = CLI.CallConv;
  bool IsVarArg = CLI.IsVarArg;
  bool IsSibCall = false;
  bool IsThisReturn = false;
  MachineFunction &MF = DAG.getMachineFunction();
 
  if (Callee.isUndef() || isNullConstant(Callee)) {
    if (!CLI.IsTailCall) {
      for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
        InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
    }
 
    return Chain;
  }
 
  if (IsVarArg) {
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported call to variadic function ");
  }
 
  if (!CLI.CB)
    report_fatal_error("unsupported libcall legalization");
 
  if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported required tail call to function ");
  }
 
  if (AMDGPU::isShader(CallConv)) {
    // Note the issue is with the CC of the called function, not of the call
    // itself.
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported call to a shader function ");
  }
 
  if (AMDGPU::isShader(MF.getFunction().getCallingConv()) &&
      CallConv != CallingConv::AMDGPU_Gfx) {
    // Only allow calls with specific calling conventions.
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported calling convention for call from "
                              "graphics shader of function ");
  }
 
  if (IsTailCall) {
    IsTailCall = isEligibleForTailCallOptimization(
      Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
    if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
      report_fatal_error("failed to perform tail call elimination on a call "
                         "site marked musttail");
    }
 
    bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
 
    // A sibling call is one where we're under the usual C ABI and not planning
    // to change that but can still do a tail call:
    if (!TailCallOpt && IsTailCall)
      IsSibCall = true;
 
    if (IsTailCall)
      ++NumTailCalls;
  }
 
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
  SmallVector<SDValue, 8> MemOpChains;
 
  // Analyze operands of the call, assigning locations to each operand.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
 
  if (CallConv != CallingConv::AMDGPU_Gfx) {
    // With a fixed ABI, allocate fixed registers before user arguments.
    passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
  }
 
  CCInfo.AnalyzeCallOperands(Outs, AssignFn);
 
  // Get a count of how many bytes are to be pushed on the stack.
  unsigned NumBytes = CCInfo.getStackSize();
 
  if (IsSibCall) {
    // Since we're not changing the ABI to make this a tail call, the memory
    // operands are already available in the caller's incoming argument space.
    NumBytes = 0;
  }
 
  // FPDiff is the byte offset of the call's argument area from the callee's.
  // Stores to callee stack arguments will be placed in FixedStackSlots offset
  // by this amount for a tail call. In a sibling call it must be 0 because the
  // caller will deallocate the entire stack and the callee still expects its
  // arguments to begin at SP+0. Completely unused for non-tail calls.
  int32_t FPDiff = 0;
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  // Adjust the stack pointer for the new arguments...
  // These operations are automatically eliminated by the prolog/epilog pass
  if (!IsSibCall) {
    Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
 
    if (!Subtarget->enableFlatScratch()) {
      SmallVector<SDValue, 4> CopyFromChains;
 
      // In the HSA case, this should be an identity copy.
      SDValue ScratchRSrcReg
        = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
      RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
      CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
      Chain = DAG.getTokenFactor(DL, CopyFromChains);
    }
  }
 
  MVT PtrVT = MVT::i32;
 
  // Walk the register/memloc assignments, inserting copies/loads.
  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
    CCValAssign &VA = ArgLocs[i];
    SDValue Arg = OutVals[i];
 
    // Promote the value if needed.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::FPExt:
      Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    if (VA.isRegLoc()) {
      RegsToPass.push_back(std::pair(VA.getLocReg(), Arg));
    } else {
      assert(VA.isMemLoc());
 
      SDValue DstAddr;
      MachinePointerInfo DstInfo;
 
      unsigned LocMemOffset = VA.getLocMemOffset();
      int32_t Offset = LocMemOffset;
 
      SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
      MaybeAlign Alignment;
 
      if (IsTailCall) {
        ISD::ArgFlagsTy Flags = Outs[i].Flags;
        unsigned OpSize = Flags.isByVal() ?
          Flags.getByValSize() : VA.getValVT().getStoreSize();
 
        // FIXME: We can have better than the minimum byval required alignment.
        Alignment =
            Flags.isByVal()
                ? Flags.getNonZeroByValAlign()
                : commonAlignment(Subtarget->getStackAlignment(), Offset);
 
        Offset = Offset + FPDiff;
        int FI = MFI.CreateFixedObject(OpSize, Offset, true);
 
        DstAddr = DAG.getFrameIndex(FI, PtrVT);
        DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
 
        // Make sure any stack arguments overlapping with where we're storing
        // are loaded before this eventual operation. Otherwise they'll be
        // clobbered.
 
        // FIXME: Why is this really necessary? This seems to just result in a
        // lot of code to copy the stack and write them back to the same
        // locations, which are supposed to be immutable?
        Chain = addTokenForArgument(Chain, DAG, MFI, FI);
      } else {
        // Stores to the argument stack area are relative to the stack pointer.
        SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(),
                                        MVT::i32);
        DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff);
        DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
        Alignment =
            commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
      }
 
      if (Outs[i].Flags.isByVal()) {
        SDValue SizeNode =
            DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
        SDValue Cpy =
            DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
                          Outs[i].Flags.getNonZeroByValAlign(),
                          /*isVol = */ false, /*AlwaysInline = */ true,
                          /*isTailCall = */ false, DstInfo,
                          MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
 
        MemOpChains.push_back(Cpy);
      } else {
        SDValue Store =
            DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment);
        MemOpChains.push_back(Store);
      }
    }
  }
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
  for (auto &RegToPass : RegsToPass) {
    Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
                             RegToPass.second, InGlue);
    InGlue = Chain.getValue(1);
  }
 
 
  // We don't usually want to end the call-sequence here because we would tidy
  // the frame up *after* the call, however in the ABI-changing tail-call case
  // we've carefully laid out the parameters so that when sp is reset they'll be
  // in the correct location.
  if (IsTailCall && !IsSibCall) {
    Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InGlue, DL);
    InGlue = Chain.getValue(1);
  }
 
  std::vector<SDValue> Ops;
  Ops.push_back(Chain);
  Ops.push_back(Callee);
  // Add a redundant copy of the callee global which will not be legalized, as
  // we need direct access to the callee later.
  if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) {
    const GlobalValue *GV = GSD->getGlobal();
    Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
  } else {
    Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
  }
 
  if (IsTailCall) {
    // Each tail call may have to adjust the stack by a different amount, so
    // this information must travel along with the operation for eventual
    // consumption by emitEpilogue.
    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
  }
 
  // Add argument registers to the end of the list so that they are known live
  // into the call.
  for (auto &RegToPass : RegsToPass) {
    Ops.push_back(DAG.getRegister(RegToPass.first,
                                  RegToPass.second.getValueType()));
  }
 
  // Add a register mask operand representing the call-preserved registers.
 
  auto *TRI = static_cast<const SIRegisterInfo*>(Subtarget->getRegisterInfo());
  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
  assert(Mask && "Missing call preserved mask for calling convention");
  Ops.push_back(DAG.getRegisterMask(Mask));
 
  if (InGlue.getNode())
    Ops.push_back(InGlue);
 
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
  // If we're doing a tall call, use a TC_RETURN here rather than an
  // actual call instruction.
  if (IsTailCall) {
    MFI.setHasTailCall();
    unsigned OPC = CallConv == CallingConv::AMDGPU_Gfx ?
                   AMDGPUISD::TC_RETURN_GFX : AMDGPUISD::TC_RETURN;
    return DAG.getNode(OPC, DL, NodeTys, Ops);
  }
 
  // Returns a chain and a flag for retval copy to use.
  SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
  Chain = Call.getValue(0);
  InGlue = Call.getValue(1);
 
  uint64_t CalleePopBytes = NumBytes;
  Chain = DAG.getCALLSEQ_END(Chain, 0, CalleePopBytes, InGlue, DL);
  if (!Ins.empty())
    InGlue = Chain.getValue(1);
 
  // Handle result values, copying them out of physregs into vregs that we
  // return.
  return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
                         InVals, IsThisReturn,
                         IsThisReturn ? OutVals[0] : SDValue());
}
 
// This is identical to the default implementation in ExpandDYNAMIC_STACKALLOC,
// except for applying the wave size scale to the increment amount.
SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl(
    SDValue Op, SelectionDAG &DAG) const {
  const MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
  SDValue Tmp1 = Op;
  SDValue Tmp2 = Op.getValue(1);
  SDValue Tmp3 = Op.getOperand(2);
  SDValue Chain = Tmp1.getOperand(0);
 
  Register SPReg = Info->getStackPtrOffsetReg();
 
  // Chain the dynamic stack allocation so that it doesn't modify the stack
  // pointer when other instructions are using the stack.
  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
 
  SDValue Size  = Tmp2.getOperand(1);
  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
  Chain = SP.getValue(1);
  MaybeAlign Alignment = cast<ConstantSDNode>(Tmp3)->getMaybeAlignValue();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
  const TargetFrameLowering *TFL = ST.getFrameLowering();
  unsigned Opc =
    TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
    ISD::ADD : ISD::SUB;
 
  SDValue ScaledSize = DAG.getNode(
      ISD::SHL, dl, VT, Size,
      DAG.getConstant(ST.getWavefrontSizeLog2(), dl, MVT::i32));
 
  Align StackAlign = TFL->getStackAlign();
  Tmp1 = DAG.getNode(Opc, dl, VT, SP, ScaledSize); // Value
  if (Alignment && *Alignment > StackAlign) {
    Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
                       DAG.getConstant(-(uint64_t)Alignment->value()
                                           << ST.getWavefrontSizeLog2(),
                                       dl, VT));
  }
 
  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);    // Output chain
  Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
 
  return DAG.getMergeValues({Tmp1, Tmp2}, dl);
}
 
SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                  SelectionDAG &DAG) const {
  // We only handle constant sizes here to allow non-entry block, static sized
  // allocas. A truly dynamic value is more difficult to support because we
  // don't know if the size value is uniform or not. If the size isn't uniform,
  // we would need to do a wave reduction to get the maximum size to know how
  // much to increment the uniform stack pointer.
  SDValue Size = Op.getOperand(1);
  if (isa<ConstantSDNode>(Size))
      return lowerDYNAMIC_STACKALLOCImpl(Op, DAG); // Use "generic" expansion.
 
  return AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(Op, DAG);
}
 
Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT,
                                             const MachineFunction &MF) const {
  Register Reg = StringSwitch<Register>(RegName)
    .Case("m0", AMDGPU::M0)
    .Case("exec", AMDGPU::EXEC)
    .Case("exec_lo", AMDGPU::EXEC_LO)
    .Case("exec_hi", AMDGPU::EXEC_HI)
    .Case("flat_scratch", AMDGPU::FLAT_SCR)
    .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
    .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
    .Default(Register());
 
  if (Reg == AMDGPU::NoRegister) {
    report_fatal_error(Twine("invalid register name \""
                             + StringRef(RegName)  + "\"."));
 
  }
 
  if (!Subtarget->hasFlatScrRegister() &&
       Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) {
    report_fatal_error(Twine("invalid register \""
                             + StringRef(RegName)  + "\" for subtarget."));
  }
 
  switch (Reg) {
  case AMDGPU::M0:
  case AMDGPU::EXEC_LO:
  case AMDGPU::EXEC_HI:
  case AMDGPU::FLAT_SCR_LO:
  case AMDGPU::FLAT_SCR_HI:
    if (VT.getSizeInBits() == 32)
      return Reg;
    break;
  case AMDGPU::EXEC:
  case AMDGPU::FLAT_SCR:
    if (VT.getSizeInBits() == 64)
      return Reg;
    break;
  default:
    llvm_unreachable("missing register type checking");
  }
 
  report_fatal_error(Twine("invalid type for register \""
                           + StringRef(RegName) + "\"."));
}
 
// If kill is not the last instruction, split the block so kill is always a
// proper terminator.
MachineBasicBlock *
SITargetLowering::splitKillBlock(MachineInstr &MI,
                                 MachineBasicBlock *BB) const {
  MachineBasicBlock *SplitBB = BB->splitAt(MI, false /*UpdateLiveIns*/);
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
  MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
  return SplitBB;
}
 
// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
// \p MI will be the only instruction in the loop body block. Otherwise, it will
// be the first instruction in the remainder block.
//
/// \returns { LoopBody, Remainder }
static std::pair<MachineBasicBlock *, MachineBasicBlock *>
splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
  MachineFunction *MF = MBB.getParent();
  MachineBasicBlock::iterator I(&MI);
 
  // To insert the loop we need to split the block. Move everything after this
  // point to a new block, and insert a new empty block between the two.
  MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
  MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
  MachineFunction::iterator MBBI(MBB);
  ++MBBI;
 
  MF->insert(MBBI, LoopBB);
  MF->insert(MBBI, RemainderBB);
 
  LoopBB->addSuccessor(LoopBB);
  LoopBB->addSuccessor(RemainderBB);
 
  // Move the rest of the block into a new block.
  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
 
  if (InstInLoop) {
    auto Next = std::next(I);
 
    // Move instruction to loop body.
    LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
 
    // Move the rest of the block.
    RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
  } else {
    RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
  }
 
  MBB.addSuccessor(LoopBB);
 
  return std::pair(LoopBB, RemainderBB);
}
 
/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
  MachineBasicBlock *MBB = MI.getParent();
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
  auto I = MI.getIterator();
  auto E = std::next(I);
 
  BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
    .addImm(0);
 
  MIBundleBuilder Bundler(*MBB, I, E);
  finalizeBundle(*MBB, Bundler.begin());
}
 
MachineBasicBlock *
SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
                                         MachineBasicBlock *BB) const {
  const DebugLoc &DL = MI.getDebugLoc();
 
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
  MachineBasicBlock *LoopBB;
  MachineBasicBlock *RemainderBB;
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
 
  // Apparently kill flags are only valid if the def is in the same block?
  if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
    Src->setIsKill(false);
 
  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
 
  MachineBasicBlock::iterator I = LoopBB->end();
 
  const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg(
    AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1);
 
  // Clear TRAP_STS.MEM_VIOL
  BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
    .addImm(0)
    .addImm(EncodedReg);
 
  bundleInstWithWaitcnt(MI);
 
  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 
  // Load and check TRAP_STS.MEM_VIOL
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
    .addImm(EncodedReg);
 
  // FIXME: Do we need to use an isel pseudo that may clobber scc?
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
    .addReg(Reg, RegState::Kill)
    .addImm(0);
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
    .addMBB(LoopBB);
 
  return RemainderBB;
}
 
// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
// wavefront. If the value is uniform and just happens to be in a VGPR, this
// will only do one iteration. In the worst case, this will loop 64 times.
//
// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
static MachineBasicBlock::iterator
emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
                       MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
                       const DebugLoc &DL, const MachineOperand &Idx,
                       unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
                       unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
                       Register &SGPRIdxReg) {
 
  MachineFunction *MF = OrigBB.getParent();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  MachineBasicBlock::iterator I = LoopBB.begin();
 
  const TargetRegisterClass *BoolRC = TRI->getBoolRC();
  Register PhiExec = MRI.createVirtualRegister(BoolRC);
  Register NewExec = MRI.createVirtualRegister(BoolRC);
  Register CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
  Register CondReg = MRI.createVirtualRegister(BoolRC);
 
  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg)
    .addReg(InitReg)
    .addMBB(&OrigBB)
    .addReg(ResultReg)
    .addMBB(&LoopBB);
 
  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec)
    .addReg(InitSaveExecReg)
    .addMBB(&OrigBB)
    .addReg(NewExec)
    .addMBB(&LoopBB);
 
  // Read the next variant <- also loop target.
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
      .addReg(Idx.getReg(), getUndefRegState(Idx.isUndef()));
 
  // Compare the just read M0 value to all possible Idx values.
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
      .addReg(CurrentIdxReg)
      .addReg(Idx.getReg(), 0, Idx.getSubReg());
 
  // Update EXEC, save the original EXEC value to VCC.
  BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32
                                                : AMDGPU::S_AND_SAVEEXEC_B64),
          NewExec)
    .addReg(CondReg, RegState::Kill);
 
  MRI.setSimpleHint(NewExec, CondReg);
 
  if (UseGPRIdxMode) {
    if (Offset == 0) {
      SGPRIdxReg = CurrentIdxReg;
    } else {
      SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
          .addReg(CurrentIdxReg, RegState::Kill)
          .addImm(Offset);
    }
  } else {
    // Move index from VCC into M0
    if (Offset == 0) {
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
        .addReg(CurrentIdxReg, RegState::Kill);
    } else {
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
        .addReg(CurrentIdxReg, RegState::Kill)
        .addImm(Offset);
    }
  }
 
  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
  MachineInstr *InsertPt =
    BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_XOR_B32_term
                                                  : AMDGPU::S_XOR_B64_term), Exec)
      .addReg(Exec)
      .addReg(NewExec);
 
  // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
  // s_cbranch_scc0?
 
  // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
    .addMBB(&LoopBB);
 
  return InsertPt->getIterator();
}
 
// This has slightly sub-optimal regalloc when the source vector is killed by
// the read. The register allocator does not understand that the kill is
// per-workitem, so is kept alive for the whole loop so we end up not re-using a
// subregister from it, using 1 more VGPR than necessary. This was saved when
// this was expanded after register allocation.
static MachineBasicBlock::iterator
loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
               unsigned InitResultReg, unsigned PhiReg, int Offset,
               bool UseGPRIdxMode, Register &SGPRIdxReg) {
  MachineFunction *MF = MBB.getParent();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  MachineRegisterInfo &MRI = MF->getRegInfo();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
  Register DstReg = MI.getOperand(0).getReg();
  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
  Register TmpExec = MRI.createVirtualRegister(BoolXExecRC);
  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
 
  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
 
  // Save the EXEC mask
  BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec)
    .addReg(Exec);
 
  MachineBasicBlock *LoopBB;
  MachineBasicBlock *RemainderBB;
  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false);
 
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
  auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
                                      InitResultReg, DstReg, PhiReg, TmpExec,
                                      Offset, UseGPRIdxMode, SGPRIdxReg);
 
  MachineBasicBlock* LandingPad = MF->CreateMachineBasicBlock();
  MachineFunction::iterator MBBI(LoopBB);
  ++MBBI;
  MF->insert(MBBI, LandingPad);
  LoopBB->removeSuccessor(RemainderBB);
  LandingPad->addSuccessor(RemainderBB);
  LoopBB->addSuccessor(LandingPad);
  MachineBasicBlock::iterator First = LandingPad->begin();
  BuildMI(*LandingPad, First, DL, TII->get(MovExecOpc), Exec)
    .addReg(SaveExec);
 
  return InsPt;
}
 
// Returns subreg index, offset
static std::pair<unsigned, int>
computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
                            const TargetRegisterClass *SuperRC,
                            unsigned VecReg,
                            int Offset) {
  int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32;
 
  // Skip out of bounds offsets, or else we would end up using an undefined
  // register.
  if (Offset >= NumElts || Offset < 0)
    return std::pair(AMDGPU::sub0, Offset);
 
  return std::pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0);
}
 
static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
                                 MachineRegisterInfo &MRI, MachineInstr &MI,
                                 int Offset) {
  MachineBasicBlock *MBB = MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
  assert(Idx->getReg() != AMDGPU::NoRegister);
 
  if (Offset == 0) {
    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0).add(*Idx);
  } else {
    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
        .add(*Idx)
        .addImm(Offset);
  }
}
 
static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
                                   MachineRegisterInfo &MRI, MachineInstr &MI,
                                   int Offset) {
  MachineBasicBlock *MBB = MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
  if (Offset == 0)
    return Idx->getReg();
 
  Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
      .add(*Idx)
      .addImm(Offset);
  return Tmp;
}
 
static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
                                          MachineBasicBlock &MBB,
                                          const GCNSubtarget &ST) {
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo &TRI = TII->getRegisterInfo();
  MachineFunction *MF = MBB.getParent();
  MachineRegisterInfo &MRI = MF->getRegInfo();
 
  Register Dst = MI.getOperand(0).getReg();
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
  Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
 
  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
  unsigned SubReg;
  std::tie(SubReg, Offset)
    = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
 
  const bool UseGPRIdxMode = ST.useVGPRIndexMode();
 
  // Check for a SGPR index.
  if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
    MachineBasicBlock::iterator I(&MI);
    const DebugLoc &DL = MI.getDebugLoc();
 
    if (UseGPRIdxMode) {
      // TODO: Look at the uses to avoid the copy. This may require rescheduling
      // to avoid interfering with other uses, so probably requires a new
      // optimization pass.
      Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
 
      const MCInstrDesc &GPRIDXDesc =
          TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
      BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
          .addReg(SrcReg)
          .addReg(Idx)
          .addImm(SubReg);
    } else {
      setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
 
      BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
        .addReg(SrcReg, 0, SubReg)
        .addReg(SrcReg, RegState::Implicit);
    }
 
    MI.eraseFromParent();
 
    return &MBB;
  }
 
  // Control flow needs to be inserted if indexing with a VGPR.
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
  Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
 
  Register SGPRIdxReg;
  auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset,
                              UseGPRIdxMode, SGPRIdxReg);
 
  MachineBasicBlock *LoopBB = InsPt->getParent();
 
  if (UseGPRIdxMode) {
    const MCInstrDesc &GPRIDXDesc =
        TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
 
    BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
        .addReg(SrcReg)
        .addReg(SGPRIdxReg)
        .addImm(SubReg);
  } else {
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
      .addReg(SrcReg, 0, SubReg)
      .addReg(SrcReg, RegState::Implicit);
  }
 
  MI.eraseFromParent();
 
  return LoopBB;
}
 
static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
                                          MachineBasicBlock &MBB,
                                          const GCNSubtarget &ST) {
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo &TRI = TII->getRegisterInfo();
  MachineFunction *MF = MBB.getParent();
  MachineRegisterInfo &MRI = MF->getRegInfo();
 
  Register Dst = MI.getOperand(0).getReg();
  const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
  const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
  // This can be an immediate, but will be folded later.
  assert(Val->getReg());
 
  unsigned SubReg;
  std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC,
                                                         SrcVec->getReg(),
                                                         Offset);
  const bool UseGPRIdxMode = ST.useVGPRIndexMode();
 
  if (Idx->getReg() == AMDGPU::NoRegister) {
    MachineBasicBlock::iterator I(&MI);
    const DebugLoc &DL = MI.getDebugLoc();
 
    assert(Offset == 0);
 
    BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
        .add(*SrcVec)
        .add(*Val)
        .addImm(SubReg);
 
    MI.eraseFromParent();
    return &MBB;
  }
 
  // Check for a SGPR index.
  if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
    MachineBasicBlock::iterator I(&MI);
    const DebugLoc &DL = MI.getDebugLoc();
 
    if (UseGPRIdxMode) {
      Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
 
      const MCInstrDesc &GPRIDXDesc =
          TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
      BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
          .addReg(SrcVec->getReg())
          .add(*Val)
          .addReg(Idx)
          .addImm(SubReg);
    } else {
      setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
 
      const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
          TRI.getRegSizeInBits(*VecRC), 32, false);
      BuildMI(MBB, I, DL, MovRelDesc, Dst)
          .addReg(SrcVec->getReg())
          .add(*Val)
          .addImm(SubReg);
    }
    MI.eraseFromParent();
    return &MBB;
  }
 
  // Control flow needs to be inserted if indexing with a VGPR.
  if (Val->isReg())
    MRI.clearKillFlags(Val->getReg());
 
  const DebugLoc &DL = MI.getDebugLoc();
 
  Register PhiReg = MRI.createVirtualRegister(VecRC);
 
  Register SGPRIdxReg;
  auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset,
                              UseGPRIdxMode, SGPRIdxReg);
  MachineBasicBlock *LoopBB = InsPt->getParent();
 
  if (UseGPRIdxMode) {
    const MCInstrDesc &GPRIDXDesc =
        TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
 
    BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
        .addReg(PhiReg)
        .add(*Val)
        .addReg(SGPRIdxReg)
        .addImm(AMDGPU::sub0);
  } else {
    const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
        TRI.getRegSizeInBits(*VecRC), 32, false);
    BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst)
        .addReg(PhiReg)
        .add(*Val)
        .addImm(AMDGPU::sub0);
  }
 
  MI.eraseFromParent();
  return LoopBB;
}
 
MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
  MachineInstr &MI, MachineBasicBlock *BB) const {
 
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
  MachineFunction *MF = BB->getParent();
  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
 
  switch (MI.getOpcode()) {
  case AMDGPU::S_UADDO_PSEUDO:
  case AMDGPU::S_USUBO_PSEUDO: {
    const DebugLoc &DL = MI.getDebugLoc();
    MachineOperand &Dest0 = MI.getOperand(0);
    MachineOperand &Dest1 = MI.getOperand(1);
    MachineOperand &Src0 = MI.getOperand(2);
    MachineOperand &Src1 = MI.getOperand(3);
 
    unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
                       ? AMDGPU::S_ADD_I32
                       : AMDGPU::S_SUB_I32;
    BuildMI(*BB, MI, DL, TII->get(Opc), Dest0.getReg()).add(Src0).add(Src1);
 
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B64), Dest1.getReg())
        .addImm(1)
        .addImm(0);
 
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::S_ADD_U64_PSEUDO:
  case AMDGPU::S_SUB_U64_PSEUDO: {
    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
    const TargetRegisterClass *BoolRC = TRI->getBoolRC();
    const DebugLoc &DL = MI.getDebugLoc();
 
    MachineOperand &Dest = MI.getOperand(0);
    MachineOperand &Src0 = MI.getOperand(1);
    MachineOperand &Src1 = MI.getOperand(2);
 
    Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
 
    MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
    MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
 
    MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
    MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
 
    bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
 
    unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
    unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
    BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0);
    BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1).add(Src0Sub1).add(Src1Sub1);
    BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
        .addReg(DestSub0)
        .addImm(AMDGPU::sub0)
        .addReg(DestSub1)
        .addImm(AMDGPU::sub1);
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::V_ADD_U64_PSEUDO:
  case AMDGPU::V_SUB_U64_PSEUDO: {
    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
    const DebugLoc &DL = MI.getDebugLoc();
 
    bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
 
    MachineOperand &Dest = MI.getOperand(0);
    MachineOperand &Src0 = MI.getOperand(1);
    MachineOperand &Src1 = MI.getOperand(2);
 
    if (IsAdd && ST.hasLshlAddB64()) {
      auto Add = BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_LSHL_ADD_U64_e64),
                         Dest.getReg())
                     .add(Src0)
                     .addImm(0)
                     .add(Src1);
      TII->legalizeOperands(*Add);
      MI.eraseFromParent();
      return BB;
    }
 
    const auto *CarryRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
 
    Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
    Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
    Register CarryReg = MRI.createVirtualRegister(CarryRC);
    Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
 
    const TargetRegisterClass *Src0RC = Src0.isReg()
                                            ? MRI.getRegClass(Src0.getReg())
                                            : &AMDGPU::VReg_64RegClass;
    const TargetRegisterClass *Src1RC = Src1.isReg()
                                            ? MRI.getRegClass(Src1.getReg())
                                            : &AMDGPU::VReg_64RegClass;
 
    const TargetRegisterClass *Src0SubRC =
        TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
    const TargetRegisterClass *Src1SubRC =
        TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
 
    MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
    MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
 
    MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
    MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
 
    unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
    MachineInstr *LoHalf = BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0)
                               .addReg(CarryReg, RegState::Define)
                               .add(SrcReg0Sub0)
                               .add(SrcReg1Sub0)
                               .addImm(0); // clamp bit
 
    unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
    MachineInstr *HiHalf =
        BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
            .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
            .add(SrcReg0Sub1)
            .add(SrcReg1Sub1)
            .addReg(CarryReg, RegState::Kill)
            .addImm(0); // clamp bit
 
    BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
        .addReg(DestSub0)
        .addImm(AMDGPU::sub0)
        .addReg(DestSub1)
        .addImm(AMDGPU::sub1);
    TII->legalizeOperands(*LoHalf);
    TII->legalizeOperands(*HiHalf);
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::S_ADD_CO_PSEUDO:
  case AMDGPU::S_SUB_CO_PSEUDO: {
    // This pseudo has a chance to be selected
    // only from uniform add/subcarry node. All the VGPR operands
    // therefore assumed to be splat vectors.
    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
    const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
    MachineBasicBlock::iterator MII = MI;
    const DebugLoc &DL = MI.getDebugLoc();
    MachineOperand &Dest = MI.getOperand(0);
    MachineOperand &CarryDest = MI.getOperand(1);
    MachineOperand &Src0 = MI.getOperand(2);
    MachineOperand &Src1 = MI.getOperand(3);
    MachineOperand &Src2 = MI.getOperand(4);
    unsigned Opc = (MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
                       ? AMDGPU::S_ADDC_U32
                       : AMDGPU::S_SUBB_U32;
    if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) {
      Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
          .addReg(Src0.getReg());
      Src0.setReg(RegOp0);
    }
    if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) {
      Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1)
          .addReg(Src1.getReg());
      Src1.setReg(RegOp1);
    }
    Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    if (TRI->isVectorRegister(MRI, Src2.getReg())) {
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2)
          .addReg(Src2.getReg());
      Src2.setReg(RegOp2);
    }
 
    const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg());
    unsigned WaveSize = TRI->getRegSizeInBits(*Src2RC);
    assert(WaveSize == 64 || WaveSize == 32);
 
    if (WaveSize == 64) {
      if (ST.hasScalarCompareEq64()) {
        BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64))
            .addReg(Src2.getReg())
            .addImm(0);
      } else {
        const TargetRegisterClass *SubRC =
            TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
        MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
            MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
        MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
            MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
        Register Src2_32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);