NVPTXISelDAGToDAG.cpp File Reference

#include "NVPTXISelDAGToDAG.h"
#include "NVPTXUtilities.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Target/TargetIntrinsicInfo.h"

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "nvptx-isel"
#define	PASS_NAME   "NVPTX DAG->DAG Pattern Instruction Selection"
#define	getOpcV2H(ty, opKind0, opKind1)    NVPTX::StoreParamV2##ty##_##opKind0##opKind1
#define	getOpcV2H1(ty, opKind0, isImm1)    (isImm1) ? getOpcV2H(ty, opKind0, i) : getOpcV2H(ty, opKind0, r)
#define	getOpcodeForVectorStParamV2(ty, isimm)    (isimm[0]) ? getOpcV2H1(ty, i, isimm[1]) : getOpcV2H1(ty, r, isimm[1])
#define	getOpcV4H(ty, opKind0, opKind1, opKind2, opKind3)    NVPTX::StoreParamV4##ty##_##opKind0##opKind1##opKind2##opKind3
#define	getOpcV4H3(ty, opKind0, opKind1, opKind2, isImm3)
#define	getOpcV4H2(ty, opKind0, opKind1, isImm2, isImm3)
#define	getOpcV4H1(ty, opKind0, isImm1, isImm2, isImm3)
#define	getOpcodeForVectorStParamV4(ty, isimm)
#define	getOpcodeForVectorStParam(n, ty, isimm)
#define	CP_ASYNC_BULK_TENSOR_OPCODE(dir, dim, mode, is_s32, suffix)
#define	CP_ASYNC_BULK_TENSOR_OPCODE_S2G_IMPL(op, dim, mode, is_ch, is_s32)
#define	GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G(dim, mode, is_reduce, is_ch, is_s32)
#define	GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S(dim, mode, is_mc, is_ch, is_s32)
#define	GET_CP_ASYNC_BULK_TENSOR_OPCODE_PREFETCH(dim, mode, is_ch)

Functions
static unsigned	getPTXCmpMode (const CondCodeSDNode &CondCode, bool FTZ)
static unsigned int	getCodeAddrSpace (MemSDNode *N)
static bool	canLowerToLDG (MemSDNode *N, const NVPTXSubtarget &Subtarget, unsigned CodeAddrSpace, MachineFunction *F)
static unsigned int	getFenceOp (NVPTX::Ordering O, NVPTX::Scope S, NVPTXSubtarget const *T)
static std::optional< unsigned >	pickOpcodeForVT (MVT::SimpleValueType VT, unsigned Opcode_i8, unsigned Opcode_i16, unsigned Opcode_i32, std::optional< unsigned > Opcode_i64, unsigned Opcode_f32, std::optional< unsigned > Opcode_f64)
static int	getLdStRegType (EVT VT)
static bool	isVectorElementTypeUpsized (EVT EltVT)
static unsigned	pickOpcodeForVectorStParam (SmallVector< SDValue, 8 > &Ops, unsigned NumElts, MVT::SimpleValueType MemTy, SelectionDAG *CurDAG, SDLoc DL)
static bool	isAddLike (const SDValue V)
static unsigned	GetCpAsyncBulkTensorS2GOpcode (size_t Dim, bool IsShared32, bool IsCacheHint, bool IsIm2Col, bool IsReduce=false)
static unsigned	GetCpAsyncBulkTensorG2SOpcode (size_t Dim, bool IsShared32, bool IsMultiCast, bool IsCacheHint, bool IsIm2Col)
static unsigned	GetCpAsyncBulkTensorPrefetchOpcode (size_t Dim, bool IsCacheHint, bool IsIm2Col)
static size_t	GetDimsFromIntrinsic (unsigned IID)

Variables
static cl::opt< bool >	EnableRsqrtOpt ("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden, cl::desc("Enable reciprocal sqrt optimization"))

Macro Definition Documentation

CP_ASYNC_BULK_TENSOR_OPCODE

#define CP_ASYNC_BULK_TENSOR_OPCODE	(		dir,
			dim,
			mode,
			is_s32,
			suffix
	)

Value:

  (is_s32                                                                      \
       ? NVPTX::CP_ASYNC_BULK_TENSOR_##dir##_##dim##_SHARED32_##mode##suffix   \
       : NVPTX::CP_ASYNC_BULK_TENSOR_##dir##_##dim##_##mode##suffix)

Definition at line 2766 of file NVPTXISelDAGToDAG.cpp.

CP_ASYNC_BULK_TENSOR_OPCODE_S2G_IMPL

#define CP_ASYNC_BULK_TENSOR_OPCODE_S2G_IMPL	(		op,
			dim,
			mode,
			is_ch,
			is_s32
	)

Value:

  (is_ch ? (CP_ASYNC_BULK_TENSOR_OPCODE(op, dim, mode, is_s32, _CH))           \
         : (CP_ASYNC_BULK_TENSOR_OPCODE(op, dim, mode, is_s32, )))

Definition at line 2771 of file NVPTXISelDAGToDAG.cpp.

DEBUG_TYPE

#define DEBUG_TYPE   "nvptx-isel"

Definition at line 30 of file NVPTXISelDAGToDAG.cpp.

GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S

#define GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S	(		dim,
			mode,
			is_mc,
			is_ch,
			is_s32
	)

Value:

  [&]() -> auto {                                                              \
    if (is_mc && is_ch)                                                        \
      return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, _MC_CH);      \
    if (is_ch)                                                                 \
      return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, _CH);         \
    if (is_mc)                                                                 \
      return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, _MC);         \
    return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, );              \
  }()

Definition at line 2782 of file NVPTXISelDAGToDAG.cpp.

GET_CP_ASYNC_BULK_TENSOR_OPCODE_PREFETCH

#define GET_CP_ASYNC_BULK_TENSOR_OPCODE_PREFETCH	(		dim,
			mode,
			is_ch
	)

Value:

  (is_ch ? NVPTX::CP_ASYNC_BULK_TENSOR_PREFETCH_##dim##_##mode##_CH            \
         : NVPTX::CP_ASYNC_BULK_TENSOR_PREFETCH_##dim##_##mode)

Definition at line 2793 of file NVPTXISelDAGToDAG.cpp.

GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G

#define GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G	(		dim,
			mode,
			is_reduce,
			is_ch,
			is_s32
	)

Value:

  (is_reduce                                                                   \
       ? (CP_ASYNC_BULK_TENSOR_OPCODE_S2G_IMPL(RED, dim, mode, is_ch, is_s32)) \
       : (CP_ASYNC_BULK_TENSOR_OPCODE_S2G_IMPL(S2G, dim, mode, is_ch,          \
                                               is_s32)))

Definition at line 2775 of file NVPTXISelDAGToDAG.cpp.

getOpcodeForVectorStParam

#define getOpcodeForVectorStParam	(		n,
			ty,
			isimm
	)

Value:

  (n == 2) ? getOpcodeForVectorStParamV2(ty, isimm)                            \
           : getOpcodeForVectorStParamV4(ty, isimm)

Definition at line 2054 of file NVPTXISelDAGToDAG.cpp.

getOpcodeForVectorStParamV2

#define getOpcodeForVectorStParamV2	(		ty,
			isimm
	)		(isimm[0]) ? getOpcV2H1(ty, i, isimm[1]) : getOpcV2H1(ty, r, isimm[1])

Definition at line 2032 of file NVPTXISelDAGToDAG.cpp.

getOpcodeForVectorStParamV4

#define getOpcodeForVectorStParamV4	(		ty,
			isimm
	)

Value:

  (isimm[0]) ? getOpcV4H1(ty, i, isimm[1], isimm[2], isimm[3])                 \
             : getOpcV4H1(ty, r, isimm[1], isimm[2], isimm[3])

Definition at line 2050 of file NVPTXISelDAGToDAG.cpp.

getOpcV2H

#define getOpcV2H	(		ty,
			opKind0,
			opKind1
	)		NVPTX::StoreParamV2##ty##_##opKind0##opKind1

Definition at line 2026 of file NVPTXISelDAGToDAG.cpp.

getOpcV2H1

#define getOpcV2H1	(		ty,
			opKind0,
			isImm1
	)		(isImm1) ? getOpcV2H(ty, opKind0, i) : getOpcV2H(ty, opKind0, r)

Definition at line 2029 of file NVPTXISelDAGToDAG.cpp.

getOpcV4H

#define getOpcV4H	(		ty,
			opKind0,
			opKind1,
			opKind2,
			opKind3
	)		NVPTX::StoreParamV4##ty##_##opKind0##opKind1##opKind2##opKind3

Definition at line 2035 of file NVPTXISelDAGToDAG.cpp.

getOpcV4H1

#define getOpcV4H1	(		ty,
			opKind0,
			isImm1,
			isImm2,
			isImm3
	)

Value:

  (isImm1) ? getOpcV4H2(ty, opKind0, i, isImm2, isImm3)                        \
           : getOpcV4H2(ty, opKind0, r, isImm2, isImm3)

Definition at line 2046 of file NVPTXISelDAGToDAG.cpp.

getOpcV4H2

#define getOpcV4H2	(		ty,
			opKind0,
			opKind1,
			isImm2,
			isImm3
	)

Value:

  (isImm2) ? getOpcV4H3(ty, opKind0, opKind1, i, isImm3)                       \
           : getOpcV4H3(ty, opKind0, opKind1, r, isImm3)

Definition at line 2042 of file NVPTXISelDAGToDAG.cpp.

getOpcV4H3

#define getOpcV4H3	(		ty,
			opKind0,
			opKind1,
			opKind2,
			isImm3
	)

Value:

  (isImm3) ? getOpcV4H(ty, opKind0, opKind1, opKind2, i)                       \
           : getOpcV4H(ty, opKind0, opKind1, opKind2, r)

Definition at line 2038 of file NVPTXISelDAGToDAG.cpp.

PASS_NAME

#define PASS_NAME   "NVPTX DAG->DAG Pattern Instruction Selection"

Definition at line 31 of file NVPTXISelDAGToDAG.cpp.

Function Documentation

canLowerToLDG()

static bool canLowerToLDG ( MemSDNode *  N, const NVPTXSubtarget &  Subtarget, unsigned  CodeAddrSpace, MachineFunction *  F  )

static

Definition at line 597 of file NVPTXISelDAGToDAG.cpp.

References A, llvm::all_of(), F, llvm::getUnderlyingObjects(), llvm::NVPTX::Global, llvm::NVPTXSubtarget::hasLDG(), if(), llvm::isKernelFunction(), and N.

getCodeAddrSpace()

static unsigned int getCodeAddrSpace ( MemSDNode *  N )

static

Definition at line 337 of file NVPTXISelDAGToDAG.cpp.

References llvm::NVPTXAS::ADDRESS_SPACE_CONST, llvm::NVPTXAS::ADDRESS_SPACE_GENERIC, llvm::NVPTXAS::ADDRESS_SPACE_GLOBAL, llvm::NVPTXAS::ADDRESS_SPACE_LOCAL, llvm::NVPTXAS::ADDRESS_SPACE_PARAM, llvm::NVPTXAS::ADDRESS_SPACE_SHARED, llvm::NVPTX::Const, llvm::NVPTX::Generic, llvm::NVPTX::Global, llvm::NVPTX::Local, N, llvm::NVPTX::Param, and llvm::NVPTX::Shared.

GetCpAsyncBulkTensorG2SOpcode()

static unsigned GetCpAsyncBulkTensorG2SOpcode ( size_t  Dim, bool  IsShared32, bool  IsMultiCast, bool  IsCacheHint, bool  IsIm2Col  )

static

Definition at line 2839 of file NVPTXISelDAGToDAG.cpp.

References D, GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S, and llvm_unreachable.

GetCpAsyncBulkTensorPrefetchOpcode()

static unsigned GetCpAsyncBulkTensorPrefetchOpcode ( size_t  Dim, bool  IsCacheHint, bool  IsIm2Col  )

static

Definition at line 2881 of file NVPTXISelDAGToDAG.cpp.

References D, GET_CP_ASYNC_BULK_TENSOR_OPCODE_PREFETCH, and llvm_unreachable.

GetCpAsyncBulkTensorS2GOpcode()

static unsigned GetCpAsyncBulkTensorS2GOpcode ( size_t  Dim, bool  IsShared32, bool  IsCacheHint, bool  IsIm2Col, bool  IsReduce = false  )

static

Definition at line 2797 of file NVPTXISelDAGToDAG.cpp.

References D, GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G, and llvm_unreachable.

GetDimsFromIntrinsic()

static size_t GetDimsFromIntrinsic ( unsigned  IID )

static

Definition at line 2914 of file NVPTXISelDAGToDAG.cpp.

References llvm_unreachable.

getFenceOp()

static unsigned int getFenceOp ( NVPTX::Ordering  O, NVPTX::Scope  S, NVPTXSubtarget const *  T  )

static

Definition at line 639 of file NVPTXISelDAGToDAG.cpp.

References llvm::NVPTX::Acquire, llvm::NVPTX::AcquireRelease, llvm::NVPTX::Block, llvm::NVPTX::Cluster, llvm::NVPTX::Device, llvm::formatv(), llvm_unreachable, llvm::NVPTX::NotAtomic, llvm::NVPTX::Relaxed, llvm::NVPTX::RelaxedMMIO, llvm::NVPTX::Release, llvm::report_fatal_error(), llvm::NVPTX::SequentiallyConsistent, llvm::NVPTX::System, llvm::NVPTX::Thread, and llvm::NVPTX::Volatile.

getLdStRegType()

static int getLdStRegType ( EVT  VT )

static

Definition at line 852 of file NVPTXISelDAGToDAG.cpp.

References llvm::NVPTX::PTXLdStInstCode::Float, llvm::EVT::getSimpleVT(), llvm::EVT::isFloatingPoint(), llvm::MVT::SimpleTy, llvm::NVPTX::PTXLdStInstCode::Unsigned, and llvm::NVPTX::PTXLdStInstCode::Untyped.

getPTXCmpMode()

static unsigned getPTXCmpMode ( const CondCodeSDNode &  CondCode, bool  FTZ  )

static

Definition at line 214 of file NVPTXISelDAGToDAG.cpp.

References CC, llvm::NVPTX::PTXCmpMode::FTZ_FLAG, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, and llvm::ISD::SETUO.

isAddLike()

static bool isAddLike ( const SDValue  V )

inlinestatic

Definition at line 2453 of file NVPTXISelDAGToDAG.cpp.

References llvm::ISD::ADD, and llvm::ISD::OR.

isVectorElementTypeUpsized()

static bool isVectorElementTypeUpsized ( EVT  EltVT )

static

Definition at line 986 of file NVPTXISelDAGToDAG.cpp.

References llvm::Isv2x16VT().

pickOpcodeForVectorStParam()

static unsigned pickOpcodeForVectorStParam ( SmallVector< SDValue, 8 > &  Ops, unsigned  NumElts, MVT::SimpleValueType  MemTy, SelectionDAG *  CurDAG, SDLoc  DL  )

static

Definition at line 2058 of file NVPTXISelDAGToDAG.cpp.

References assert(), DL, F32, F64, llvm::ConstantFPSDNode::getConstantFPValue(), llvm::ConstantSDNode::getConstantIntValue(), getOpcodeForVectorStParam, getOpcodeForVectorStParamV2, llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetConstantFP(), and llvm_unreachable.

pickOpcodeForVT()

static std::optional< unsigned > pickOpcodeForVT ( MVT::SimpleValueType  VT, unsigned  Opcode_i8, unsigned  Opcode_i16, unsigned  Opcode_i32, std::optional< unsigned >  Opcode_i64, unsigned  Opcode_f32, std::optional< unsigned >  Opcode_f64  )

static

Definition at line 821 of file NVPTXISelDAGToDAG.cpp.

Variable Documentation

EnableRsqrtOpt

cl::opt< bool > EnableRsqrtOpt("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden, cl::desc("Enable reciprocal sqrt optimization")) ( "nvptx-rsqrt-approx-opt"  , cl::init(true)  , cl::Hidden  , cl::desc("Enable reciprocal sqrt optimization")    )

static