doxygen/AMDGPUISelDAGToDAG_8h_source.html

//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//==-----------------------------------------------------------------------===//

//

/// \file

/// Defines an instruction selector for the AMDGPU target.

//

//===----------------------------------------------------------------------===//


#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H

#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H


#include "GCNSubtarget.h"

#include "SIMachineFunctionInfo.h"

#include "SIModeRegisterDefaults.h"

#include "llvm/CodeGen/SelectionDAGISel.h"

#include "llvm/Target/TargetMachine.h"


using namespace llvm;


namespace {


static inline bool isNullConstantOrUndef(SDValue V) {

  return V.isUndef() || isNullConstant(V);

}


static inline bool getConstantValue(SDValue N, uint32_t &Out) {

  // This is only used for packed vectors, where using 0 for undef should

  // always be good.

  if (N.isUndef()) {

    Out = 0;

    return true;

  }


  if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {

    Out = C->getAPIntValue().getSExtValue();

    return true;

  }


  if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {

    Out = C->getValueAPF().bitcastToAPInt().getSExtValue();

    return true;

  }


  return false;

}


// TODO: Handle undef as zero

static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,

                                        bool Negate = false) {

  assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);

  uint32_t LHSVal, RHSVal;

  if (getConstantValue(N->getOperand(0), LHSVal) &&

      getConstantValue(N->getOperand(1), RHSVal)) {

    SDLoc SL(N);

    uint32_t K = Negate ? (-LHSVal & 0xffff) | (-RHSVal << 16)

                        : (LHSVal & 0xffff) | (RHSVal << 16);

    return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),

                              DAG.getTargetConstant(K, SL, MVT::i32));

  }


  return nullptr;

}


static inline SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {

  return packConstantV2I16(N, DAG, true);

}

} // namespace


/// AMDGPU specific code to select AMDGPU machine instructions for

/// SelectionDAG operations.

class AMDGPUDAGToDAGISel : public SelectionDAGISel {

  // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can

  // make the right decision when generating code for different targets.

  const GCNSubtarget *Subtarget;


  // Default FP mode for the current function.

  SIModeRegisterDefaults Mode;


  bool EnableLateStructurizeCFG;


  // Instructions that will be lowered with a final instruction that zeros the

  // high result bits.

  bool fp16SrcZerosHighBits(unsigned Opc) const;


public:

  static char ID;


  AMDGPUDAGToDAGISel() = delete;


  explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel);

  ~AMDGPUDAGToDAGISel() override = default;


  void getAnalysisUsage(AnalysisUsage &AU) const override;


  bool matchLoadD16FromBuildVector(SDNode *N) const;


  bool runOnMachineFunction(MachineFunction &MF) override;

  void PreprocessISelDAG() override;

  void Select(SDNode *N) override;

  StringRef getPassName() const override;

  void PostprocessISelDAG() override;


protected:

  void SelectBuildVector(SDNode *N, unsigned RegClassID);


private:

  std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;

  bool isInlineImmediate(const SDNode *N, bool Negated = false) const;

  bool isNegInlineImmediate(const SDNode *N) const {

    return isInlineImmediate(N, true);

  }


  bool isInlineImmediate16(int64_t Imm) const {

    return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm());

  }


  bool isInlineImmediate32(int64_t Imm) const {

    return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm());

  }


  bool isInlineImmediate64(int64_t Imm) const {

    return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm());

  }


  bool isInlineImmediate(const APFloat &Imm) const {

    return Subtarget->getInstrInfo()->isInlineConstant(Imm);

  }


  bool isVGPRImm(const SDNode *N) const;

  bool isUniformLoad(const SDNode *N) const;

  bool isUniformBr(const SDNode *N) const;


  // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's

  // `ShAmtBits` bits is unneeded.

  bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;


  bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,

                                  SDValue &RHS) const;


  MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;


  SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;

  SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;

  SDNode *glueCopyToM0LDSInit(SDNode *N) const;


  const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;

  virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);

  virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);

  bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;

  bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,

                        unsigned Size) const;


  bool isFlatScratchBaseLegal(SDValue Addr) const;

  bool isFlatScratchBaseLegalSV(SDValue Addr) const;

  bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;


  bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;

  bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,

                                 SDValue &Offset1) const;

  bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,

                                  SDValue &Offset1) const;

  bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,

                          SDValue &Offset1, unsigned Size) const;

  bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,

                   SDValue &SOffset, SDValue &Offset, SDValue &Offen,

                   SDValue &Idxen, SDValue &Addr64) const;

  bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,

                         SDValue &SOffset, SDValue &Offset) const;

  bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,

                               SDValue &VAddr, SDValue &SOffset,

                               SDValue &ImmOffset) const;

  bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,

                                SDValue &Soffset, SDValue &Offset) const;


  bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,

                         SDValue &Offset) const;


  bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,

                            SDValue &Offset, uint64_t FlatVariant) const;

  bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,

                        SDValue &Offset) const;

  bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,

                          SDValue &Offset) const;

  bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,

                           SDValue &Offset) const;

  bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,

                         SDValue &VOffset, SDValue &Offset) const;

  bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,

                          SDValue &Offset) const;

  bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,

                                     uint64_t ImmOffset) const;

  bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,

                           SDValue &SAddr, SDValue &Offset) const;


  bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue *SOffset,

                        SDValue *Offset, bool Imm32Only = false,

                        bool IsBuffer = false) const;

  SDValue Expand32BitAddress(SDValue Addr) const;

  bool SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase, SDValue *SOffset,

                            SDValue *Offset, bool Imm32Only = false,

                            bool IsBuffer = false) const;

  bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue *SOffset,

                  SDValue *Offset, bool Imm32Only = false) const;

  bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;

  bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;

  bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &SOffset) const;

  bool SelectSMRDSgprImm(SDValue Addr, SDValue &SBase, SDValue &SOffset,

                         SDValue &Offset) const;

  bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;

  bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;

  bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,

                               SDValue &Offset) const;

  bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;


  bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,

                          bool IsCanonicalizing = true,

                          bool AllowAbs = true) const;

  bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;

  bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,

                                       SDValue &SrcMods) const;

  bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;

  bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;

  bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,

                       SDValue &Clamp, SDValue &Omod) const;

  bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,

                        SDValue &Clamp, SDValue &Omod) const;

  bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,

                         SDValue &Clamp, SDValue &Omod) const;


  bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,

                             bool OpSel) const;

  bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;

  bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;


  bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,

                       SDValue &Omod) const;


  bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,

                       bool IsDOT = false) const;

  bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;


  bool SelectDotIUVOP3PMods(SDValue In, SDValue &Src) const;

  bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;


  bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;


  bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;

  bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,

                                 unsigned &Mods) const;

  bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,

                                SDValue &SrcMods) const;

  bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;


  SDValue getHi16Elt(SDValue In) const;


  SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;


  void SelectADD_SUB_I64(SDNode *N);

  void SelectAddcSubb(SDNode *N);

  void SelectUADDO_USUBO(SDNode *N);

  void SelectDIV_SCALE(SDNode *N);

  void SelectMAD_64_32(SDNode *N);

  void SelectMUL_LOHI(SDNode *N);

  void SelectFMA_W_CHAIN(SDNode *N);

  void SelectFMUL_W_CHAIN(SDNode *N);

  SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,

                   uint32_t Width);

  void SelectS_BFEFromShifts(SDNode *N);

  void SelectS_BFE(SDNode *N);

  bool isCBranchSCC(const SDNode *N) const;

  void SelectBRCOND(SDNode *N);

  void SelectFMAD_FMA(SDNode *N);

  void SelectFP_EXTEND(SDNode *N);

  void SelectDSAppendConsume(SDNode *N, unsigned IntrID);

  void SelectDSBvhStackIntrinsic(SDNode *N);

  void SelectDS_GWS(SDNode *N, unsigned IntrID);

  void SelectInterpP1F16(SDNode *N);

  void SelectINTRINSIC_W_CHAIN(SDNode *N);

  void SelectINTRINSIC_WO_CHAIN(SDNode *N);

  void SelectINTRINSIC_VOID(SDNode *N);

  void SelectWAVE_ADDRESS(SDNode *N);

  void SelectSTACKRESTORE(SDNode *N);


protected:

  // Include the pieces autogenerated from the target description.

#include "AMDGPUGenDAGISel.inc"

};


#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H

DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:75

Select
amdgpu AMDGPU Register Bank Select
Definition: AMDGPURegBankSelect.cpp:46

Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79

Size
uint64_t Size
Definition: ELFObjHandler.cpp:81

GCNSubtarget.h
AMD GCN specific subclass of TargetSubtarget.

assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

SIMachineFunctionInfo.h

SIModeRegisterDefaults.h

SelectionDAGISel.h

Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:74

AMDGPUDAGToDAGISel
AMDGPU specific code to select AMDGPU machine instructions for SelectionDAG operations.
Definition: AMDGPUISelDAGToDAG.h:76

AMDGPUDAGToDAGISel::SelectBuildVector
void SelectBuildVector(SDNode *N, unsigned RegClassID)
Definition: AMDGPUISelDAGToDAG.cpp:443

AMDGPUDAGToDAGISel::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: AMDGPUISelDAGToDAG.cpp:202

AMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel
~AMDGPUDAGToDAGISel() override=default

AMDGPUDAGToDAGISel::runOnMachineFunction
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Definition: AMDGPUISelDAGToDAG.cpp:126

AMDGPUDAGToDAGISel::ID
static char ID
Definition: AMDGPUISelDAGToDAG.h:91

AMDGPUDAGToDAGISel::PreprocessISelDAG
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: AMDGPUISelDAGToDAG.cpp:292

AMDGPUDAGToDAGISel::PostprocessISelDAG
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: AMDGPUISelDAGToDAG.cpp:3192

AMDGPUDAGToDAGISel::getPassName
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
Definition: AMDGPUISelDAGToDAG.cpp:782

AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel
AMDGPUDAGToDAGISel()=delete

AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector
bool matchLoadD16FromBuildVector(SDNode *N) const
Definition: AMDGPUISelDAGToDAG.cpp:212

llvm::AMDGPUSubtarget::hasInv2PiInlineImm
bool hasInv2PiInlineImm() const
Definition: AMDGPUSubtarget.h:201

llvm::APFloat
Definition: APFloat.h:780

llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47

llvm::ConstantFPSDNode
Definition: SelectionDAGNodes.h:1647

llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1602

llvm::GCNSubtarget
Definition: GCNSubtarget.h:35

llvm::GCNSubtarget::getInstrInfo
const SIInstrInfo * getInstrInfo() const override
Definition: GCNSubtarget.h:237

llvm::MachineFunction
Definition: MachineFunction.h:259

llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2934

llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1115

llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:472

llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145

llvm::SIInstrInfo::isInlineConstant
bool isInlineConstant(const APInt &Imm) const
Definition: SIInstrInfo.cpp:4076

llvm::SelectionDAGISel
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
Definition: SelectionDAGISel.h:41

llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:43

llvm::SelectionDAGISel::MF
MachineFunction * MF
Definition: SelectionDAGISel.h:47

llvm::SelectionDAGISel::OptLevel
CodeGenOptLevel OptLevel
Definition: SelectionDAGISel.h:54

llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225

llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:10503

llvm::SelectionDAG::getTargetConstant
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:674

llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50

llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78

llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:45

uint32_t

uint64_t

TargetMachine.h

llvm::AMDGPU::isInlinableLiteral16
bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi)
Definition: AMDGPUBaseInfo.cpp:2465

llvm::AMDGPU::isInlinableLiteral32
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
Definition: AMDGPUBaseInfo.cpp:2439

llvm::AMDGPU::isInlinableLiteral64
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
Definition: AMDGPUBaseInfo.cpp:2422

llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34

llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515

llvm::M68k::MemAddrModeKind::V
@ V

llvm::M68k::MemAddrModeKind::K
@ K

llvm::dwarf::Index
Index
Definition: Dwarf.h:558

llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58

llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18

llvm::Offset
@ Offset
Definition: DWP.cpp:456

llvm::isNullConstant
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
Definition: SelectionDAG.cpp:11534

llvm::CodeGenOptLevel
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54

N
#define N

llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34

llvm::SIModeRegisterDefaults
Definition: SIModeRegisterDefaults.h:18