23#include "llvm/IR/IntrinsicsAMDGPU.h"
24#include "llvm/IR/IntrinsicsR600.h"
28#include "R600GenCallingConv.inc"
146 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32},
Custom);
149 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32},
Custom);
186 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
187 for (
MVT VT : ScalarIntVTs)
206 if (std::next(
I) ==
I->getParent()->end())
208 return std::next(
I)->getOpcode() == R600::RETURN;
219 switch (
MI.getOpcode()) {
223 if (
TII->isLDSRetInstr(
MI.getOpcode())) {
224 int DstIdx =
TII->getOperandIdx(
MI.getOpcode(), R600::OpName::dst);
229 if (!
MRI.use_empty(
MI.getOperand(DstIdx).getReg()) ||
230 MI.getOpcode() == R600::LDS_CMPST_RET)
242 case R600::FABS_R600: {
244 *BB,
I, R600::MOV,
MI.getOperand(0).getReg(),
245 MI.getOperand(1).getReg());
250 case R600::FNEG_R600: {
252 *BB,
I, R600::MOV,
MI.getOperand(0).getReg(),
253 MI.getOperand(1).getReg());
258 case R600::MASK_WRITE: {
259 Register maskedRegister =
MI.getOperand(0).getReg();
266 case R600::MOV_IMM_F32:
267 TII->buildMovImm(*BB,
I,
MI.getOperand(0).getReg(),
MI.getOperand(1)
274 case R600::MOV_IMM_I32:
275 TII->buildMovImm(*BB,
I,
MI.getOperand(0).getReg(),
276 MI.getOperand(1).getImm());
279 case R600::MOV_IMM_GLOBAL_ADDR: {
281 auto MIB =
TII->buildDefaultInstruction(
282 *BB,
MI, R600::MOV,
MI.getOperand(0).getReg(), R600::ALU_LITERAL_X);
283 int Idx =
TII->getOperandIdx(*MIB, R600::OpName::literal);
291 case R600::CONST_COPY: {
293 *BB,
MI, R600::MOV,
MI.getOperand(0).getReg(), R600::ALU_CONST);
294 TII->setImmOperand(*NewMI, R600::OpName::src0_sel,
295 MI.getOperand(1).getImm());
299 case R600::RAT_WRITE_CACHELESS_32_eg:
300 case R600::RAT_WRITE_CACHELESS_64_eg:
301 case R600::RAT_WRITE_CACHELESS_128_eg:
303 .
add(
MI.getOperand(0))
304 .
add(
MI.getOperand(1))
308 case R600::RAT_STORE_TYPED_eg:
310 .
add(
MI.getOperand(0))
311 .
add(
MI.getOperand(1))
312 .
add(
MI.getOperand(2))
318 .
add(
MI.getOperand(0));
321 case R600::BRANCH_COND_f32: {
325 .
add(
MI.getOperand(1))
330 .
add(
MI.getOperand(0))
335 case R600::BRANCH_COND_i32: {
339 .
add(
MI.getOperand(1))
340 .
addImm(R600::PRED_SETNE_INT)
344 .
add(
MI.getOperand(0))
349 case R600::EG_ExportSwz:
350 case R600::R600_ExportSwz: {
352 bool isLastInstructionOfItsType =
true;
353 unsigned InstExportType =
MI.getOperand(1).getImm();
355 EndBlock = BB->
end(); NextExportInst != EndBlock;
356 NextExportInst = std::next(NextExportInst)) {
357 if (NextExportInst->getOpcode() == R600::EG_ExportSwz ||
358 NextExportInst->getOpcode() == R600::R600_ExportSwz) {
359 unsigned CurrentInstExportType = NextExportInst->getOperand(1)
361 if (CurrentInstExportType == InstExportType) {
362 isLastInstructionOfItsType =
false;
368 if (!EOP && !isLastInstructionOfItsType)
370 unsigned CfInst = (
MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40;
372 .
add(
MI.getOperand(0))
373 .
add(
MI.getOperand(1))
374 .
add(
MI.getOperand(2))
375 .
add(
MI.getOperand(3))
376 .
add(
MI.getOperand(4))
377 .
add(
MI.getOperand(5))
378 .
add(
MI.getOperand(6))
388 MI.eraseFromParent();
399 switch (
Op.getOpcode()) {
414 assert((!Result.getNode() ||
415 Result.getNode()->getNumValues() == 2) &&
416 "Load should return a value and a chain");
424 return lowerADDRSPACECAST(
Op, DAG);
427 unsigned IntrinsicID =
Op.getConstantOperandVal(1);
428 switch (IntrinsicID) {
429 case Intrinsic::r600_store_swizzle: {
451 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
452 EVT VT =
Op.getValueType();
454 switch (IntrinsicID) {
455 case Intrinsic::r600_tex:
456 case Intrinsic::r600_texc: {
458 switch (IntrinsicID) {
459 case Intrinsic::r600_tex:
462 case Intrinsic::r600_texc:
492 case Intrinsic::r600_dot4: {
514 case Intrinsic::r600_implicitarg_ptr: {
519 case Intrinsic::r600_read_ngroups_x:
520 return LowerImplicitParameter(DAG, VT,
DL, 0);
521 case Intrinsic::r600_read_ngroups_y:
522 return LowerImplicitParameter(DAG, VT,
DL, 1);
523 case Intrinsic::r600_read_ngroups_z:
524 return LowerImplicitParameter(DAG, VT,
DL, 2);
525 case Intrinsic::r600_read_global_size_x:
526 return LowerImplicitParameter(DAG, VT,
DL, 3);
527 case Intrinsic::r600_read_global_size_y:
528 return LowerImplicitParameter(DAG, VT,
DL, 4);
529 case Intrinsic::r600_read_global_size_z:
530 return LowerImplicitParameter(DAG, VT,
DL, 5);
531 case Intrinsic::r600_read_local_size_x:
532 return LowerImplicitParameter(DAG, VT,
DL, 6);
533 case Intrinsic::r600_read_local_size_y:
534 return LowerImplicitParameter(DAG, VT,
DL, 7);
535 case Intrinsic::r600_read_local_size_z:
536 return LowerImplicitParameter(DAG, VT,
DL, 8);
538 case Intrinsic::r600_read_tgid_x:
539 case Intrinsic::amdgcn_workgroup_id_x:
542 case Intrinsic::r600_read_tgid_y:
543 case Intrinsic::amdgcn_workgroup_id_y:
546 case Intrinsic::r600_read_tgid_z:
547 case Intrinsic::amdgcn_workgroup_id_z:
550 case Intrinsic::r600_read_tidig_x:
551 case Intrinsic::amdgcn_workitem_id_x:
554 case Intrinsic::r600_read_tidig_y:
555 case Intrinsic::amdgcn_workitem_id_y:
558 case Intrinsic::r600_read_tidig_z:
559 case Intrinsic::amdgcn_workitem_id_z:
563 case Intrinsic::r600_recipsqrt_ieee:
566 case Intrinsic::r600_recipsqrt_clamped:
582 switch (
N->getOpcode()) {
587 if (
N->getValueType(0) == MVT::i1) {
588 Results.push_back(lowerFP_TO_UINT(
N->getOperand(0), DAG));
596 if (
N->getValueType(0) == MVT::i1) {
597 Results.push_back(lowerFP_TO_SINT(
N->getOperand(0), DAG));
642 if (isa<ConstantSDNode>(
Index) ||
658 if (isa<ConstantSDNode>(
Index) ||
665 return vectorToVerticalVector(DAG, Insert);
686 EVT VT =
Op.getValueType();
697 switch (
Op.getOpcode()) {
725 unsigned mainop,
unsigned ovf)
const {
727 EVT VT =
Op.getValueType();
764 unsigned DwordOffset)
const {
765 unsigned ByteOffset = DwordOffset * 4;
770 assert(isInt<16>(ByteOffset));
777bool R600TargetLowering::isZero(
SDValue Op)
const {
779 return Cst->isZero();
781 return CstFP->isZero();
787bool R600TargetLowering::isHWTrueValue(
SDValue Op)
const {
789 return CFP->isExactlyValue(1.0);
794bool R600TargetLowering::isHWFalseValue(
SDValue Op)
const {
796 return CFP->getValueAPF().isZero();
803 EVT VT =
Op.getValueType();
812 if (VT == MVT::f32) {
820 EVT CompareVT =
LHS.getValueType();
834 if (isHWTrueValue(False) && isHWFalseValue(True)) {
850 if (isHWTrueValue(True) && isHWFalseValue(False) &&
851 (CompareVT == VT || VT == MVT::i32)) {
889 if (CompareVT != VT) {
921 if (CompareVT == MVT::f32) {
924 }
else if (CompareVT == MVT::i32) {
945 EVT VT =
Op.getValueType();
955 return DAG.
getConstant(
TM.getNullPointerValue(DestAS), SL, VT);
987void R600TargetLowering::getStackAddress(
unsigned StackWidth,
990 unsigned &PtrIncr)
const {
991 switch (StackWidth) {
1002 Channel = ElemIdx % 2;
1021 ||
Store->getValue().getValueType() == MVT::i8);
1025 if (
Store->getMemoryVT() == MVT::i8) {
1028 }
else if (
Store->getMemoryVT() == MVT::i16) {
1058 Chain = Dst.getValue(1);
1078 MaskedValue, ShiftAmt);
1085 DstMask = DAG.
getNOT(
DL, DstMask, MVT::i32);
1116 EVT PtrVT =
Ptr.getValueType();
1134 StoreNode = cast<StoreSDNode>(NewStore);
1154 if (TruncatingStore) {
1157 if (MemVT == MVT::i8) {
1160 assert(MemVT == MVT::i16);
1188 Op->getVTList(), Args, MemVT,
1207 if (MemVT.
bitsLT(MVT::i32))
1208 return lowerPrivateTruncStore(StoreNode, DAG);
1230 return 512 + 4096 * 2;
1232 return 512 + 4096 * 3;
1234 return 512 + 4096 * 4;
1236 return 512 + 4096 * 5;
1238 return 512 + 4096 * 6;
1240 return 512 + 4096 * 7;
1242 return 512 + 4096 * 8;
1244 return 512 + 4096 * 9;
1246 return 512 + 4096 * 10;
1248 return 512 + 4096 * 11;
1250 return 512 + 4096 * 12;
1252 return 512 + 4096 * 13;
1254 return 512 + 4096 * 14;
1256 return 512 + 4096 * 15;
1267 EVT MemVT =
Load->getMemoryVT();
1326 return lowerPrivateExtLoad(
Op, DAG);
1330 EVT VT =
Op.getValueType();
1344 if (ConstantBlock > -1 &&
1349 isa<ConstantSDNode>(
Ptr)) {
1382 assert(!MemVT.
isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
1389 SDValue MergedValues[2] = { Res, Chain };
1432 bool IsVarArg)
const {
1472 for (
unsigned i = 0, e = Ins.size(); i < e; ++i) {
1544 unsigned *IsFast)
const {
1548 if (!VT.
isSimple() || VT == MVT::Other)
1558 return VT.
bitsGT(MVT::i32) && Alignment >=
Align(4);
1570 for (
unsigned i = 0; i < 4; i++)
1574 for (
unsigned i = 0; i < 4; i++) {
1579 RemapSwizzle[i] = 7;
1582 RemapSwizzle[i] = 4;
1583 NewBldVec[i] = DAG.
getUNDEF(MVT::f32);
1584 }
else if (
C->isExactlyValue(1.0)) {
1585 RemapSwizzle[i] = 5;
1586 NewBldVec[i] = DAG.
getUNDEF(MVT::f32);
1593 for (
unsigned j = 0; j < i; j++) {
1594 if (NewBldVec[i] == NewBldVec[j]) {
1595 NewBldVec[i] = DAG.
getUNDEF(NewBldVec[i].getValueType());
1596 RemapSwizzle[i] = j;
1614 bool isUnmovable[4] = {
false,
false,
false,
false};
1615 for (
unsigned i = 0; i < 4; i++)
1619 for (
unsigned i = 0; i < 4; i++) {
1620 RemapSwizzle[i] = i;
1624 isUnmovable[
Idx] =
true;
1628 for (
unsigned i = 0; i < 4; i++) {
1631 if (isUnmovable[
Idx])
1651 for (
unsigned i = 0; i < 4; i++) {
1657 SwizzleRemap.
clear();
1659 for (
unsigned i = 0; i < 4; i++) {
1686 for (
unsigned i = 0; i < 4; i++) {
1696 EVT NewVT = MVT::v4i32;
1697 unsigned NumElements = 4;
1723 switch (
N->getOpcode()) {
1779 if (!isa<ConstantSDNode>(EltNo))
1798 if (Elt < Ops.
size()) {
1801 EVT OpVT = Ops[0].getValueType();
1818 if (
ConstantSDNode *Const = dyn_cast<ConstantSDNode>(
N->getOperand(1))) {
1819 unsigned Element = Const->getZExtValue();
1827 if (
ConstantSDNode *Const = dyn_cast<ConstantSDNode>(
N->getOperand(1))) {
1828 unsigned Element = Const->getZExtValue();
1854 ISD::CondCode NCC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
1856 if (
LHS.getOperand(2).getNode() != True.
getNode() ||
1857 LHS.getOperand(3).getNode() != False.
getNode() ||
1897 NewArgs[1] = OptimizeSwizzle(
N->getOperand(1), &NewArgs[4], DAG,
DL);
1926 NewArgs[1] = OptimizeSwizzle(
N->getOperand(1), &NewArgs[2], DAG,
DL);
1934 isa<ConstantSDNode>(
Ptr))
1945bool R600TargetLowering::FoldOperand(
SDNode *ParentNode,
unsigned SrcIdx,
1950 if (!Src.isMachineOpcode())
1953 switch (Src.getMachineOpcode()) {
1954 case R600::FNEG_R600:
1957 Src = Src.getOperand(0);
1960 case R600::FABS_R600:
1963 Src = Src.getOperand(0);
1966 case R600::CONST_COPY: {
1968 bool HasDst =
TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
1973 SDValue CstOffset = Src.getOperand(0);
1978 int SrcIndices[] = {
1979 TII->getOperandIdx(Opcode, R600::OpName::src0),
1980 TII->getOperandIdx(Opcode, R600::OpName::src1),
1981 TII->getOperandIdx(Opcode, R600::OpName::src2),
1982 TII->getOperandIdx(Opcode, R600::OpName::src0_X),
1983 TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
1984 TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
1985 TII->getOperandIdx(Opcode, R600::OpName::src0_W),
1986 TII->getOperandIdx(Opcode, R600::OpName::src1_X),
1987 TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
1988 TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
1989 TII->getOperandIdx(Opcode, R600::OpName::src1_W)
1991 std::vector<unsigned> Consts;
1992 for (
int OtherSrcIdx : SrcIndices) {
1993 int OtherSelIdx =
TII->getSelIdx(Opcode, OtherSrcIdx);
1994 if (OtherSrcIdx < 0 || OtherSelIdx < 0)
2001 dyn_cast<RegisterSDNode>(ParentNode->
getOperand(OtherSrcIdx))) {
2002 if (
Reg->getReg() == R600::ALU_CONST) {
2010 if (!
TII->fitsConstReadLimitations(Consts)) {
2018 case R600::MOV_IMM_GLOBAL_ADDR:
2020 if (
Imm->getAsZExtVal())
2022 Imm = Src.getOperand(0);
2023 Src = DAG.
getRegister(R600::ALU_LITERAL_X, MVT::i32);
2025 case R600::MOV_IMM_I32:
2026 case R600::MOV_IMM_F32: {
2027 unsigned ImmReg = R600::ALU_LITERAL_X;
2030 if (Src.getMachineOpcode() == R600::MOV_IMM_F32) {
2033 if (FloatValue == 0.0) {
2034 ImmReg = R600::ZERO;
2035 }
else if (FloatValue == 0.5) {
2036 ImmReg = R600::HALF;
2037 }
else if (FloatValue == 1.0) {
2045 ImmReg = R600::ZERO;
2046 }
else if (
Value == 1) {
2047 ImmReg = R600::ONE_INT;
2056 if (ImmReg == R600::ALU_LITERAL_X) {
2060 if (
C->getZExtValue())
2076 if (!
Node->isMachineOpcode())
2079 unsigned Opcode =
Node->getMachineOpcode();
2082 std::vector<SDValue> Ops(
Node->op_begin(),
Node->op_end());
2084 if (Opcode == R600::DOT_4) {
2085 int OperandIdx[] = {
2086 TII->getOperandIdx(Opcode, R600::OpName::src0_X),
2087 TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
2088 TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
2089 TII->getOperandIdx(Opcode, R600::OpName::src0_W),
2090 TII->getOperandIdx(Opcode, R600::OpName::src1_X),
2091 TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
2092 TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
2093 TII->getOperandIdx(Opcode, R600::OpName::src1_W)
2096 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_X),
2097 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Y),
2098 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Z),
2099 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_W),
2100 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_X),
2101 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Y),
2102 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Z),
2103 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_W)
2106 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_X),
2107 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Y),
2108 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Z),
2109 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_W),
2110 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_X),
2111 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Y),
2112 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Z),
2113 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_W)
2115 for (
unsigned i = 0; i < 8; i++) {
2116 if (OperandIdx[i] < 0)
2118 SDValue &Src = Ops[OperandIdx[i] - 1];
2119 SDValue &Neg = Ops[NegIdx[i] - 1];
2120 SDValue &Abs = Ops[AbsIdx[i] - 1];
2121 bool HasDst =
TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2122 int SelIdx =
TII->getSelIdx(Opcode, OperandIdx[i]);
2125 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2126 if (FoldOperand(
Node, i, Src, Neg, Abs, Sel, FakeOp, DAG))
2129 }
else if (Opcode == R600::REG_SEQUENCE) {
2130 for (
unsigned i = 1, e =
Node->getNumOperands(); i <
e; i += 2) {
2132 if (FoldOperand(
Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG))
2136 if (!
TII->hasInstrModifiers(Opcode))
2138 int OperandIdx[] = {
2139 TII->getOperandIdx(Opcode, R600::OpName::src0),
2140 TII->getOperandIdx(Opcode, R600::OpName::src1),
2141 TII->getOperandIdx(Opcode, R600::OpName::src2)
2144 TII->getOperandIdx(Opcode, R600::OpName::src0_neg),
2145 TII->getOperandIdx(Opcode, R600::OpName::src1_neg),
2146 TII->getOperandIdx(Opcode, R600::OpName::src2_neg)
2149 TII->getOperandIdx(Opcode, R600::OpName::src0_abs),
2150 TII->getOperandIdx(Opcode, R600::OpName::src1_abs),
2153 for (
unsigned i = 0; i < 3; i++) {
2154 if (OperandIdx[i] < 0)
2156 SDValue &Src = Ops[OperandIdx[i] - 1];
2157 SDValue &Neg = Ops[NegIdx[i] - 1];
2159 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2160 bool HasDst =
TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2161 int SelIdx =
TII->getSelIdx(Opcode, OperandIdx[i]);
2162 int ImmIdx =
TII->getOperandIdx(Opcode, R600::OpName::literal);
2167 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2169 if (FoldOperand(
Node, i, Src, Neg, Abs, Sel, Imm, DAG))
2178R600TargetLowering::shouldExpandAtomicRMWInIR(
AtomicRMWInst *RMW)
const {
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
static bool isUndef(ArrayRef< int > Mask)
const char LLVMTargetMachineRef TM
static bool isEOP(MachineBasicBlock::iterator I)
static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, DenseMap< unsigned, unsigned > &RemapSwizzle)
static int ConstantAddressBlock(unsigned AddressSpace)
static SDValue CompactSwizzlableVector(SelectionDAG &DAG, SDValue VectorEntry, DenseMap< unsigned, unsigned > &RemapSwizzle)
R600 DAG Lowering interface definition.
Interface definition for R600InstrInfo.
Provides R600 specific target descriptions.
AMDGPU R600 specific subclass of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
unsigned getStackWidth(const MachineFunction &MF) const
SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, DAGCombinerInfo &DCI) const
Generate Min/Max node.
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue CreateLiveInRegisterRaw(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT) const
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
float convertToFloat() const
Converts this APFloat to host float value.
APInt bitcastToAPInt() const
uint64_t getZExtValue() const
Get zero extended value.
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
an instruction that atomically reads a memory location, combines it with another value,...
@ UIncWrap
Increment one up to a maximum value.
@ UDecWrap
Decrement one until a minimum value or zero.
BinOp getOperation() const
CCState - This class holds information needed while lowering arguments and return values.
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
int64_t getLocMemOffset() const
const APFloat & getValueAPF() const
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
uint64_t getZExtValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static auto integer_valuetypes()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
unsigned getTargetFlags() const
int64_t getOffset() const
Return the offset from the symbol in this operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
const R600FrameLowering * getFrameLowering() const override
const R600RegisterInfo * getRegisterInfo() const override
const R600InstrInfo * getInstrInfo() const override
bool hasBCNT(unsigned Size) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
R600TargetLowering(const TargetMachine &TM, const R600Subtarget &STI)
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT VT) const override
Return the ValueType of the result of SETCC operations.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const override
Determine if the target supports unaligned memory accesses.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
XXX Only kernel functions are supported, so we can assume for now that every function is a kernel fun...
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
const TargetMachine & getTargetMachine() const
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrNegativeOneBooleanContent
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
Primary interface to the complete machine description for the target machine.
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ PARAM_I_ADDRESS
Address space for indirect addressable parameter memory (VTX1).
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUILD_VERTICAL_VECTOR
This node is for VLIW targets and it is used to represent a vector that is stored in consecutive regi...
@ CONST_DATA_PTR
Pointer to the start of the shader's constant data.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
bool isShader(CallingConv::ID cc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ FADD
Simple binary floating point operators.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
int getLDSNoRetOp(uint16_t Opcode)
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
DWARFExpression::Operation Op
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This class contains a discriminated union of information about pointers in memory operands,...
bool isBeforeLegalizeOps() const