LLVM  3.7.0
AMDGPUISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- AMDILISelDAGToDAG.cpp - A dag to dag inst selector for AMDIL ------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //==-----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Defines an instruction selector for the AMDGPU target.
12 //
13 //===----------------------------------------------------------------------===//
14 #include "AMDGPUInstrInfo.h"
15 #include "AMDGPUISelLowering.h" // For AMDGPUISD
16 #include "AMDGPURegisterInfo.h"
17 #include "AMDGPUSubtarget.h"
18 #include "R600InstrInfo.h"
19 #include "SIDefines.h"
20 #include "SIISelLowering.h"
21 #include "SIMachineFunctionInfo.h"
28 #include "llvm/IR/Function.h"
29 
30 using namespace llvm;
31 
32 //===----------------------------------------------------------------------===//
33 // Instruction Selector Implementation
34 //===----------------------------------------------------------------------===//
35 
36 namespace {
37 /// AMDGPU specific code to select AMDGPU machine instructions for
38 /// SelectionDAG operations.
39 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
40  // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
41  // make the right decision when generating code for different targets.
42  const AMDGPUSubtarget *Subtarget;
43 public:
44  AMDGPUDAGToDAGISel(TargetMachine &TM);
45  virtual ~AMDGPUDAGToDAGISel();
46  bool runOnMachineFunction(MachineFunction &MF) override;
47  SDNode *Select(SDNode *N) override;
48  const char *getPassName() const override;
49  void PostprocessISelDAG() override;
50 
51 private:
52  bool isInlineImmediate(SDNode *N) const;
53  bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
54  const R600InstrInfo *TII);
55  bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
56  bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
57 
58  // Complex pattern selectors
59  bool SelectADDRParam(SDValue Addr, SDValue& R1, SDValue& R2);
60  bool SelectADDR(SDValue N, SDValue &R1, SDValue &R2);
61  bool SelectADDR64(SDValue N, SDValue &R1, SDValue &R2);
62 
63  static bool checkType(const Value *ptr, unsigned int addrspace);
64  static bool checkPrivateAddress(const MachineMemOperand *Op);
65 
66  static bool isGlobalStore(const StoreSDNode *N);
67  static bool isFlatStore(const StoreSDNode *N);
68  static bool isPrivateStore(const StoreSDNode *N);
69  static bool isLocalStore(const StoreSDNode *N);
70  static bool isRegionStore(const StoreSDNode *N);
71 
72  bool isCPLoad(const LoadSDNode *N) const;
73  bool isConstantLoad(const LoadSDNode *N, int cbID) const;
74  bool isGlobalLoad(const LoadSDNode *N) const;
75  bool isFlatLoad(const LoadSDNode *N) const;
76  bool isParamLoad(const LoadSDNode *N) const;
77  bool isPrivateLoad(const LoadSDNode *N) const;
78  bool isLocalLoad(const LoadSDNode *N) const;
79  bool isRegionLoad(const LoadSDNode *N) const;
80 
81  SDNode *glueCopyToM0(SDNode *N) const;
82 
83  const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
84  bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
85  bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
86  SDValue& Offset);
87  bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
88  bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
89  bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
90  unsigned OffsetBits) const;
91  bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
92  bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
93  SDValue &Offset1) const;
94  void SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
95  SDValue &SOffset, SDValue &Offset, SDValue &Offen,
96  SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
97  SDValue &TFE) const;
98  bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
99  SDValue &SOffset, SDValue &Offset, SDValue &GLC,
100  SDValue &SLC, SDValue &TFE) const;
101  bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
102  SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
103  SDValue &SLC) const;
104  bool SelectMUBUFScratch(SDValue Addr, SDValue &RSrc, SDValue &VAddr,
105  SDValue &SOffset, SDValue &ImmOffset) const;
106  bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
107  SDValue &Offset, SDValue &GLC, SDValue &SLC,
108  SDValue &TFE) const;
109  bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
110  SDValue &Offset, SDValue &GLC) const;
111  SDNode *SelectAddrSpaceCast(SDNode *N);
112  bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
113  bool SelectVOP3NoMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
114  bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
115  SDValue &Clamp, SDValue &Omod) const;
116  bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
117  SDValue &Clamp, SDValue &Omod) const;
118 
119  bool SelectVOP3Mods0Clamp(SDValue In, SDValue &Src, SDValue &SrcMods,
120  SDValue &Omod) const;
121  bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
122  SDValue &Clamp,
123  SDValue &Omod) const;
124 
125  SDNode *SelectADD_SUB_I64(SDNode *N);
126  SDNode *SelectDIV_SCALE(SDNode *N);
127 
128  SDNode *getS_BFE(unsigned Opcode, SDLoc DL, SDValue Val,
129  uint32_t Offset, uint32_t Width);
130  SDNode *SelectS_BFEFromShifts(SDNode *N);
131  SDNode *SelectS_BFE(SDNode *N);
132 
133  // Include the pieces autogenerated from the target description.
134 #include "AMDGPUGenDAGISel.inc"
135 };
136 } // end anonymous namespace
137 
138 /// \brief This pass converts a legalized DAG into a AMDGPU-specific
139 // DAG, ready for instruction scheduling.
141  return new AMDGPUDAGToDAGISel(TM);
142 }
143 
144 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM)
145  : SelectionDAGISel(TM) {}
146 
147 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
148  Subtarget = &static_cast<const AMDGPUSubtarget &>(MF.getSubtarget());
149  return SelectionDAGISel::runOnMachineFunction(MF);
150 }
151 
152 AMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel() {
153 }
154 
155 bool AMDGPUDAGToDAGISel::isInlineImmediate(SDNode *N) const {
156  const SITargetLowering *TL
157  = static_cast<const SITargetLowering *>(getTargetLowering());
158  return TL->analyzeImmediate(N) == 0;
159 }
160 
161 /// \brief Determine the register class for \p OpNo
162 /// \returns The register class of the virtual register that will be used for
163 /// the given operand number \OpNo or NULL if the register class cannot be
164 /// determined.
165 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
166  unsigned OpNo) const {
167  if (!N->isMachineOpcode())
168  return nullptr;
169 
170  switch (N->getMachineOpcode()) {
171  default: {
172  const MCInstrDesc &Desc =
173  Subtarget->getInstrInfo()->get(N->getMachineOpcode());
174  unsigned OpIdx = Desc.getNumDefs() + OpNo;
175  if (OpIdx >= Desc.getNumOperands())
176  return nullptr;
177  int RegClass = Desc.OpInfo[OpIdx].RegClass;
178  if (RegClass == -1)
179  return nullptr;
180 
181  return Subtarget->getRegisterInfo()->getRegClass(RegClass);
182  }
183  case AMDGPU::REG_SEQUENCE: {
184  unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
185  const TargetRegisterClass *SuperRC =
186  Subtarget->getRegisterInfo()->getRegClass(RCID);
187 
188  SDValue SubRegOp = N->getOperand(OpNo + 1);
189  unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
190  return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
191  SubRegIdx);
192  }
193  }
194 }
195 
196 bool AMDGPUDAGToDAGISel::SelectADDRParam(
197  SDValue Addr, SDValue& R1, SDValue& R2) {
198 
199  if (Addr.getOpcode() == ISD::FrameIndex) {
200  if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
201  R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32);
202  R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
203  } else {
204  R1 = Addr;
205  R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
206  }
207  } else if (Addr.getOpcode() == ISD::ADD) {
208  R1 = Addr.getOperand(0);
209  R2 = Addr.getOperand(1);
210  } else {
211  R1 = Addr;
212  R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
213  }
214  return true;
215 }
216 
217 bool AMDGPUDAGToDAGISel::SelectADDR(SDValue Addr, SDValue& R1, SDValue& R2) {
218  if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
220  return false;
221  }
222  return SelectADDRParam(Addr, R1, R2);
223 }
224 
225 
226 bool AMDGPUDAGToDAGISel::SelectADDR64(SDValue Addr, SDValue& R1, SDValue& R2) {
227  if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
229  return false;
230  }
231 
232  if (Addr.getOpcode() == ISD::FrameIndex) {
233  if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
234  R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64);
235  R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
236  } else {
237  R1 = Addr;
238  R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
239  }
240  } else if (Addr.getOpcode() == ISD::ADD) {
241  R1 = Addr.getOperand(0);
242  R2 = Addr.getOperand(1);
243  } else {
244  R1 = Addr;
245  R2 = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i64);
246  }
247  return true;
248 }
249 
250 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
251  if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
252  !checkType(cast<MemSDNode>(N)->getMemOperand()->getValue(),
254  return N;
255 
256  const SITargetLowering& Lowering =
257  *static_cast<const SITargetLowering*>(getTargetLowering());
258 
259  // Write max value to m0 before each load operation
260 
261  SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
262  CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
263 
264  SDValue Glue = M0.getValue(1);
265 
267  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
268  Ops.push_back(N->getOperand(i));
269  }
270  Ops.push_back(Glue);
271  CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
272 
273  return N;
274 }
275 
277  unsigned int Opc = N->getOpcode();
278  if (N->isMachineOpcode()) {
279  N->setNodeId(-1);
280  return nullptr; // Already selected.
281  }
282 
283  if (isa<AtomicSDNode>(N))
284  N = glueCopyToM0(N);
285 
286  switch (Opc) {
287  default: break;
288  // We are selecting i64 ADD here instead of custom lower it during
289  // DAG legalization, so we can fold some i64 ADDs used for address
290  // calculation into the LOAD and STORE instructions.
291  case ISD::ADD:
292  case ISD::SUB: {
293  if (N->getValueType(0) != MVT::i64 ||
294  Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
295  break;
296 
297  return SelectADD_SUB_I64(N);
298  }
301  case ISD::BUILD_VECTOR: {
302  unsigned RegClassID;
303  const AMDGPURegisterInfo *TRI = Subtarget->getRegisterInfo();
304  EVT VT = N->getValueType(0);
305  unsigned NumVectorElts = VT.getVectorNumElements();
306  EVT EltVT = VT.getVectorElementType();
307  assert(EltVT.bitsEq(MVT::i32));
308  if (Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
309  bool UseVReg = true;
310  for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
311  U != E; ++U) {
312  if (!U->isMachineOpcode()) {
313  continue;
314  }
315  const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
316  if (!RC) {
317  continue;
318  }
319  if (static_cast<const SIRegisterInfo *>(TRI)->isSGPRClass(RC)) {
320  UseVReg = false;
321  }
322  }
323  switch(NumVectorElts) {
324  case 1: RegClassID = UseVReg ? AMDGPU::VGPR_32RegClassID :
325  AMDGPU::SReg_32RegClassID;
326  break;
327  case 2: RegClassID = UseVReg ? AMDGPU::VReg_64RegClassID :
328  AMDGPU::SReg_64RegClassID;
329  break;
330  case 4: RegClassID = UseVReg ? AMDGPU::VReg_128RegClassID :
331  AMDGPU::SReg_128RegClassID;
332  break;
333  case 8: RegClassID = UseVReg ? AMDGPU::VReg_256RegClassID :
334  AMDGPU::SReg_256RegClassID;
335  break;
336  case 16: RegClassID = UseVReg ? AMDGPU::VReg_512RegClassID :
337  AMDGPU::SReg_512RegClassID;
338  break;
339  default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
340  }
341  } else {
342  // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
343  // that adds a 128 bits reg copy when going through TwoAddressInstructions
344  // pass. We want to avoid 128 bits copies as much as possible because they
345  // can't be bundled by our scheduler.
346  switch(NumVectorElts) {
347  case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
348  case 4:
350  RegClassID = AMDGPU::R600_Reg128VerticalRegClassID;
351  else
352  RegClassID = AMDGPU::R600_Reg128RegClassID;
353  break;
354  default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
355  }
356  }
357 
358  SDLoc DL(N);
359  SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
360 
361  if (NumVectorElts == 1) {
362  return CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT,
363  N->getOperand(0), RegClass);
364  }
365 
366  assert(NumVectorElts <= 16 && "Vectors with more than 16 elements not "
367  "supported yet");
368  // 16 = Max Num Vector Elements
369  // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
370  // 1 = Vector Register Class
371  SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
372 
373  RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
374  bool IsRegSeq = true;
375  unsigned NOps = N->getNumOperands();
376  for (unsigned i = 0; i < NOps; i++) {
377  // XXX: Why is this here?
378  if (isa<RegisterSDNode>(N->getOperand(i))) {
379  IsRegSeq = false;
380  break;
381  }
382  RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
383  RegSeqArgs[1 + (2 * i) + 1] =
384  CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL,
385  MVT::i32);
386  }
387 
388  if (NOps != NumVectorElts) {
389  // Fill in the missing undef elements if this was a scalar_to_vector.
390  assert(Opc == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
391 
392  MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
393  DL, EltVT);
394  for (unsigned i = NOps; i < NumVectorElts; ++i) {
395  RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
396  RegSeqArgs[1 + (2 * i) + 1] =
397  CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL, MVT::i32);
398  }
399  }
400 
401  if (!IsRegSeq)
402  break;
403  return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(),
404  RegSeqArgs);
405  }
406  case ISD::BUILD_PAIR: {
407  SDValue RC, SubReg0, SubReg1;
408  if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
409  break;
410  }
411  SDLoc DL(N);
412  if (N->getValueType(0) == MVT::i128) {
413  RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32);
414  SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
415  SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
416  } else if (N->getValueType(0) == MVT::i64) {
417  RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
418  SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
419  SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
420  } else {
421  llvm_unreachable("Unhandled value type for BUILD_PAIR");
422  }
423  const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
424  N->getOperand(1), SubReg1 };
425  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
426  DL, N->getValueType(0), Ops);
427  }
428 
429  case ISD::Constant:
430  case ISD::ConstantFP: {
431  if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
432  N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
433  break;
434 
435  uint64_t Imm;
436  if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
437  Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
438  else {
439  ConstantSDNode *C = cast<ConstantSDNode>(N);
440  Imm = C->getZExtValue();
441  }
442 
443  SDLoc DL(N);
444  SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
445  CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
446  MVT::i32));
447  SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
448  CurDAG->getConstant(Imm >> 32, DL, MVT::i32));
449  const SDValue Ops[] = {
450  CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
451  SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
452  SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
453  };
454 
455  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
456  N->getValueType(0), Ops);
457  }
458 
459  case ISD::LOAD: {
460  LoadSDNode *LD = cast<LoadSDNode>(N);
461  SDLoc SL(N);
462  EVT VT = N->getValueType(0);
463 
464  if (VT != MVT::i64 || LD->getExtensionType() != ISD::NON_EXTLOAD) {
465  N = glueCopyToM0(N);
466  break;
467  }
468 
469  // To simplify the TableGen patters, we replace all i64 loads with
470  // v2i32 loads. Alternatively, we could promote i64 loads to v2i32
471  // during DAG legalization, however, so places (ExpandUnalignedLoad)
472  // in the DAG legalizer assume that if i64 is legal, so doing this
473  // promotion early can cause problems.
474 
475  SDValue NewLoad = CurDAG->getLoad(MVT::v2i32, SDLoc(N), LD->getChain(),
476  LD->getBasePtr(), LD->getMemOperand());
477  SDValue BitCast = CurDAG->getNode(ISD::BITCAST, SL,
478  MVT::i64, NewLoad);
479  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLoad.getValue(1));
480  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), BitCast);
481  SDNode *Load = glueCopyToM0(NewLoad.getNode());
482  SelectCode(Load);
483  N = BitCast.getNode();
484  break;
485  }
486 
487  case ISD::STORE: {
488  // Handle i64 stores here for the same reason mentioned above for loads.
489  StoreSDNode *ST = cast<StoreSDNode>(N);
490  SDValue Value = ST->getValue();
491  if (Value.getValueType() == MVT::i64 && !ST->isTruncatingStore()) {
492 
493  SDValue NewValue = CurDAG->getNode(ISD::BITCAST, SDLoc(N),
494  MVT::v2i32, Value);
495  SDValue NewStore = CurDAG->getStore(ST->getChain(), SDLoc(N), NewValue,
496  ST->getBasePtr(), ST->getMemOperand());
497 
498  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewStore);
499 
500  if (NewValue.getOpcode() == ISD::BITCAST) {
501  Select(NewStore.getNode());
502  return SelectCode(NewValue.getNode());
503  }
504 
505  // getNode() may fold the bitcast if its input was another bitcast. If that
506  // happens we should only select the new store.
507  N = NewStore.getNode();
508  }
509 
510  N = glueCopyToM0(N);
511  break;
512  }
513 
515  if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS)
516  break;
517  SDValue Addr, Offset;
518 
519  SDLoc DL(N);
520  SelectADDRIndirect(N->getOperand(1), Addr, Offset);
521  const SDValue Ops[] = {
522  Addr,
523  Offset,
524  CurDAG->getTargetConstant(0, DL, MVT::i32),
525  N->getOperand(0),
526  };
527  return CurDAG->getMachineNode(AMDGPU::SI_RegisterLoad, DL,
528  CurDAG->getVTList(MVT::i32, MVT::i64,
529  MVT::Other),
530  Ops);
531  }
533  if (Subtarget->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS)
534  break;
535  SDValue Addr, Offset;
536  SelectADDRIndirect(N->getOperand(2), Addr, Offset);
537  SDLoc DL(N);
538  const SDValue Ops[] = {
539  N->getOperand(1),
540  Addr,
541  Offset,
542  CurDAG->getTargetConstant(0, DL, MVT::i32),
543  N->getOperand(0),
544  };
545  return CurDAG->getMachineNode(AMDGPU::SI_RegisterStorePseudo, DL,
546  CurDAG->getVTList(MVT::Other),
547  Ops);
548  }
549 
550  case AMDGPUISD::BFE_I32:
551  case AMDGPUISD::BFE_U32: {
552  if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
553  break;
554 
555  // There is a scalar version available, but unlike the vector version which
556  // has a separate operand for the offset and width, the scalar version packs
557  // the width and offset into a single operand. Try to move to the scalar
558  // version if the offsets are constant, so that we can try to keep extended
559  // loads of kernel arguments in SGPRs.
560 
561  // TODO: Technically we could try to pattern match scalar bitshifts of
562  // dynamic values, but it's probably not useful.
564  if (!Offset)
565  break;
566 
568  if (!Width)
569  break;
570 
571  bool Signed = Opc == AMDGPUISD::BFE_I32;
572 
573  uint32_t OffsetVal = Offset->getZExtValue();
574  uint32_t WidthVal = Width->getZExtValue();
575 
576  return getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, SDLoc(N),
577  N->getOperand(0), OffsetVal, WidthVal);
578 
579  }
580  case AMDGPUISD::DIV_SCALE: {
581  return SelectDIV_SCALE(N);
582  }
583  case ISD::CopyToReg: {
584  const SITargetLowering& Lowering =
585  *static_cast<const SITargetLowering*>(getTargetLowering());
586  Lowering.legalizeTargetIndependentNode(N, *CurDAG);
587  break;
588  }
589  case ISD::ADDRSPACECAST:
590  return SelectAddrSpaceCast(N);
591  case ISD::AND:
592  case ISD::SRL:
593  case ISD::SRA:
594  if (N->getValueType(0) != MVT::i32 ||
595  Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
596  break;
597 
598  return SelectS_BFE(N);
599  }
600 
601  return SelectCode(N);
602 }
603 
604 
605 bool AMDGPUDAGToDAGISel::checkType(const Value *Ptr, unsigned AS) {
606  assert(AS != 0 && "Use checkPrivateAddress instead.");
607  if (!Ptr)
608  return false;
609 
610  return Ptr->getType()->getPointerAddressSpace() == AS;
611 }
612 
613 bool AMDGPUDAGToDAGISel::checkPrivateAddress(const MachineMemOperand *Op) {
614  if (Op->getPseudoValue())
615  return true;
616 
617  if (PointerType *PT = dyn_cast<PointerType>(Op->getValue()->getType()))
618  return PT->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS;
619 
620  return false;
621 }
622 
623 bool AMDGPUDAGToDAGISel::isGlobalStore(const StoreSDNode *N) {
625 }
626 
627 bool AMDGPUDAGToDAGISel::isPrivateStore(const StoreSDNode *N) {
628  const Value *MemVal = N->getMemOperand()->getValue();
629  return (!checkType(MemVal, AMDGPUAS::LOCAL_ADDRESS) &&
632 }
633 
634 bool AMDGPUDAGToDAGISel::isLocalStore(const StoreSDNode *N) {
636 }
637 
638 bool AMDGPUDAGToDAGISel::isFlatStore(const StoreSDNode *N) {
640 }
641 
642 bool AMDGPUDAGToDAGISel::isRegionStore(const StoreSDNode *N) {
644 }
645 
646 bool AMDGPUDAGToDAGISel::isConstantLoad(const LoadSDNode *N, int CbId) const {
647  const Value *MemVal = N->getMemOperand()->getValue();
648  if (CbId == -1)
649  return checkType(MemVal, AMDGPUAS::CONSTANT_ADDRESS);
650 
651  return checkType(MemVal, AMDGPUAS::CONSTANT_BUFFER_0 + CbId);
652 }
653 
654 bool AMDGPUDAGToDAGISel::isGlobalLoad(const LoadSDNode *N) const {
656  if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
657  N->getMemoryVT().bitsLT(MVT::i32))
658  return true;
659 
661 }
662 
663 bool AMDGPUDAGToDAGISel::isParamLoad(const LoadSDNode *N) const {
665 }
666 
667 bool AMDGPUDAGToDAGISel::isLocalLoad(const LoadSDNode *N) const {
669 }
670 
671 bool AMDGPUDAGToDAGISel::isFlatLoad(const LoadSDNode *N) const {
673 }
674 
675 bool AMDGPUDAGToDAGISel::isRegionLoad(const LoadSDNode *N) const {
677 }
678 
679 bool AMDGPUDAGToDAGISel::isCPLoad(const LoadSDNode *N) const {
680  MachineMemOperand *MMO = N->getMemOperand();
681  if (checkPrivateAddress(N->getMemOperand())) {
682  if (MMO) {
683  const PseudoSourceValue *PSV = MMO->getPseudoValue();
684  if (PSV && PSV == PseudoSourceValue::getConstantPool()) {
685  return true;
686  }
687  }
688  }
689  return false;
690 }
691 
692 bool AMDGPUDAGToDAGISel::isPrivateLoad(const LoadSDNode *N) const {
693  if (checkPrivateAddress(N->getMemOperand())) {
694  // Check to make sure we are not a constant pool load or a constant load
695  // that is marked as a private load
696  if (isCPLoad(N) || isConstantLoad(N, -1)) {
697  return false;
698  }
699  }
700 
701  const Value *MemVal = N->getMemOperand()->getValue();
702  if (!checkType(MemVal, AMDGPUAS::LOCAL_ADDRESS) &&
704  !checkType(MemVal, AMDGPUAS::FLAT_ADDRESS) &&
709  return true;
710  }
711  return false;
712 }
713 
714 const char *AMDGPUDAGToDAGISel::getPassName() const {
715  return "AMDGPU DAG->DAG Pattern Instruction Selection";
716 }
717 
718 #ifdef DEBUGTMP
719 #undef INT64_C
720 #endif
721 #undef DEBUGTMP
722 
723 //===----------------------------------------------------------------------===//
724 // Complex Patterns
725 //===----------------------------------------------------------------------===//
726 
727 bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
728  SDValue& IntPtr) {
729  if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
730  IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
731  true);
732  return true;
733  }
734  return false;
735 }
736 
737 bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
738  SDValue& BaseReg, SDValue &Offset) {
739  if (!isa<ConstantSDNode>(Addr)) {
740  BaseReg = Addr;
741  Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
742  return true;
743  }
744  return false;
745 }
746 
747 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
748  SDValue &Offset) {
749  ConstantSDNode *IMMOffset;
750 
751  if (Addr.getOpcode() == ISD::ADD
752  && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
753  && isInt<16>(IMMOffset->getZExtValue())) {
754 
755  Base = Addr.getOperand(0);
756  Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
757  MVT::i32);
758  return true;
759  // If the pointer address is constant, we can move it to the offset field.
760  } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
761  && isInt<16>(IMMOffset->getZExtValue())) {
762  Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
763  SDLoc(CurDAG->getEntryNode()),
764  AMDGPU::ZERO, MVT::i32);
765  Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
766  MVT::i32);
767  return true;
768  }
769 
770  // Default case, no offset
771  Base = Addr;
772  Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
773  return true;
774 }
775 
776 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
777  SDValue &Offset) {
778  ConstantSDNode *C;
779  SDLoc DL(Addr);
780 
781  if ((C = dyn_cast<ConstantSDNode>(Addr))) {
782  Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
783  Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
784  } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
785  (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
786  Base = Addr.getOperand(0);
787  Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
788  } else {
789  Base = Addr;
790  Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
791  }
792 
793  return true;
794 }
795 
796 SDNode *AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
797  SDLoc DL(N);
798  SDValue LHS = N->getOperand(0);
799  SDValue RHS = N->getOperand(1);
800 
801  bool IsAdd = (N->getOpcode() == ISD::ADD);
802 
803  SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
804  SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
805 
806  SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
807  DL, MVT::i32, LHS, Sub0);
808  SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
809  DL, MVT::i32, LHS, Sub1);
810 
811  SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
812  DL, MVT::i32, RHS, Sub0);
813  SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
814  DL, MVT::i32, RHS, Sub1);
815 
816  SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
817  SDValue AddLoArgs[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
818 
819 
820  unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
821  unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
822 
823  SDNode *AddLo = CurDAG->getMachineNode( Opc, DL, VTList, AddLoArgs);
824  SDValue Carry(AddLo, 1);
825  SDNode *AddHi
826  = CurDAG->getMachineNode(CarryOpc, DL, MVT::i32,
827  SDValue(Hi0, 0), SDValue(Hi1, 0), Carry);
828 
829  SDValue Args[5] = {
830  CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
831  SDValue(AddLo,0),
832  Sub0,
833  SDValue(AddHi,0),
834  Sub1,
835  };
836  return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, MVT::i64, Args);
837 }
838 
839 // We need to handle this here because tablegen doesn't support matching
840 // instructions with multiple outputs.
841 SDNode *AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
842  SDLoc SL(N);
843  EVT VT = N->getValueType(0);
844 
845  assert(VT == MVT::f32 || VT == MVT::f64);
846 
847  unsigned Opc
848  = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
849 
850  // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod
851  SDValue Ops[8];
852 
853  SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
854  SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]);
855  SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]);
856  return CurDAG->SelectNodeTo(N, Opc, VT, MVT::i1, Ops);
857 }
858 
859 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
860  unsigned OffsetBits) const {
861  if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
862  (OffsetBits == 8 && !isUInt<8>(Offset)))
863  return false;
864 
865  if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS ||
866  Subtarget->unsafeDSOffsetFoldingEnabled())
867  return true;
868 
869  // On Southern Islands instruction with a negative base value and an offset
870  // don't seem to work.
871  return CurDAG->SignBitIsZero(Base);
872 }
873 
874 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
875  SDValue &Offset) const {
876  if (CurDAG->isBaseWithConstantOffset(Addr)) {
877  SDValue N0 = Addr.getOperand(0);
878  SDValue N1 = Addr.getOperand(1);
879  ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
880  if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
881  // (add n0, c0)
882  Base = N0;
883  Offset = N1;
884  return true;
885  }
886  }
887 
888  SDLoc DL(Addr);
889 
890  // If we have a constant address, prefer to put the constant into the
891  // offset. This can save moves to load the constant address since multiple
892  // operations can share the zero base address register, and enables merging
893  // into read2 / write2 instructions.
894  if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
895  if (isUInt<16>(CAddr->getZExtValue())) {
896  SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
897  MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
898  DL, MVT::i32, Zero);
899  Base = SDValue(MovZero, 0);
900  Offset = Addr;
901  return true;
902  }
903  }
904 
905  // default case
906  Base = Addr;
907  Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
908  return true;
909 }
910 
911 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
912  SDValue &Offset0,
913  SDValue &Offset1) const {
914  SDLoc DL(Addr);
915 
916  if (CurDAG->isBaseWithConstantOffset(Addr)) {
917  SDValue N0 = Addr.getOperand(0);
918  SDValue N1 = Addr.getOperand(1);
919  ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
920  unsigned DWordOffset0 = C1->getZExtValue() / 4;
921  unsigned DWordOffset1 = DWordOffset0 + 1;
922  // (add n0, c0)
923  if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
924  Base = N0;
925  Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
926  Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
927  return true;
928  }
929  }
930 
931  if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
932  unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
933  unsigned DWordOffset1 = DWordOffset0 + 1;
934  assert(4 * DWordOffset0 == CAddr->getZExtValue());
935 
936  if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
937  SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
938  MachineSDNode *MovZero
939  = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
940  DL, MVT::i32, Zero);
941  Base = SDValue(MovZero, 0);
942  Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
943  Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
944  return true;
945  }
946  }
947 
948  // default case
949  Base = Addr;
950  Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
951  Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
952  return true;
953 }
954 
955 static bool isLegalMUBUFImmOffset(const ConstantSDNode *Imm) {
956  return isUInt<12>(Imm->getZExtValue());
957 }
958 
959 void AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
960  SDValue &VAddr, SDValue &SOffset,
961  SDValue &Offset, SDValue &Offen,
962  SDValue &Idxen, SDValue &Addr64,
963  SDValue &GLC, SDValue &SLC,
964  SDValue &TFE) const {
965  SDLoc DL(Addr);
966 
967  GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
968  SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
969  TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
970 
971  Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
972  Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
973  Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
974  SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
975 
976  if (CurDAG->isBaseWithConstantOffset(Addr)) {
977  SDValue N0 = Addr.getOperand(0);
978  SDValue N1 = Addr.getOperand(1);
979  ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
980 
981  if (N0.getOpcode() == ISD::ADD) {
982  // (add (add N2, N3), C1) -> addr64
983  SDValue N2 = N0.getOperand(0);
984  SDValue N3 = N0.getOperand(1);
985  Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
986  Ptr = N2;
987  VAddr = N3;
988  } else {
989 
990  // (add N0, C1) -> offset
991  VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
992  Ptr = N0;
993  }
994 
995  if (isLegalMUBUFImmOffset(C1)) {
996  Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
997  return;
998  } else if (isUInt<32>(C1->getZExtValue())) {
999  // Illegal offset, store it in soffset.
1000  Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1001  SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
1002  CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1003  0);
1004  return;
1005  }
1006  }
1007 
1008  if (Addr.getOpcode() == ISD::ADD) {
1009  // (add N0, N1) -> addr64
1010  SDValue N0 = Addr.getOperand(0);
1011  SDValue N1 = Addr.getOperand(1);
1012  Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1013  Ptr = N0;
1014  VAddr = N1;
1015  Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1016  return;
1017  }
1018 
1019  // default case -> offset
1020  VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1021  Ptr = Addr;
1022  Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1023 
1024 }
1025 
1026 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1027  SDValue &VAddr, SDValue &SOffset,
1028  SDValue &Offset, SDValue &GLC,
1029  SDValue &SLC, SDValue &TFE) const {
1030  SDValue Ptr, Offen, Idxen, Addr64;
1031 
1032  // addr64 bit was removed for volcanic islands.
1033  if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
1034  return false;
1035 
1036  SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1037  GLC, SLC, TFE);
1038 
1039  ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1040  if (C->getSExtValue()) {
1041  SDLoc DL(Addr);
1042 
1043  const SITargetLowering& Lowering =
1044  *static_cast<const SITargetLowering*>(getTargetLowering());
1045 
1046  SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1047  return true;
1048  }
1049 
1050  return false;
1051 }
1052 
1053 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1054  SDValue &VAddr, SDValue &SOffset,
1055  SDValue &Offset,
1056  SDValue &SLC) const {
1057  SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
1058  SDValue GLC, TFE;
1059 
1060  return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
1061 }
1062 
1063 bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc,
1064  SDValue &VAddr, SDValue &SOffset,
1065  SDValue &ImmOffset) const {
1066 
1067  SDLoc DL(Addr);
1068  MachineFunction &MF = CurDAG->getMachineFunction();
1069  const SIRegisterInfo *TRI =
1070  static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
1071  MachineRegisterInfo &MRI = MF.getRegInfo();
1072  const SITargetLowering& Lowering =
1073  *static_cast<const SITargetLowering*>(getTargetLowering());
1074 
1075  unsigned ScratchOffsetReg =
1076  TRI->getPreloadedValue(MF, SIRegisterInfo::SCRATCH_WAVE_OFFSET);
1077  Lowering.CreateLiveInRegister(*CurDAG, &AMDGPU::SReg_32RegClass,
1078  ScratchOffsetReg, MVT::i32);
1079  SDValue Sym0 = CurDAG->getExternalSymbol("SCRATCH_RSRC_DWORD0", MVT::i32);
1080  SDValue ScratchRsrcDword0 =
1081  SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, Sym0), 0);
1082 
1083  SDValue Sym1 = CurDAG->getExternalSymbol("SCRATCH_RSRC_DWORD1", MVT::i32);
1084  SDValue ScratchRsrcDword1 =
1085  SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, Sym1), 0);
1086 
1087  const SDValue RsrcOps[] = {
1088  CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
1089  ScratchRsrcDword0,
1090  CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
1091  ScratchRsrcDword1,
1092  CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32),
1093  };
1094  SDValue ScratchPtr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1095  MVT::v2i32, RsrcOps), 0);
1096  Rsrc = SDValue(Lowering.buildScratchRSRC(*CurDAG, DL, ScratchPtr), 0);
1097  SOffset = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
1098  MRI.getLiveInVirtReg(ScratchOffsetReg), MVT::i32);
1099 
1100  // (add n0, c1)
1101  if (CurDAG->isBaseWithConstantOffset(Addr)) {
1102  SDValue N0 = Addr.getOperand(0);
1103  SDValue N1 = Addr.getOperand(1);
1104  // Offsets in vaddr must be positive.
1105  if (CurDAG->SignBitIsZero(N0)) {
1106  ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1107  if (isLegalMUBUFImmOffset(C1)) {
1108  VAddr = N0;
1109  ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1110  return true;
1111  }
1112  }
1113  }
1114 
1115  // (node)
1116  VAddr = Addr;
1117  ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1118  return true;
1119 }
1120 
1121 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1122  SDValue &SOffset, SDValue &Offset,
1123  SDValue &GLC, SDValue &SLC,
1124  SDValue &TFE) const {
1125  SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1126  const SIInstrInfo *TII =
1127  static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1128 
1129  SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1130  GLC, SLC, TFE);
1131 
1132  if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1133  !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1134  !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1135  uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1136  APInt::getAllOnesValue(32).getZExtValue(); // Size
1137  SDLoc DL(Addr);
1138 
1139  const SITargetLowering& Lowering =
1140  *static_cast<const SITargetLowering*>(getTargetLowering());
1141 
1142  SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1143  return true;
1144  }
1145  return false;
1146 }
1147 
1148 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1149  SDValue &Soffset, SDValue &Offset,
1150  SDValue &GLC) const {
1151  SDValue SLC, TFE;
1152 
1153  return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
1154 }
1155 
1156 // FIXME: This is incorrect and only enough to be able to compile.
1157 SDNode *AMDGPUDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
1158  AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(N);
1159  SDLoc DL(N);
1160 
1161  assert(Subtarget->hasFlatAddressSpace() &&
1162  "addrspacecast only supported with flat address space!");
1163 
1164  assert((ASC->getSrcAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS &&
1166  "Cannot cast address space to / from constant address!");
1167 
1168  assert((ASC->getSrcAddressSpace() == AMDGPUAS::FLAT_ADDRESS ||
1170  "Can only cast to / from flat address space!");
1171 
1172  // The flat instructions read the address as the index of the VGPR holding the
1173  // address, so casting should just be reinterpreting the base VGPR, so just
1174  // insert trunc / bitcast / zext.
1175 
1176  SDValue Src = ASC->getOperand(0);
1177  EVT DestVT = ASC->getValueType(0);
1178  EVT SrcVT = Src.getValueType();
1179 
1180  unsigned SrcSize = SrcVT.getSizeInBits();
1181  unsigned DestSize = DestVT.getSizeInBits();
1182 
1183  if (SrcSize > DestSize) {
1184  assert(SrcSize == 64 && DestSize == 32);
1185  return CurDAG->getMachineNode(
1187  DL,
1188  DestVT,
1189  Src,
1190  CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32));
1191  }
1192 
1193 
1194  if (DestSize > SrcSize) {
1195  assert(SrcSize == 32 && DestSize == 64);
1196 
1197  // FIXME: This is probably wrong, we should never be defining
1198  // a register class with both VGPRs and SGPRs
1199  SDValue RC = CurDAG->getTargetConstant(AMDGPU::VS_64RegClassID, DL,
1200  MVT::i32);
1201 
1202  const SDValue Ops[] = {
1203  RC,
1204  Src,
1205  CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
1206  SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
1207  CurDAG->getConstant(0, DL, MVT::i32)), 0),
1208  CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
1209  };
1210 
1211  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
1212  DL, N->getValueType(0), Ops);
1213  }
1214 
1215  assert(SrcSize == 64 && DestSize == 64);
1216  return CurDAG->getNode(ISD::BITCAST, DL, DestVT, Src).getNode();
1217 }
1218 
1219 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, SDLoc DL, SDValue Val,
1220  uint32_t Offset, uint32_t Width) {
1221  // Transformation function, pack the offset and width of a BFE into
1222  // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1223  // source, bits [5:0] contain the offset and bits [22:16] the width.
1224  uint32_t PackedVal = Offset | (Width << 16);
1225  SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
1226 
1227  return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
1228 }
1229 
1230 SDNode *AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
1231  // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
1232  // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
1233  // Predicate: 0 < b <= c < 32
1234 
1235  const SDValue &Shl = N->getOperand(0);
1238 
1239  if (B && C) {
1240  uint32_t BVal = B->getZExtValue();
1241  uint32_t CVal = C->getZExtValue();
1242 
1243  if (0 < BVal && BVal <= CVal && CVal < 32) {
1244  bool Signed = N->getOpcode() == ISD::SRA;
1245  unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1246 
1247  return getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0),
1248  CVal - BVal, 32 - CVal);
1249  }
1250  }
1251  return SelectCode(N);
1252 }
1253 
1254 SDNode *AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
1255  switch (N->getOpcode()) {
1256  case ISD::AND:
1257  if (N->getOperand(0).getOpcode() == ISD::SRL) {
1258  // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
1259  // Predicate: isMask(mask)
1260  const SDValue &Srl = N->getOperand(0);
1263 
1264  if (Shift && Mask) {
1265  uint32_t ShiftVal = Shift->getZExtValue();
1266  uint32_t MaskVal = Mask->getZExtValue();
1267 
1268  if (isMask_32(MaskVal)) {
1269  uint32_t WidthVal = countPopulation(MaskVal);
1270 
1271  return getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), Srl.getOperand(0),
1272  ShiftVal, WidthVal);
1273  }
1274  }
1275  }
1276  break;
1277  case ISD::SRL:
1278  if (N->getOperand(0).getOpcode() == ISD::AND) {
1279  // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
1280  // Predicate: isMask(mask >> b)
1281  const SDValue &And = N->getOperand(0);
1284 
1285  if (Shift && Mask) {
1286  uint32_t ShiftVal = Shift->getZExtValue();
1287  uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
1288 
1289  if (isMask_32(MaskVal)) {
1290  uint32_t WidthVal = countPopulation(MaskVal);
1291 
1292  return getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), And.getOperand(0),
1293  ShiftVal, WidthVal);
1294  }
1295  }
1296  } else if (N->getOperand(0).getOpcode() == ISD::SHL)
1297  return SelectS_BFEFromShifts(N);
1298  break;
1299  case ISD::SRA:
1300  if (N->getOperand(0).getOpcode() == ISD::SHL)
1301  return SelectS_BFEFromShifts(N);
1302  break;
1303  }
1304 
1305  return SelectCode(N);
1306 }
1307 
1308 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
1309  SDValue &SrcMods) const {
1310 
1311  unsigned Mods = 0;
1312 
1313  Src = In;
1314 
1315  if (Src.getOpcode() == ISD::FNEG) {
1316  Mods |= SISrcMods::NEG;
1317  Src = Src.getOperand(0);
1318  }
1319 
1320  if (Src.getOpcode() == ISD::FABS) {
1321  Mods |= SISrcMods::ABS;
1322  Src = Src.getOperand(0);
1323  }
1324 
1325  SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
1326 
1327  return true;
1328 }
1329 
1330 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src,
1331  SDValue &SrcMods) const {
1332  bool Res = SelectVOP3Mods(In, Src, SrcMods);
1333  return Res && cast<ConstantSDNode>(SrcMods)->isNullValue();
1334 }
1335 
1336 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
1337  SDValue &SrcMods, SDValue &Clamp,
1338  SDValue &Omod) const {
1339  SDLoc DL(In);
1340  // FIXME: Handle Clamp and Omod
1341  Clamp = CurDAG->getTargetConstant(0, DL, MVT::i32);
1342  Omod = CurDAG->getTargetConstant(0, DL, MVT::i32);
1343 
1344  return SelectVOP3Mods(In, Src, SrcMods);
1345 }
1346 
1347 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods0(SDValue In, SDValue &Src,
1348  SDValue &SrcMods, SDValue &Clamp,
1349  SDValue &Omod) const {
1350  bool Res = SelectVOP3Mods0(In, Src, SrcMods, Clamp, Omod);
1351 
1352  return Res && cast<ConstantSDNode>(SrcMods)->isNullValue() &&
1353  cast<ConstantSDNode>(Clamp)->isNullValue() &&
1354  cast<ConstantSDNode>(Omod)->isNullValue();
1355 }
1356 
1357 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp(SDValue In, SDValue &Src,
1358  SDValue &SrcMods,
1359  SDValue &Omod) const {
1360  // FIXME: Handle Omod
1361  Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
1362 
1363  return SelectVOP3Mods(In, Src, SrcMods);
1364 }
1365 
1366 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
1367  SDValue &SrcMods,
1368  SDValue &Clamp,
1369  SDValue &Omod) const {
1370  Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
1371  return SelectVOP3Mods(In, Src, SrcMods);
1372 }
1373 
1374 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
1375  const AMDGPUTargetLowering& Lowering =
1376  *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
1377  bool IsModified = false;
1378  do {
1379  IsModified = false;
1380  // Go over all selected nodes and try to fold them a bit more
1381  for (SDNode &Node : CurDAG->allnodes()) {
1382  MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
1383  if (!MachineNode)
1384  continue;
1385 
1386  SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
1387  if (ResNode != &Node) {
1388  ReplaceUses(&Node, ResNode);
1389  IsModified = true;
1390  }
1391  }
1392  CurDAG->RemoveDeadNodes();
1393  } while (IsModified);
1394 }
bool isUInt< 8 >(uint64_t x)
Definition: MathExtras.h:294
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:477
SDValue getValue(unsigned R) const
AMDGPU specific subclass of TargetSubtarget.
void legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const
Legalize target independent instructions (e.g.
Interface definition for R600InstrInfo.
unsigned getDestAddressSpace() const
SDVTList getVTList() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:191
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
unsigned getSrcAddressSpace() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:122
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
Address space for local memory.
Definition: AMDGPU.h:112
void setNodeId(int Id)
Set unique node id.
#define R2(n)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: Type.cpp:216
const SDValue & getBasePtr() const
virtual SDNode * PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:189
MachineSDNode * buildScratchRSRC(SelectionDAG &DAG, SDLoc DL, SDValue Ptr) const
static Type * checkType(Type *Ty)
Definition: Value.cpp:42
MachineMemOperand - A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const HexagonInstrInfo * TII
MachineSDNode * buildRSRC(SelectionDAG &DAG, SDLoc DL, SDValue Ptr, uint32_t RsrcDword1, uint64_t RsrcDword2And3) const
Return a resource descriptor with the 'Add TID' bit enabled The TID (Thread ID) is multipled by the s...
Shift and rotation operations.
Definition: ISDOpcodes.h:332
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:181
int32_t analyzeImmediate(const SDNode *N) const
Analyze the possible immediate value Op.
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:161
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:591
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
TargetRegisterInfo interface that is implemented by all hw codegen targets.
const SDValue & getBasePtr() const
EVT getMemoryVT() const
Return the type of the in-memory value.
PointerType - Class to represent pointers.
Definition: DerivedTypes.h:449
This class is used to represent ISD::STORE nodes.
Address space for region memory.
Definition: AMDGPU.h:114
This node is for VLIW targets and it is used to represent a vector that is stored in consecutive regi...
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:262
static bool isLegalMUBUFImmOffset(const ConstantSDNode *Imm)
SDNode * getNode() const
get the SDNode which holds the desired result
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
Definition: TargetOpcodes.h:52
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, unsigned Reg, EVT VT) const override
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
const SDValue & getOperand(unsigned i) const
Address space for constant memory.
Definition: AMDGPU.h:111
Address space for private memory.
Definition: AMDGPU.h:109
FunctionPass * createAMDGPUISelDag(TargetMachine &tm)
This pass converts a legalized DAG into a AMDGPU-specific.
unsigned getSubRegFromChannel(unsigned Channel) const
unsigned getLiveInVirtReg(unsigned PReg) const
getLiveInVirtReg - If PReg is a live-in physical register, return the corresponding live-in physical ...
MachineSDNode * wrapAddr64Rsrc(SelectionDAG &DAG, SDLoc DL, SDValue Ptr) const
unsigned getPreloadedValue(const MachineFunction &MF, enum PreloadedValue Value) const
Returns the physical register that Value is stored in.
SI DAG Lowering interface definition.
This class provides iterator support for SDUse operands that use a specific SDNode.
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
uint64_t getDefaultRsrcDataFormat() const
const PseudoSourceValue * getPseudoValue() const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Address space for flat memory.
Definition: AMDGPU.h:113
const SDValue & getValue() const
EVT - Extended Value Type.
Definition: ValueTypes.h:31
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:171
static bool FoldOperand(SDNode *ParentNode, unsigned SrcIdx, SDValue &Src, SDValue &Neg, SDValue &Abs, SDValue &Sel, SDValue &Imm, SelectionDAG &DAG)
REG_SEQUENCE - This variadic instruction is used to form a register that represents a consecutive seq...
Definition: TargetOpcodes.h:82
EXTRACT_SUBREG - This instruction takes two operands: a register that has subregisters, and a subregister index.
Definition: TargetOpcodes.h:41
unsigned countPopulation(T Value)
Count the number of set bits in a value.
Definition: MathExtras.h:449
shadow stack gc Shadow Stack GC Lowering
Interface definition of the TargetLowering class that is common to all AMD GPUs.
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:481
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
const Value * getValue() const
getValue - Return the base address of the memory access.
bool isUInt< 32 >(uint64_t x)
Definition: MathExtras.h:302
PseudoSourceValue - Special value supplied for machine level alias analysis.
int64_t getSExtValue() const
bool isMask_32(uint32_t Value)
isMask_32 - This function returns true if the argument is a non-empty sequence of ones starting at th...
Definition: MathExtras.h:328
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1890
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:62
Address space for indirect addressible parameter memory (VTX1)
Definition: AMDGPU.h:116
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
COPY_TO_REGCLASS - This instruction is a placeholder for a plain register-to-register copy into a spe...
Definition: TargetOpcodes.h:66
#define N
bool isInt< 16 >(int64_t x)
Definition: MathExtras.h:272
Address space for direct addressible parameter memory (CONST0)
Definition: AMDGPU.h:115
EVT getValueType() const
Return the ValueType of the referenced return value.
LLVM Value Representation.
Definition: Value.h:69
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, SDLoc DL, SDValue V) const
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:185
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:149
bool isUInt< 16 >(uint64_t x)
Definition: MathExtras.h:298
Primary interface to the complete machine description for the target machine.
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:506
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:309
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
Address space for global memory (RAT0, VTX0).
Definition: AMDGPU.h:110
uint64_t getZExtValue() const
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
This class is used to represent ISD::LOAD nodes.