LLVM  16.0.0git
VECustomDAG.cpp
Go to the documentation of this file.
1 //===-- VECustomDAG.h - VE Custom DAG Nodes ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that VE uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 
16 #ifndef DEBUG_TYPE
17 #define DEBUG_TYPE "vecustomdag"
18 #endif
19 
20 namespace llvm {
21 
22 bool isPackedVectorType(EVT SomeVT) {
23  if (!SomeVT.isVector())
24  return false;
25  return SomeVT.getVectorNumElements() > StandardVectorWidth;
26 }
27 
29  if (!VT.isVector())
30  return VT;
32 }
33 
37 }
38 
40  assert(VT.isVector());
42 }
43 
44 bool isMaskType(EVT SomeVT) {
45  if (!SomeVT.isVector())
46  return false;
47  return SomeVT.getVectorElementType() == MVT::i1;
48 }
49 
51  switch (Op.getOpcode()) {
52  default:
53  return false;
54  case ISD::AND:
55  case ISD::XOR:
56  case ISD::OR:
57  return isMaskType(Op.getValueType());
58  }
59 }
60 
61 /// \returns the VVP_* SDNode opcode corresponsing to \p OC.
62 Optional<unsigned> getVVPOpcode(unsigned Opcode) {
63  switch (Opcode) {
64  case ISD::MLOAD:
65  return VEISD::VVP_LOAD;
66  case ISD::MSTORE:
67  return VEISD::VVP_STORE;
68 #define HANDLE_VP_TO_VVP(VPOPC, VVPNAME) \
69  case ISD::VPOPC: \
70  return VEISD::VVPNAME;
71 #define ADD_VVP_OP(VVPNAME, SDNAME) \
72  case VEISD::VVPNAME: \
73  case ISD::SDNAME: \
74  return VEISD::VVPNAME;
75 #include "VVPNodes.def"
76  // TODO: Map those in VVPNodes.def too
77  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
78  return VEISD::VVP_LOAD;
79  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
80  return VEISD::VVP_STORE;
81  }
82  return None;
83 }
84 
86  auto VVPOpc = getVVPOpcode(Op->getOpcode());
87  auto Opc = VVPOpc.value_or(Op->getOpcode());
88 
89  switch (Opc) {
90  case VEISD::VVP_SDIV:
91  case VEISD::VVP_UDIV:
92  case VEISD::VVP_FDIV:
93  case VEISD::VVP_SELECT:
94  return false;
95 
96  default:
97  return true;
98  }
99 }
100 
101 bool supportsPackedMode(unsigned Opcode, EVT IdiomVT) {
102  bool IsPackedOp = isPackedVectorType(IdiomVT);
103  bool IsMaskOp = isMaskType(IdiomVT);
104  switch (Opcode) {
105  default:
106  return false;
107 
109  return true;
110 #define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:
111 #include "VVPNodes.def"
112  return IsPackedOp && !IsMaskOp;
113  }
114 }
115 
116 bool isPackingSupportOpcode(unsigned Opc) {
117  switch (Opc) {
118  case VEISD::VEC_PACK:
121  return true;
122  }
123  return false;
124 }
125 
126 bool isVVPOrVEC(unsigned Opcode) {
127  switch (Opcode) {
129 #define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:
130 #include "VVPNodes.def"
131  return true;
132  }
133  return false;
134 }
135 
136 bool isVVPUnaryOp(unsigned VVPOpcode) {
137  switch (VVPOpcode) {
138 #define ADD_UNARY_VVP_OP(VVPNAME, ...) \
139  case VEISD::VVPNAME: \
140  return true;
141 #include "VVPNodes.def"
142  }
143  return false;
144 }
145 
146 bool isVVPBinaryOp(unsigned VVPOpcode) {
147  switch (VVPOpcode) {
148 #define ADD_BINARY_VVP_OP(VVPNAME, ...) \
149  case VEISD::VVPNAME: \
150  return true;
151 #include "VVPNodes.def"
152  }
153  return false;
154 }
155 
156 bool isVVPReductionOp(unsigned Opcode) {
157  switch (Opcode) {
158 #define ADD_REDUCE_VVP_OP(VVP_NAME, SDNAME) case VEISD::VVP_NAME:
159 #include "VVPNodes.def"
160  return true;
161  }
162  return false;
163 }
164 
165 // Return the AVL operand position for this VVP or VEC Op.
166 Optional<int> getAVLPos(unsigned Opc) {
167  // This is only available for VP SDNodes
168  auto PosOpt = ISD::getVPExplicitVectorLengthIdx(Opc);
169  if (PosOpt)
170  return *PosOpt;
171 
172  // VVP Opcodes.
173  if (isVVPBinaryOp(Opc))
174  return 3;
175 
176  // VM Opcodes.
177  switch (Opc) {
179  return 1;
180  case VEISD::VVP_SELECT:
181  return 3;
182  case VEISD::VVP_LOAD:
183  return 4;
184  case VEISD::VVP_STORE:
185  return 5;
186  }
187 
188  return None;
189 }
190 
191 Optional<int> getMaskPos(unsigned Opc) {
192  // This is only available for VP SDNodes
193  auto PosOpt = ISD::getVPMaskIdx(Opc);
194  if (PosOpt)
195  return *PosOpt;
196 
197  // VVP Opcodes.
198  if (isVVPBinaryOp(Opc))
199  return 2;
200 
201  // Other opcodes.
202  switch (Opc) {
203  case ISD::MSTORE:
204  return 4;
205  case ISD::MLOAD:
206  return 3;
207  case VEISD::VVP_SELECT:
208  return 2;
209  }
210 
211  return None;
212 }
213 
214 bool isLegalAVL(SDValue AVL) { return AVL->getOpcode() == VEISD::LEGALAVL; }
215 
216 /// Node Properties {
217 
219  if (MemSDNode *MemN = dyn_cast<MemSDNode>(Op.getNode()))
220  return MemN->getChain();
221 
222  switch (Op->getOpcode()) {
223  case VEISD::VVP_LOAD:
224  case VEISD::VVP_STORE:
225  return Op->getOperand(0);
226  }
227  return SDValue();
228 }
229 
231  if (auto *MemN = dyn_cast<MemSDNode>(Op.getNode()))
232  return MemN->getBasePtr();
233 
234  switch (Op->getOpcode()) {
235  case VEISD::VVP_LOAD:
236  return Op->getOperand(1);
237  case VEISD::VVP_STORE:
238  return Op->getOperand(2);
239  }
240  return SDValue();
241 }
242 
244  unsigned OC = Op->getOpcode();
245 
246  // For memory ops -> the transfered data type
247  if (auto MemN = dyn_cast<MemSDNode>(Op))
248  return MemN->getMemoryVT();
249 
250  switch (OC) {
251  // Standard ISD.
252  case ISD::SELECT: // not aliased with VVP_SELECT
253  case ISD::CONCAT_VECTORS:
255  case ISD::VECTOR_SHUFFLE:
256  case ISD::BUILD_VECTOR:
258  return Op->getValueType(0);
259  }
260 
261  // Translate to VVP where possible.
262  unsigned OriginalOC = OC;
263  if (auto VVPOpc = getVVPOpcode(OC))
264  OC = *VVPOpc;
265 
266  if (isVVPReductionOp(OC))
267  return Op->getOperand(hasReductionStartParam(OriginalOC) ? 1 : 0)
268  .getValueType();
269 
270  switch (OC) {
271  default:
272  case VEISD::VVP_SETCC:
273  return Op->getOperand(0).getValueType();
274 
275  case VEISD::VVP_SELECT:
276 #define ADD_BINARY_VVP_OP(VVP_NAME, ...) case VEISD::VVP_NAME:
277 #include "VVPNodes.def"
278  return Op->getValueType(0);
279 
280  case VEISD::VVP_LOAD:
281  return Op->getValueType(0);
282 
283  case VEISD::VVP_STORE:
284  return Op->getOperand(1)->getValueType(0);
285 
286  // VEC
288  return Op->getValueType(0);
289  }
290 }
291 
293  switch (Op->getOpcode()) {
294  case VEISD::VVP_STORE:
295  return Op->getOperand(3);
296  case VEISD::VVP_LOAD:
297  return Op->getOperand(2);
298  }
299 
300  if (auto *StoreN = dyn_cast<VPStridedStoreSDNode>(Op.getNode()))
301  return StoreN->getStride();
302  if (auto *StoreN = dyn_cast<VPStridedLoadSDNode>(Op.getNode()))
303  return StoreN->getStride();
304 
305  if (isa<MemSDNode>(Op.getNode())) {
306  // Regular MLOAD/MSTORE/LOAD/STORE
307  // No stride argument -> use the contiguous element size as stride.
308  uint64_t ElemStride = getIdiomaticVectorType(Op.getNode())
309  ->getVectorElementType()
310  .getStoreSize();
311  return CDAG.getConstant(ElemStride, MVT::i64);
312  }
313  return SDValue();
314 }
315 
317  if (auto *N = dyn_cast<MaskedGatherScatterSDNode>(Op.getNode()))
318  return N->getIndex();
319  if (auto *N = dyn_cast<VPGatherScatterSDNode>(Op.getNode()))
320  return N->getIndex();
321  return SDValue();
322 }
323 
325  if (auto *N = dyn_cast<MaskedGatherScatterSDNode>(Op.getNode()))
326  return N->getScale();
327  if (auto *N = dyn_cast<VPGatherScatterSDNode>(Op.getNode()))
328  return N->getScale();
329  return SDValue();
330 }
331 
333  switch (Op->getOpcode()) {
334  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
335  case VEISD::VVP_STORE:
336  return Op->getOperand(1);
337  }
338  if (auto *StoreN = dyn_cast<StoreSDNode>(Op.getNode()))
339  return StoreN->getValue();
340  if (auto *StoreN = dyn_cast<MaskedStoreSDNode>(Op.getNode()))
341  return StoreN->getValue();
342  if (auto *StoreN = dyn_cast<VPStridedStoreSDNode>(Op.getNode()))
343  return StoreN->getValue();
344  if (auto *StoreN = dyn_cast<VPStoreSDNode>(Op.getNode()))
345  return StoreN->getValue();
346  if (auto *StoreN = dyn_cast<MaskedScatterSDNode>(Op.getNode()))
347  return StoreN->getValue();
348  if (auto *StoreN = dyn_cast<VPScatterSDNode>(Op.getNode()))
349  return StoreN->getValue();
350  return SDValue();
351 }
352 
354  if (auto *N = dyn_cast<MaskedLoadSDNode>(Op.getNode()))
355  return N->getPassThru();
356  if (auto *N = dyn_cast<MaskedGatherSDNode>(Op.getNode()))
357  return N->getPassThru();
358 
359  return SDValue();
360 }
361 
362 bool hasReductionStartParam(unsigned OPC) {
363  // TODO: Ordered reduction opcodes.
364  if (ISD::isVPReduction(OPC))
365  return true;
366  return false;
367 }
368 
369 unsigned getScalarReductionOpcode(unsigned VVPOC, bool IsMask) {
370  assert(!IsMask && "Mask reduction isel");
371 
372  switch (VVPOC) {
373 #define HANDLE_VVP_REDUCE_TO_SCALAR(VVP_RED_ISD, REDUCE_ISD) \
374  case VEISD::VVP_RED_ISD: \
375  return ISD::REDUCE_ISD;
376 #include "VVPNodes.def"
377  default:
378  break;
379  }
380  llvm_unreachable("Cannot not scalarize this reduction Opcode!");
381 }
382 
383 /// } Node Properties
384 
386  auto PosOpt = getAVLPos(Op->getOpcode());
387  return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
388 }
389 
391  auto PosOpt = getMaskPos(Op->getOpcode());
392  return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
393 }
394 
395 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue Op) {
396  SDValue AVL = getNodeAVL(Op);
397  if (!AVL)
398  return {SDValue(), true};
399  if (isLegalAVL(AVL))
400  return {AVL->getOperand(0), true};
401  return {AVL, false};
402 }
403 
405  bool IsOpaque) const {
406  return DAG.getConstant(Val, DL, VT, IsTarget, IsOpaque);
407 }
408 
410  auto MaskVT = getLegalVectorType(Packing, MVT::i1);
411 
412  // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
413  auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
414  auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
415  auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
416  if (AllTrue)
417  return Res;
418 
419  return DAG.getNOT(DL, Res, Res.getValueType());
420 }
421 
423  SDValue AVL) const {
424  // Constant mask splat.
425  if (auto BcConst = dyn_cast<ConstantSDNode>(Scalar))
426  return getConstantMask(getTypePacking(ResultVT),
427  BcConst->getSExtValue() != 0);
428 
429  // Expand the broadcast to a vector comparison.
430  auto ScalarBoolVT = Scalar.getSimpleValueType();
431  assert(ScalarBoolVT == MVT::i32);
432 
433  // Cast to i32 ty.
434  SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);
435  unsigned ElemCount = ResultVT.getVectorNumElements();
436  MVT CmpVecTy = MVT::getVectorVT(ScalarBoolVT, ElemCount);
437 
438  // Broadcast to vector.
439  SDValue BCVec =
440  DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});
441  SDValue ZeroVec =
442  getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);
443 
444  MVT BoolVecTy = MVT::getVectorVT(MVT::i1, ElemCount);
445 
446  // Broadcast(Data) != Broadcast(0)
447  // TODO: Use a VVP operation for this.
448  return DAG.getSetCC(DL, BoolVecTy, BCVec, ZeroVec, ISD::CondCode::SETNE);
449 }
450 
452  SDValue AVL) const {
453  assert(ResultVT.isVector());
454  auto ScaVT = Scalar.getValueType();
455 
456  if (isMaskType(ResultVT))
457  return getMaskBroadcast(ResultVT, Scalar, AVL);
458 
459  if (isPackedVectorType(ResultVT)) {
460  // v512x packed mode broadcast
461  // Replicate the scalar reg (f32 or i32) onto the opposing half of the full
462  // scalar register. If it's an I64 type, assume that this has already
463  // happened.
464  if (ScaVT == MVT::f32) {
465  Scalar = getNode(VEISD::REPL_F32, MVT::i64, Scalar);
466  } else if (ScaVT == MVT::i32) {
467  Scalar = getNode(VEISD::REPL_I32, MVT::i64, Scalar);
468  }
469  }
470 
471  return getNode(VEISD::VEC_BROADCAST, ResultVT, {Scalar, AVL});
472 }
473 
475  if (isLegalAVL(AVL))
476  return AVL;
477  return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
478 }
479 
481  SDValue AVL) const {
482  assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
483 
484  // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
485  unsigned OC =
487  return DAG.getNode(OC, DL, DestVT, Vec, AVL);
488 }
489 
491  SDValue AVL) const {
492  assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
493 
494  // TODO: Peek through VEC_UNPACK_LO|HI operands.
495  return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
496 }
497 
499  PackElem Part) const {
500  // Adjust AVL for this part
501  SDValue NewAVL;
502  SDValue OneV = getConstant(1, MVT::i32);
503  if (Part == PackElem::Hi)
504  NewAVL = getNode(ISD::ADD, MVT::i32, {RawAVL, OneV});
505  else
506  NewAVL = RawAVL;
507  NewAVL = getNode(ISD::SRL, MVT::i32, {NewAVL, OneV});
508 
509  NewAVL = annotateLegalAVL(NewAVL);
510 
511  // Legalize Mask (unpack or all-true)
512  SDValue NewMask;
513  if (!RawMask)
514  NewMask = getConstantMask(Packing::Normal, true);
515  else
516  NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
517 
518  return VETargetMasks(NewMask, NewAVL);
519 }
520 
522  PackElem Part) const {
523  // High starts at base ptr but has more significant bits in the 64bit vector
524  // element.
525  if (Part == PackElem::Hi)
526  return Ptr;
527  return getNode(ISD::ADD, MVT::i64, {Ptr, ByteStride});
528 }
529 
531  if (auto ConstBytes = dyn_cast<ConstantSDNode>(PackStride))
532  return getConstant(2 * ConstBytes->getSExtValue(), MVT::i64);
533  return getNode(ISD::SHL, MVT::i64, {PackStride, getConstant(1, MVT::i32)});
534 }
535 
538  SDValue AVL) const {
539  EVT IndexVT = Index.getValueType();
540 
541  // Apply scale.
542  SDValue ScaledIndex;
543  if (!Scale || isOneConstant(Scale))
544  ScaledIndex = Index;
545  else {
546  SDValue ScaleBroadcast = getBroadcast(IndexVT, Scale, AVL);
547  ScaledIndex =
548  getNode(VEISD::VVP_MUL, IndexVT, {Index, ScaleBroadcast, Mask, AVL});
549  }
550 
551  // Add basePtr.
552  if (isNullConstant(BasePtr))
553  return ScaledIndex;
554 
555  // re-constitute pointer vector (basePtr + index * scale)
556  SDValue BaseBroadcast = getBroadcast(IndexVT, BasePtr, AVL);
557  auto ResPtr =
558  getNode(VEISD::VVP_ADD, IndexVT, {BaseBroadcast, ScaledIndex, Mask, AVL});
559  return ResPtr;
560 }
561 
563  SDValue StartV, SDValue VectorV,
564  SDValue Mask, SDValue AVL,
565  SDNodeFlags Flags) const {
566 
567  // Optionally attach the start param with a scalar op (where it is
568  // unsupported).
569  bool scalarizeStartParam = StartV && !hasReductionStartParam(VVPOpcode);
570  bool IsMaskReduction = isMaskType(VectorV.getValueType());
571  assert(!IsMaskReduction && "TODO Implement");
572  auto AttachStartValue = [&](SDValue ReductionResV) {
573  if (!scalarizeStartParam)
574  return ReductionResV;
575  auto ScalarOC = getScalarReductionOpcode(VVPOpcode, IsMaskReduction);
576  return getNode(ScalarOC, ResVT, {StartV, ReductionResV});
577  };
578 
579  // Fixup: Always Use sequential 'fmul' reduction.
580  if (!scalarizeStartParam && StartV) {
581  assert(hasReductionStartParam(VVPOpcode));
582  return AttachStartValue(
583  getNode(VVPOpcode, ResVT, {StartV, VectorV, Mask, AVL}, Flags));
584  } else
585  return AttachStartValue(
586  getNode(VVPOpcode, ResVT, {VectorV, Mask, AVL}, Flags));
587 }
588 
589 } // namespace llvm
VECustomDAG.h
llvm::ISD::VECTOR_SHUFFLE
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:586
llvm::MVT::getVectorElementType
MVT getVectorElementType() const
Definition: MachineValueType.h:542
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Packing::Normal
@ Normal
llvm::getNodeChain
SDValue getNodeChain(SDValue Op)
Node Properties {.
Definition: VECustomDAG.cpp:218
llvm::VEISD::REPL_F32
@ REPL_F32
Definition: VEISelLowering.h:56
llvm::PackElem
PackElem
Definition: VECustomDAG.h:130
llvm::VEISD::REPL_I32
@ REPL_I32
Definition: VEISelLowering.h:55
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::ISD::SETNE
@ SETNE
Definition: ISDOpcodes.h:1449
llvm::isOneConstant
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Definition: SelectionDAG.cpp:10802
llvm::ISD::CONCAT_VECTORS
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:542
llvm::getNodeMask
SDValue getNodeMask(SDValue Op)
Definition: VECustomDAG.cpp:390
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::isPackedVectorType
bool isPackedVectorType(EVT SomeVT)
Definition: VECustomDAG.cpp:22
llvm::ISD::MLOAD
@ MLOAD
Definition: ISDOpcodes.h:1207
llvm::ISD::getVPExplicitVectorLengthIdx
Optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
Definition: SelectionDAG.cpp:508
llvm::MVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: MachineValueType.h:386
llvm::VECustomDAG::getMaskBroadcast
SDValue getMaskBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const
Definition: VECustomDAG.cpp:422
llvm::isMaskArithmetic
bool isMaskArithmetic(SDValue Op)
Definition: VECustomDAG.cpp:50
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::splitVectorType
MVT splitVectorType(MVT VT)
Definition: VECustomDAG.cpp:28
llvm::Optional< unsigned >
llvm::MemSDNode
This is an abstract virtual class for memory operations.
Definition: SelectionDAGNodes.h:1275
llvm::Packing::Dense
@ Dense
llvm::getTypePacking
Packing getTypePacking(EVT VT)
Definition: VECustomDAG.cpp:39
llvm::VECustomDAG::getPack
SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const
Definition: VECustomDAG.cpp:490
llvm::getMaskPos
Optional< int > getMaskPos(unsigned Opc)
Definition: VECustomDAG.cpp:191
llvm::VEISD::LEGALAVL
@ LEGALAVL
Definition: VEISelLowering.h:61
llvm::VEISD::VEC_UNPACK_LO
@ VEC_UNPACK_LO
Definition: VEISelLowering.h:47
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:644
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:713
llvm::ISD::isVPReduction
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
Definition: SelectionDAG.cpp:483
llvm::getStoredValue
SDValue getStoredValue(SDValue Op)
Definition: VECustomDAG.cpp:332
llvm::VEISD::VEC_BROADCAST
@ VEC_BROADCAST
Definition: VEISelLowering.h:53
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1514
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
llvm::VEISD::VEC_UNPACK_HI
@ VEC_UNPACK_HI
Definition: VEISelLowering.h:48
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::VECustomDAG::getGatherScatterAddress
SDValue getGatherScatterAddress(SDValue BasePtr, SDValue Scale, SDValue Index, SDValue Mask, SDValue AVL) const
Definition: VECustomDAG.cpp:536
llvm::VECustomDAG::getBroadcast
SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const
Definition: VECustomDAG.cpp:451
llvm::StandardVectorWidth
static const unsigned StandardVectorWidth
Definition: VE.h:468
llvm::getNodeAVL
SDValue getNodeAVL(SDValue Op)
} Node Properties
Definition: VECustomDAG.cpp:385
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::PackElem::Hi
@ Hi
llvm::getAnnotatedNodeAVL
std::pair< SDValue, bool > getAnnotatedNodeAVL(SDValue Op)
Definition: VECustomDAG.cpp:395
llvm::isPackingSupportOpcode
bool isPackingSupportOpcode(unsigned Opc)
Definition: VECustomDAG.cpp:116
llvm::VECustomDAG::getConstantMask
SDValue getConstantMask(Packing Packing, bool AllTrue) const
Definition: VECustomDAG.cpp:409
Index
uint32_t Index
Definition: ELFObjHandler.cpp:83
uint64_t
llvm::supportsPackedMode
bool supportsPackedMode(unsigned Opcode, EVT IdiomVT)
Definition: VECustomDAG.cpp:101
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:921
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9072
llvm::SelectionDAG::getNOT
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
Definition: SelectionDAG.cpp:1470
llvm::SelectionDAG::getSExtOrTrunc
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
Definition: SelectionDAG.cpp:1414
llvm::PackedVectorWidth
static const unsigned PackedVectorWidth
Definition: VE.h:469
llvm::PackElem::Lo
@ Lo
llvm::ISD::MSTORE
@ MSTORE
Definition: ISDOpcodes.h:1208
llvm::VECustomDAG::annotateLegalAVL
SDValue annotateLegalAVL(SDValue AVL) const
Definition: VECustomDAG.cpp:474
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::VECustomDAG::getUnpack
SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const
} Legalizing getNode
Definition: VECustomDAG.cpp:480
llvm::VECustomDAG::getLegalReductionOpVVP
SDValue getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT, SDValue StartV, SDValue VectorV, SDValue Mask, SDValue AVL, SDNodeFlags Flags) const
} getNode
Definition: VECustomDAG.cpp:562
llvm::SystemZISD::OC
@ OC
Definition: SystemZISelLowering.h:122
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MVT::v256i1
@ v256i1
Definition: MachineValueType.h:74
llvm::VECustomDAG::getSplitPtrOffset
SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride, PackElem Part) const
Definition: VECustomDAG.cpp:521
llvm::MVT::getVectorVT
static MVT getVectorVT(MVT VT, unsigned NumElements)
Definition: MachineValueType.h:1263
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:154
llvm::ISD::getVPMaskIdx
Optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
Definition: SelectionDAG.cpp:496
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::getIdiomaticVectorType
Optional< EVT > getIdiomaticVectorType(SDNode *Op)
} AVL Functions
Definition: VECustomDAG.cpp:243
llvm::VECustomDAG
Definition: VECustomDAG.h:142
llvm::isLegalAVL
bool isLegalAVL(SDValue AVL)
Definition: VECustomDAG.cpp:214
llvm::isVVPOrVEC
bool isVVPOrVEC(unsigned Opcode)
Definition: VECustomDAG.cpp:126
llvm::maySafelyIgnoreMask
bool maySafelyIgnoreMask(SDValue Op)
Definition: VECustomDAG.cpp:85
llvm::getGatherScatterIndex
SDValue getGatherScatterIndex(SDValue Op)
Definition: VECustomDAG.cpp:316
llvm::isVVPBinaryOp
bool isVVPBinaryOp(unsigned VVPOpcode)
Definition: VECustomDAG.cpp:146
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:668
llvm::VEISD::VEC_PACK
@ VEC_PACK
Definition: VEISelLowering.h:50
llvm::getScalarReductionOpcode
unsigned getScalarReductionOpcode(unsigned VVPOC, bool IsMask)
Definition: VECustomDAG.cpp:369
llvm::ISD::EXTRACT_SUBVECTOR
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:572
llvm::None
constexpr std::nullopt_t None
Definition: None.h:27
llvm::isNullConstant
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
Definition: SelectionDAG.cpp:10787
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::isVVPUnaryOp
bool isVVPUnaryOp(unsigned VVPOpcode)
Definition: VECustomDAG.cpp:136
llvm::getNodePassthru
SDValue getNodePassthru(SDValue Op)
Definition: VECustomDAG.cpp:353
llvm::VECustomDAG::getConstant
SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget=false, bool IsOpaque=false) const
Definition: VECustomDAG.cpp:404
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:48
llvm::Packing
Packing
} Node Properties
Definition: VECustomDAG.h:119
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::getVVPOpcode
Optional< unsigned > getVVPOpcode(unsigned Opcode)
Definition: VECustomDAG.cpp:62
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::VETargetMasks
Definition: VECustomDAG.h:135
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::isVVPReductionOp
bool isVVPReductionOp(unsigned Opcode)
Definition: VECustomDAG.cpp:156
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
llvm::VECustomDAG::getSplitPtrStride
SDValue getSplitPtrStride(SDValue PackStride) const
Definition: VECustomDAG.cpp:530
llvm::VECustomDAG::getNode
SDValue getNode(unsigned OC, SDVTList VTL, ArrayRef< SDValue > OpV, Optional< SDNodeFlags > Flags=None) const
getNode {
Definition: VECustomDAG.h:156
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:691
llvm::getGatherScatterScale
SDValue getGatherScatterScale(SDValue Op)
Definition: VECustomDAG.cpp:324
N
#define N
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:693
llvm::getAVLPos
Optional< int > getAVLPos(unsigned Opc)
The VE backend uses a two-staged process to lower and legalize vector instructions:
Definition: VECustomDAG.cpp:166
llvm::getLegalVectorType
MVT getLegalVectorType(Packing P, MVT ElemVT)
Definition: VECustomDAG.cpp:34
llvm::getLoadStoreStride
SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG)
Definition: VECustomDAG.cpp:292
llvm::tgtok::TrueVal
@ TrueVal
Definition: TGLexer.h:62
llvm::isMaskType
bool isMaskType(EVT SomeVT)
Definition: VECustomDAG.cpp:44
llvm::MVT::f32
@ f32
Definition: MachineValueType.h:57
llvm::hasReductionStartParam
bool hasReductionStartParam(unsigned OPC)
Definition: VECustomDAG.cpp:362
llvm::ISD::SCALAR_TO_VECTOR
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:606
llvm::getMemoryPtr
SDValue getMemoryPtr(SDValue Op)
Definition: VECustomDAG.cpp:230
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1133
llvm::VECustomDAG::getTargetSplitMask
VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL, PackElem Part) const
Definition: VECustomDAG.cpp:498