LLVM  11.0.0git
HexagonISelLoweringHVX.cpp
Go to the documentation of this file.
1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "HexagonISelLowering.h"
10 #include "HexagonRegisterInfo.h"
11 #include "HexagonSubtarget.h"
12 #include "llvm/IR/IntrinsicsHexagon.h"
14 
15 using namespace llvm;
16 
21 
22 
23 void
24 HexagonTargetLowering::initializeHVXLowering() {
25  if (Subtarget.useHVX64BOps()) {
26  addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass);
27  addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
28  addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
29  addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
30  addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
31  addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
32  // These "short" boolean vector types should be legal because
33  // they will appear as results of vector compares. If they were
34  // not legal, type legalization would try to make them legal
35  // and that would require using operations that do not use or
36  // produce such types. That, in turn, would imply using custom
37  // nodes, which would be unoptimizable by the DAG combiner.
38  // The idea is to rely on target-independent operations as much
39  // as possible.
40  addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
41  addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
42  addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
43  } else if (Subtarget.useHVX128BOps()) {
44  addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass);
45  addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass);
46  addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass);
47  addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass);
48  addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
49  addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass);
50  addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
51  addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
52  addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
53  }
54 
55  // Set up operation actions.
56 
57  bool Use64b = Subtarget.useHVX64BOps();
58  ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
59  ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
60  MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
61  MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
62 
63  auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
64  setOperationAction(Opc, FromTy, Promote);
65  AddPromotedToType(Opc, FromTy, ToTy);
66  };
67 
68  // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
69  // Note: v16i1 -> i16 is handled in type legalization instead of op
70  // legalization.
80 
81  for (MVT T : LegalV) {
84 
92  if (T != ByteV) {
96  }
97 
104  // Make concat-vectors custom to handle concats of more than 2 vectors.
113  if (T != ByteV) {
115  // HVX only has shifts of words and halfwords.
119 
120  // Promote all shuffles to operate on vectors of bytes.
121  setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
122  }
123 
131  }
132 
133  for (MVT T : LegalW) {
134  // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
135  // independent) handling of it would convert it to a load, which is
136  // not always the optimal choice.
138  // Make concat-vectors custom to handle concats of more than 2 vectors.
140 
141  // Custom-lower these operations for pairs. Expand them into a concat
142  // of the corresponding operations on individual vectors.
150 
156 
167  if (T != ByteW) {
171 
172  // Promote all shuffles to operate on vectors of bytes.
173  setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
174  }
175  }
176 
177  // Boolean vectors.
178 
179  for (MVT T : LegalW) {
180  // Boolean types for vector pairs will overlap with the boolean
181  // types for single vectors, e.g.
182  // v64i8 -> v64i1 (single)
183  // v64i16 -> v64i1 (pair)
184  // Set these actions first, and allow the single actions to overwrite
185  // any duplicates.
186  MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
191  }
192 
193  for (MVT T : LegalV) {
194  MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
204  }
205 
206  if (Use64b) {
209  } else {
212  }
213 
215 }
216 
217 SDValue
218 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
219  const SDLoc &dl, SelectionDAG &DAG) const {
220  SmallVector<SDValue,4> IntOps;
221  IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
222  for (const SDValue &Op : Ops)
223  IntOps.push_back(Op);
224  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
225 }
226 
227 MVT
228 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
229  assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
230 
231  MVT ElemTy = Tys.first.getVectorElementType();
232  return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
233  Tys.second.getVectorNumElements());
234 }
235 
236 HexagonTargetLowering::TypePair
237 HexagonTargetLowering::typeSplit(MVT VecTy) const {
238  assert(VecTy.isVector());
239  unsigned NumElem = VecTy.getVectorNumElements();
240  assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
241  MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
242  return { HalfTy, HalfTy };
243 }
244 
245 MVT
246 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
247  MVT ElemTy = VecTy.getVectorElementType();
248  MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
249  return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
250 }
251 
252 MVT
253 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
254  MVT ElemTy = VecTy.getVectorElementType();
255  MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
256  return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
257 }
258 
259 SDValue
260 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
261  SelectionDAG &DAG) const {
262  if (ty(Vec).getVectorElementType() == ElemTy)
263  return Vec;
264  MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
265  return DAG.getBitcast(CastTy, Vec);
266 }
267 
268 SDValue
269 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
270  SelectionDAG &DAG) const {
271  return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
272  Ops.second, Ops.first);
273 }
274 
275 HexagonTargetLowering::VectorPair
276 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
277  SelectionDAG &DAG) const {
278  TypePair Tys = typeSplit(ty(Vec));
279  if (Vec.getOpcode() == HexagonISD::QCAT)
280  return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
281  return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
282 }
283 
284 bool
285 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
286  return Subtarget.isHVXVectorType(Ty) &&
287  Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
288 }
289 
290 bool
291 HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
292  return Subtarget.isHVXVectorType(Ty) &&
293  Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
294 }
295 
296 bool
297 HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
298  return Subtarget.isHVXVectorType(Ty, true) &&
300 }
301 
302 bool
303 HexagonTargetLowering::allowsHvxMemoryAccess(MVT VecTy, unsigned Alignment,
304  MachineMemOperand::Flags Flags, bool *Fast) const {
305  // Bool vectors are excluded by default, but make it explicit to
306  // emphasize that bool vectors cannot be loaded or stored.
307  // Also, disallow double vector stores (to prevent unnecessary
308  // store widening in DAG combiner).
309  if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
310  return false;
311  if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
312  return false;
313  if (Fast)
314  *Fast = true;
315  return true;
316 }
317 
318 bool
319 HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(MVT VecTy,
320  unsigned Align, MachineMemOperand::Flags Flags, bool *Fast) const {
321  if (!Subtarget.isHVXVectorType(VecTy))
322  return false;
323  // XXX Should this be false? vmemu are a bit slower than vmem.
324  if (Fast)
325  *Fast = true;
326  return true;
327 }
328 
329 SDValue
330 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
331  SelectionDAG &DAG) const {
332  if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
333  ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
334 
335  unsigned ElemWidth = ElemTy.getSizeInBits();
336  if (ElemWidth == 8)
337  return ElemIdx;
338 
339  unsigned L = Log2_32(ElemWidth/8);
340  const SDLoc &dl(ElemIdx);
341  return DAG.getNode(ISD::SHL, dl, MVT::i32,
342  {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
343 }
344 
345 SDValue
346 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
347  SelectionDAG &DAG) const {
348  unsigned ElemWidth = ElemTy.getSizeInBits();
349  assert(ElemWidth >= 8 && ElemWidth <= 32);
350  if (ElemWidth == 32)
351  return Idx;
352 
353  if (ty(Idx) != MVT::i32)
354  Idx = DAG.getBitcast(MVT::i32, Idx);
355  const SDLoc &dl(Idx);
356  SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
357  SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
358  return SubIdx;
359 }
360 
361 SDValue
362 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
364  SelectionDAG &DAG) const {
365  MVT OpTy = ty(Op0);
366  assert(OpTy == ty(Op1));
367 
368  MVT ElemTy = OpTy.getVectorElementType();
369  if (ElemTy == MVT::i8)
370  return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
371  assert(ElemTy.getSizeInBits() >= 8);
372 
373  MVT ResTy = tyVector(OpTy, MVT::i8);
374  unsigned ElemSize = ElemTy.getSizeInBits() / 8;
375 
376  SmallVector<int,128> ByteMask;
377  for (int M : Mask) {
378  if (M < 0) {
379  for (unsigned I = 0; I != ElemSize; ++I)
380  ByteMask.push_back(-1);
381  } else {
382  int NewM = M*ElemSize;
383  for (unsigned I = 0; I != ElemSize; ++I)
384  ByteMask.push_back(NewM+I);
385  }
386  }
387  assert(ResTy.getVectorNumElements() == ByteMask.size());
388  return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
389  opCastElem(Op1, MVT::i8, DAG), ByteMask);
390 }
391 
392 SDValue
393 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
394  const SDLoc &dl, MVT VecTy,
395  SelectionDAG &DAG) const {
396  unsigned VecLen = Values.size();
398  MVT ElemTy = VecTy.getVectorElementType();
399  unsigned ElemWidth = ElemTy.getSizeInBits();
400  unsigned HwLen = Subtarget.getVectorLength();
401 
402  unsigned ElemSize = ElemWidth / 8;
403  assert(ElemSize*VecLen == HwLen);
405 
406  if (VecTy.getVectorElementType() != MVT::i32) {
407  assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
408  unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
409  MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
410  for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
411  SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
412  Words.push_back(DAG.getBitcast(MVT::i32, W));
413  }
414  } else {
415  Words.assign(Values.begin(), Values.end());
416  }
417 
418  unsigned NumWords = Words.size();
419  bool IsSplat = true, IsUndef = true;
420  SDValue SplatV;
421  for (unsigned i = 0; i != NumWords && IsSplat; ++i) {
422  if (isUndef(Words[i]))
423  continue;
424  IsUndef = false;
425  if (!SplatV.getNode())
426  SplatV = Words[i];
427  else if (SplatV != Words[i])
428  IsSplat = false;
429  }
430  if (IsUndef)
431  return DAG.getUNDEF(VecTy);
432  if (IsSplat) {
433  assert(SplatV.getNode());
434  auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
435  if (IdxN && IdxN->isNullValue())
436  return getZero(dl, VecTy, DAG);
437  return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV);
438  }
439 
440  // Delay recognizing constant vectors until here, so that we can generate
441  // a vsplat.
442  SmallVector<ConstantInt*, 128> Consts(VecLen);
443  bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
444  if (AllConst) {
445  ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
446  (Constant**)Consts.end());
447  Constant *CV = ConstantVector::get(Tmp);
448  Align Alignment(HwLen);
449  SDValue CP =
450  LowerConstantPool(DAG.getConstantPool(CV, VecTy, Alignment), DAG);
451  return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
452  MachinePointerInfo::getConstantPool(MF), Alignment);
453  }
454 
455  // A special case is a situation where the vector is built entirely from
456  // elements extracted from another vector. This could be done via a shuffle
457  // more efficiently, but typically, the size of the source vector will not
458  // match the size of the vector being built (which precludes the use of a
459  // shuffle directly).
460  // This only handles a single source vector, and the vector being built
461  // should be of a sub-vector type of the source vector type.
462  auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
463  SmallVectorImpl<int> &SrcIdx) {
464  SDValue Vec;
465  for (SDValue V : Values) {
466  if (isUndef(V)) {
467  SrcIdx.push_back(-1);
468  continue;
469  }
470  if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
471  return false;
472  // All extracts should come from the same vector.
473  SDValue T = V.getOperand(0);
474  if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
475  return false;
476  Vec = T;
477  ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
478  if (C == nullptr)
479  return false;
480  int I = C->getSExtValue();
481  assert(I >= 0 && "Negative element index");
482  SrcIdx.push_back(I);
483  }
484  SrcVec = Vec;
485  return true;
486  };
487 
488  SmallVector<int,128> ExtIdx;
489  SDValue ExtVec;
490  if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
491  MVT ExtTy = ty(ExtVec);
492  unsigned ExtLen = ExtTy.getVectorNumElements();
493  if (ExtLen == VecLen || ExtLen == 2*VecLen) {
494  // Construct a new shuffle mask that will produce a vector with the same
495  // number of elements as the input vector, and such that the vector we
496  // want will be the initial subvector of it.
498  BitVector Used(ExtLen);
499 
500  for (int M : ExtIdx) {
501  Mask.push_back(M);
502  if (M >= 0)
503  Used.set(M);
504  }
505  // Fill the rest of the mask with the unused elements of ExtVec in hopes
506  // that it will result in a permutation of ExtVec's elements. It's still
507  // fine if it doesn't (e.g. if undefs are present, or elements are
508  // repeated), but permutations can always be done efficiently via vdelta
509  // and vrdelta.
510  for (unsigned I = 0; I != ExtLen; ++I) {
511  if (Mask.size() == ExtLen)
512  break;
513  if (!Used.test(I))
514  Mask.push_back(I);
515  }
516 
517  SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
518  DAG.getUNDEF(ExtTy), Mask);
519  if (ExtLen == VecLen)
520  return S;
521  return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, VecTy, S);
522  }
523  }
524 
525  // Construct two halves in parallel, then or them together.
526  assert(4*Words.size() == Subtarget.getVectorLength());
527  SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
528  SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
529  SDValue S = DAG.getConstant(4, dl, MVT::i32);
530  for (unsigned i = 0; i != NumWords/2; ++i) {
531  SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
532  {HalfV0, Words[i]});
533  SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
534  {HalfV1, Words[i+NumWords/2]});
535  HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
536  HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
537  }
538 
539  HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
540  {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
541  SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
542  return DstV;
543 }
544 
545 SDValue
546 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
547  unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
548  MVT PredTy = ty(PredV);
549  unsigned HwLen = Subtarget.getVectorLength();
550  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
551 
552  if (Subtarget.isHVXVectorType(PredTy, true)) {
553  // Move the vector predicate SubV to a vector register, and scale it
554  // down to match the representation (bytes per type element) that VecV
555  // uses. The scaling down will pick every 2nd or 4th (every Scale-th
556  // in general) element and put them at the front of the resulting
557  // vector. This subvector will then be inserted into the Q2V of VecV.
558  // To avoid having an operation that generates an illegal type (short
559  // vector), generate a full size vector.
560  //
561  SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
562  SmallVector<int,128> Mask(HwLen);
563  // Scale = BitBytes(PredV) / Given BitBytes.
564  unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
565  unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
566 
567  for (unsigned i = 0; i != HwLen; ++i) {
568  unsigned Num = i % Scale;
569  unsigned Off = i / Scale;
570  Mask[BlockLen*Num + Off] = i;
571  }
572  SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
573  if (!ZeroFill)
574  return S;
575  // Fill the bytes beyond BlockLen with 0s.
576  MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
577  SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
578  {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
579  SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
580  return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
581  }
582 
583  // Make sure that this is a valid scalar predicate.
584  assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
585 
586  unsigned Bytes = 8 / PredTy.getVectorNumElements();
587  SmallVector<SDValue,4> Words[2];
588  unsigned IdxW = 0;
589 
590  auto Lo32 = [&DAG, &dl] (SDValue P) {
591  return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P);
592  };
593  auto Hi32 = [&DAG, &dl] (SDValue P) {
594  return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P);
595  };
596 
597  SDValue W0 = isUndef(PredV)
598  ? DAG.getUNDEF(MVT::i64)
599  : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
600  Words[IdxW].push_back(Hi32(W0));
601  Words[IdxW].push_back(Lo32(W0));
602 
603  while (Bytes < BitBytes) {
604  IdxW ^= 1;
605  Words[IdxW].clear();
606 
607  if (Bytes < 4) {
608  for (const SDValue &W : Words[IdxW ^ 1]) {
609  SDValue T = expandPredicate(W, dl, DAG);
610  Words[IdxW].push_back(Hi32(T));
611  Words[IdxW].push_back(Lo32(T));
612  }
613  } else {
614  for (const SDValue &W : Words[IdxW ^ 1]) {
615  Words[IdxW].push_back(W);
616  Words[IdxW].push_back(W);
617  }
618  }
619  Bytes *= 2;
620  }
621 
622  assert(Bytes == BitBytes);
623 
624  SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
625  SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
626  for (const SDValue &W : Words[IdxW]) {
627  Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
628  Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
629  }
630 
631  return Vec;
632 }
633 
634 SDValue
635 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
636  const SDLoc &dl, MVT VecTy,
637  SelectionDAG &DAG) const {
638  // Construct a vector V of bytes, such that a comparison V >u 0 would
639  // produce the required vector predicate.
640  unsigned VecLen = Values.size();
641  unsigned HwLen = Subtarget.getVectorLength();
642  assert(VecLen <= HwLen || VecLen == 8*HwLen);
644  bool AllT = true, AllF = true;
645 
646  auto IsTrue = [] (SDValue V) {
647  if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
648  return !N->isNullValue();
649  return false;
650  };
651  auto IsFalse = [] (SDValue V) {
652  if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
653  return N->isNullValue();
654  return false;
655  };
656 
657  if (VecLen <= HwLen) {
658  // In the hardware, each bit of a vector predicate corresponds to a byte
659  // of a vector register. Calculate how many bytes does a bit of VecTy
660  // correspond to.
661  assert(HwLen % VecLen == 0);
662  unsigned BitBytes = HwLen / VecLen;
663  for (SDValue V : Values) {
664  AllT &= IsTrue(V);
665  AllF &= IsFalse(V);
666 
667  SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
668  : DAG.getUNDEF(MVT::i8);
669  for (unsigned B = 0; B != BitBytes; ++B)
670  Bytes.push_back(Ext);
671  }
672  } else {
673  // There are as many i1 values, as there are bits in a vector register.
674  // Divide the values into groups of 8 and check that each group consists
675  // of the same value (ignoring undefs).
676  for (unsigned I = 0; I != VecLen; I += 8) {
677  unsigned B = 0;
678  // Find the first non-undef value in this group.
679  for (; B != 8; ++B) {
680  if (!Values[I+B].isUndef())
681  break;
682  }
683  SDValue F = Values[I+B];
684  AllT &= IsTrue(F);
685  AllF &= IsFalse(F);
686 
687  SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
688  : DAG.getUNDEF(MVT::i8);
689  Bytes.push_back(Ext);
690  // Verify that the rest of values in the group are the same as the
691  // first.
692  for (; B != 8; ++B)
693  assert(Values[I+B].isUndef() || Values[I+B] == F);
694  }
695  }
696 
697  if (AllT)
698  return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
699  if (AllF)
700  return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
701 
702  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
703  SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
704  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
705 }
706 
707 SDValue
708 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
709  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
710  MVT ElemTy = ty(VecV).getVectorElementType();
711 
712  unsigned ElemWidth = ElemTy.getSizeInBits();
713  assert(ElemWidth >= 8 && ElemWidth <= 32);
714  (void)ElemWidth;
715 
716  SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
717  SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
718  {VecV, ByteIdx});
719  if (ElemTy == MVT::i32)
720  return ExWord;
721 
722  // Have an extracted word, need to extract the smaller element out of it.
723  // 1. Extract the bits of (the original) IdxV that correspond to the index
724  // of the desired element in the 32-bit word.
725  SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
726  // 2. Extract the element from the word.
727  SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
728  return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
729 }
730 
731 SDValue
732 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
733  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
734  // Implement other return types if necessary.
735  assert(ResTy == MVT::i1);
736 
737  unsigned HwLen = Subtarget.getVectorLength();
738  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
739  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
740 
741  unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
742  SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
743  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
744 
745  SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
746  SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
747  return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
748 }
749 
750 SDValue
751 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
752  SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
753  MVT ElemTy = ty(VecV).getVectorElementType();
754 
755  unsigned ElemWidth = ElemTy.getSizeInBits();
756  assert(ElemWidth >= 8 && ElemWidth <= 32);
757  (void)ElemWidth;
758 
759  auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
760  SDValue ByteIdxV) {
761  MVT VecTy = ty(VecV);
762  unsigned HwLen = Subtarget.getVectorLength();
763  SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
764  {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
765  SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
766  SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
767  SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
768  {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
769  SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
770  return TorV;
771  };
772 
773  SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
774  if (ElemTy == MVT::i32)
775  return InsertWord(VecV, ValV, ByteIdx);
776 
777  // If this is not inserting a 32-bit word, convert it into such a thing.
778  // 1. Extract the existing word from the target vector.
779  SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
780  {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
781  SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
782  dl, MVT::i32, DAG);
783 
784  // 2. Treating the extracted word as a 32-bit vector, insert the given
785  // value into it.
786  SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
787  MVT SubVecTy = tyVector(ty(Ext), ElemTy);
788  SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
789  ValV, SubIdx, dl, ElemTy, DAG);
790 
791  // 3. Insert the 32-bit word back into the original vector.
792  return InsertWord(VecV, Ins, ByteIdx);
793 }
794 
795 SDValue
796 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
797  SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
798  unsigned HwLen = Subtarget.getVectorLength();
799  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
800  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
801 
802  unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
803  SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
804  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
805  ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
806 
807  SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
808  return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
809 }
810 
811 SDValue
812 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
813  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
814  MVT VecTy = ty(VecV);
815  unsigned HwLen = Subtarget.getVectorLength();
816  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
817  MVT ElemTy = VecTy.getVectorElementType();
818  unsigned ElemWidth = ElemTy.getSizeInBits();
819 
820  // If the source vector is a vector pair, get the single vector containing
821  // the subvector of interest. The subvector will never overlap two single
822  // vectors.
823  if (isHvxPairTy(VecTy)) {
824  unsigned SubIdx;
825  if (Idx * ElemWidth >= 8*HwLen) {
826  SubIdx = Hexagon::vsub_hi;
827  Idx -= VecTy.getVectorNumElements() / 2;
828  } else {
829  SubIdx = Hexagon::vsub_lo;
830  }
831  VecTy = typeSplit(VecTy).first;
832  VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV);
833  if (VecTy == ResTy)
834  return VecV;
835  }
836 
837  // The only meaningful subvectors of a single HVX vector are those that
838  // fit in a scalar register.
839  assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
840 
841  MVT WordTy = tyVector(VecTy, MVT::i32);
842  SDValue WordVec = DAG.getBitcast(WordTy, VecV);
843  unsigned WordIdx = (Idx*ElemWidth) / 32;
844 
845  SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
846  SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
847  if (ResTy.getSizeInBits() == 32)
848  return DAG.getBitcast(ResTy, W0);
849 
850  SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
851  SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
852  SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
853  return DAG.getBitcast(ResTy, WW);
854 }
855 
856 SDValue
857 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
858  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
859  MVT VecTy = ty(VecV);
860  unsigned HwLen = Subtarget.getVectorLength();
861  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
862  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
863  // IdxV is required to be a constant.
864  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
865 
866  unsigned ResLen = ResTy.getVectorNumElements();
867  unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
868  unsigned Offset = Idx * BitBytes;
869  SDValue Undef = DAG.getUNDEF(ByteTy);
871 
872  if (Subtarget.isHVXVectorType(ResTy, true)) {
873  // Converting between two vector predicates. Since the result is shorter
874  // than the source, it will correspond to a vector predicate with the
875  // relevant bits replicated. The replication count is the ratio of the
876  // source and target vector lengths.
877  unsigned Rep = VecTy.getVectorNumElements() / ResLen;
878  assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
879  for (unsigned i = 0; i != HwLen/Rep; ++i) {
880  for (unsigned j = 0; j != Rep; ++j)
881  Mask.push_back(i + Offset);
882  }
883  SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
884  return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
885  }
886 
887  // Converting between a vector predicate and a scalar predicate. In the
888  // vector predicate, a group of BitBytes bits will correspond to a single
889  // i1 element of the source vector type. Those bits will all have the same
890  // value. The same will be true for ByteVec, where each byte corresponds
891  // to a bit in the vector predicate.
892  // The algorithm is to traverse the ByteVec, going over the i1 values from
893  // the source vector, and generate the corresponding representation in an
894  // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
895  // elements so that the interesting 8 bytes will be in the low end of the
896  // vector.
897  unsigned Rep = 8 / ResLen;
898  // Make sure the output fill the entire vector register, so repeat the
899  // 8-byte groups as many times as necessary.
900  for (unsigned r = 0; r != HwLen/ResLen; ++r) {
901  // This will generate the indexes of the 8 interesting bytes.
902  for (unsigned i = 0; i != ResLen; ++i) {
903  for (unsigned j = 0; j != Rep; ++j)
904  Mask.push_back(Offset + i*BitBytes);
905  }
906  }
907 
908  SDValue Zero = getZero(dl, MVT::i32, DAG);
909  SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
910  // Combine the two low words from ShuffV into a v8i8, and byte-compare
911  // them against 0.
912  SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
914  {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
915  SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
916  return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
917  {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
918 }
919 
920 SDValue
921 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
922  SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
923  MVT VecTy = ty(VecV);
924  MVT SubTy = ty(SubV);
925  unsigned HwLen = Subtarget.getVectorLength();
926  MVT ElemTy = VecTy.getVectorElementType();
927  unsigned ElemWidth = ElemTy.getSizeInBits();
928 
929  bool IsPair = isHvxPairTy(VecTy);
930  MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
931  // The two single vectors that VecV consists of, if it's a pair.
932  SDValue V0, V1;
933  SDValue SingleV = VecV;
934  SDValue PickHi;
935 
936  if (IsPair) {
937  V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV);
938  V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV);
939 
940  SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
941  dl, MVT::i32);
942  PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
943  if (isHvxSingleTy(SubTy)) {
944  if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
945  unsigned Idx = CN->getZExtValue();
946  assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
947  unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
948  return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
949  }
950  // If IdxV is not a constant, generate the two variants: with the
951  // SubV as the high and as the low subregister, and select the right
952  // pair based on the IdxV.
953  SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
954  SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
955  return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
956  }
957  // The subvector being inserted must be entirely contained in one of
958  // the vectors V0 or V1. Set SingleV to the correct one, and update
959  // IdxV to be the index relative to the beginning of that vector.
960  SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
961  IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
962  SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
963  }
964 
965  // The only meaningful subvectors of a single HVX vector are those that
966  // fit in a scalar register.
967  assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
968  // Convert IdxV to be index in bytes.
969  auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
970  if (!IdxN || !IdxN->isNullValue()) {
971  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
972  DAG.getConstant(ElemWidth/8, dl, MVT::i32));
973  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
974  }
975  // When inserting a single word, the rotation back to the original position
976  // would be by HwLen-Idx, but if two words are inserted, it will need to be
977  // by (HwLen-4)-Idx.
978  unsigned RolBase = HwLen;
979  if (VecTy.getSizeInBits() == 32) {
980  SDValue V = DAG.getBitcast(MVT::i32, SubV);
981  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V);
982  } else {
983  SDValue V = DAG.getBitcast(MVT::i64, SubV);
984  SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
985  SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
986  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
987  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
988  DAG.getConstant(4, dl, MVT::i32));
989  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
990  RolBase = HwLen-4;
991  }
992  // If the vector wasn't ror'ed, don't ror it back.
993  if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) {
994  SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
995  DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
996  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
997  }
998 
999  if (IsPair) {
1000  SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
1001  SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
1002  return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
1003  }
1004  return SingleV;
1005 }
1006 
1007 SDValue
1008 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1009  SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1010  MVT VecTy = ty(VecV);
1011  MVT SubTy = ty(SubV);
1012  assert(Subtarget.isHVXVectorType(VecTy, true));
1013  // VecV is an HVX vector predicate. SubV may be either an HVX vector
1014  // predicate as well, or it can be a scalar predicate.
1015 
1016  unsigned VecLen = VecTy.getVectorNumElements();
1017  unsigned HwLen = Subtarget.getVectorLength();
1018  assert(HwLen % VecLen == 0 && "Unexpected vector type");
1019 
1020  unsigned Scale = VecLen / SubTy.getVectorNumElements();
1021  unsigned BitBytes = HwLen / VecLen;
1022  unsigned BlockLen = HwLen / Scale;
1023 
1024  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1025  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1026  SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
1027  SDValue ByteIdx;
1028 
1029  auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
1030  if (!IdxN || !IdxN->isNullValue()) {
1031  ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
1032  DAG.getConstant(BitBytes, dl, MVT::i32));
1033  ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
1034  }
1035 
1036  // ByteVec is the target vector VecV rotated in such a way that the
1037  // subvector should be inserted at index 0. Generate a predicate mask
1038  // and use vmux to do the insertion.
1039  MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
1040  SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1041  {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1042  ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1043  // Rotate ByteVec back, and convert to a vector predicate.
1044  if (!IdxN || !IdxN->isNullValue()) {
1045  SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
1046  SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
1047  ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
1048  }
1049  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
1050 }
1051 
1052 SDValue
1053 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1054  MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1055  // Sign- and any-extending of a vector predicate to a vector register is
1056  // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1057  // a vector of 1s (where the 1s are of type matching the vector type).
1058  assert(Subtarget.isHVXVectorType(ResTy));
1059  if (!ZeroExt)
1060  return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
1061 
1062  assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1063  SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
1064  DAG.getConstant(1, dl, MVT::i32));
1065  SDValue False = getZero(dl, ResTy, DAG);
1066  return DAG.getSelect(dl, ResTy, VecV, True, False);
1067 }
1068 
1069 SDValue
1070 HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1071  MVT ResTy, SelectionDAG &DAG) const {
1072  // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1073  // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1074  // vector register. The remaining bits of the vector register are
1075  // unspecified.
1076 
1077  MachineFunction &MF = DAG.getMachineFunction();
1078  unsigned HwLen = Subtarget.getVectorLength();
1079  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1080  MVT PredTy = ty(VecQ);
1081  unsigned PredLen = PredTy.getVectorNumElements();
1082  assert(HwLen % PredLen == 0);
1083  MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(8*HwLen/PredLen), PredLen);
1084 
1085  Type *Int8Ty = Type::getInt8Ty(*DAG.getContext());
1087  // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1088  // These are bytes with the LSB rotated left with respect to their index.
1089  for (unsigned i = 0; i != HwLen/8; ++i) {
1090  for (unsigned j = 0; j != 8; ++j)
1091  Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j));
1092  }
1093  Constant *CV = ConstantVector::get(Tmp);
1094  Align Alignment(HwLen);
1095  SDValue CP =
1096  LowerConstantPool(DAG.getConstantPool(CV, ByteTy, Alignment), DAG);
1097  SDValue Bytes =
1098  DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP,
1099  MachinePointerInfo::getConstantPool(MF), Alignment);
1100 
1101  // Select the bytes that correspond to true bits in the vector predicate.
1102  SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes),
1103  getZero(dl, VecTy, DAG));
1104  // Calculate the OR of all bytes in each group of 8. That will compress
1105  // all the individual bits into a single byte.
1106  // First, OR groups of 4, via vrmpy with 0x01010101.
1107  SDValue All1 =
1108  DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32));
1109  SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1110  // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1111  SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1112  {Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG);
1113  SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot});
1114 
1115  // Pick every 8th byte and coalesce them at the beginning of the output.
1116  // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1117  // byte and so on.
1119  for (unsigned i = 0; i != HwLen; ++i)
1120  Mask.push_back((8*i) % HwLen + i/(HwLen/8));
1121  SDValue Collect =
1122  DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask);
1123  return DAG.getBitcast(ResTy, Collect);
1124 }
1125 
1126 SDValue
1127 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1128  const {
1129  const SDLoc &dl(Op);
1130  MVT VecTy = ty(Op);
1131 
1132  unsigned Size = Op.getNumOperands();
1134  for (unsigned i = 0; i != Size; ++i)
1135  Ops.push_back(Op.getOperand(i));
1136 
1137  if (VecTy.getVectorElementType() == MVT::i1)
1138  return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1139 
1140  if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
1141  ArrayRef<SDValue> A(Ops);
1142  MVT SingleTy = typeSplit(VecTy).first;
1143  SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
1144  SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
1145  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
1146  }
1147 
1148  return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1149 }
1150 
1151 SDValue
1152 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1153  const {
1154  // Vector concatenation of two integer (non-bool) vectors does not need
1155  // special lowering. Custom-lower concats of bool vectors and expand
1156  // concats of more than 2 vectors.
1157  MVT VecTy = ty(Op);
1158  const SDLoc &dl(Op);
1159  unsigned NumOp = Op.getNumOperands();
1160  if (VecTy.getVectorElementType() != MVT::i1) {
1161  if (NumOp == 2)
1162  return Op;
1163  // Expand the other cases into a build-vector.
1164  SmallVector<SDValue,8> Elems;
1165  for (SDValue V : Op.getNode()->ops())
1166  DAG.ExtractVectorElements(V, Elems);
1167  // A vector of i16 will be broken up into a build_vector of i16's.
1168  // This is a problem, since at the time of operation legalization,
1169  // all operations are expected to be type-legalized, and i16 is not
1170  // a legal type. If any of the extracted elements is not of a valid
1171  // type, sign-extend it to a valid one.
1172  for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
1173  SDValue V = Elems[i];
1174  MVT Ty = ty(V);
1175  if (!isTypeLegal(Ty)) {
1176  EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty);
1177  if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1178  Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1179  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1180  V.getOperand(0), V.getOperand(1)),
1181  DAG.getValueType(Ty));
1182  continue;
1183  }
1184  // A few less complicated cases.
1185  if (V.getOpcode() == ISD::Constant)
1186  Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1187  else if (V.isUndef())
1188  Elems[i] = DAG.getUNDEF(NTy);
1189  else
1190  llvm_unreachable("Unexpected vector element");
1191  }
1192  }
1193  return DAG.getBuildVector(VecTy, dl, Elems);
1194  }
1195 
1196  assert(VecTy.getVectorElementType() == MVT::i1);
1197  unsigned HwLen = Subtarget.getVectorLength();
1198  assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1199 
1200  SDValue Op0 = Op.getOperand(0);
1201 
1202  // If the operands are HVX types (i.e. not scalar predicates), then
1203  // defer the concatenation, and create QCAT instead.
1204  if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1205  if (NumOp == 2)
1206  return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1207 
1208  ArrayRef<SDUse> U(Op.getNode()->ops());
1209  SmallVector<SDValue,4> SV(U.begin(), U.end());
1210  ArrayRef<SDValue> Ops(SV);
1211 
1212  MVT HalfTy = typeSplit(VecTy).first;
1213  SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1214  Ops.take_front(NumOp/2));
1215  SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1216  Ops.take_back(NumOp/2));
1217  return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1218  }
1219 
1220  // Count how many bytes (in a vector register) each bit in VecTy
1221  // corresponds to.
1222  unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1223 
1224  SmallVector<SDValue,8> Prefixes;
1225  for (SDValue V : Op.getNode()->op_values()) {
1226  SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1227  Prefixes.push_back(P);
1228  }
1229 
1230  unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1231  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1232  SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1233  SDValue Res = getZero(dl, ByteTy, DAG);
1234  for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1235  Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1236  Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1237  }
1238  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1239 }
1240 
1241 SDValue
1242 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1243  const {
1244  // Change the type of the extracted element to i32.
1245  SDValue VecV = Op.getOperand(0);
1246  MVT ElemTy = ty(VecV).getVectorElementType();
1247  const SDLoc &dl(Op);
1248  SDValue IdxV = Op.getOperand(1);
1249  if (ElemTy == MVT::i1)
1250  return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1251 
1252  return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1253 }
1254 
1255 SDValue
1256 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1257  const {
1258  const SDLoc &dl(Op);
1259  SDValue VecV = Op.getOperand(0);
1260  SDValue ValV = Op.getOperand(1);
1261  SDValue IdxV = Op.getOperand(2);
1262  MVT ElemTy = ty(VecV).getVectorElementType();
1263  if (ElemTy == MVT::i1)
1264  return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1265 
1266  return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1267 }
1268 
1269 SDValue
1270 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1271  const {
1272  SDValue SrcV = Op.getOperand(0);
1273  MVT SrcTy = ty(SrcV);
1274  MVT DstTy = ty(Op);
1275  SDValue IdxV = Op.getOperand(1);
1276  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
1277  assert(Idx % DstTy.getVectorNumElements() == 0);
1278  (void)Idx;
1279  const SDLoc &dl(Op);
1280 
1281  MVT ElemTy = SrcTy.getVectorElementType();
1282  if (ElemTy == MVT::i1)
1283  return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1284 
1285  return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1286 }
1287 
1288 SDValue
1289 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1290  const {
1291  // Idx does not need to be a constant.
1292  SDValue VecV = Op.getOperand(0);
1293  SDValue ValV = Op.getOperand(1);
1294  SDValue IdxV = Op.getOperand(2);
1295 
1296  const SDLoc &dl(Op);
1297  MVT VecTy = ty(VecV);
1298  MVT ElemTy = VecTy.getVectorElementType();
1299  if (ElemTy == MVT::i1)
1300  return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1301 
1302  return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1303 }
1304 
1305 SDValue
1306 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1307  // Lower any-extends of boolean vectors to sign-extends, since they
1308  // translate directly to Q2V. Zero-extending could also be done equally
1309  // fast, but Q2V is used/recognized in more places.
1310  // For all other vectors, use zero-extend.
1311  MVT ResTy = ty(Op);
1312  SDValue InpV = Op.getOperand(0);
1313  MVT ElemTy = ty(InpV).getVectorElementType();
1314  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1315  return LowerHvxSignExt(Op, DAG);
1316  return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
1317 }
1318 
1319 SDValue
1320 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1321  MVT ResTy = ty(Op);
1322  SDValue InpV = Op.getOperand(0);
1323  MVT ElemTy = ty(InpV).getVectorElementType();
1324  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1325  return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1326  return Op;
1327 }
1328 
1329 SDValue
1330 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1331  MVT ResTy = ty(Op);
1332  SDValue InpV = Op.getOperand(0);
1333  MVT ElemTy = ty(InpV).getVectorElementType();
1334  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1335  return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1336  return Op;
1337 }
1338 
1339 SDValue
1340 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1341  // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
1342  // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1343  const SDLoc &dl(Op);
1344  MVT ResTy = ty(Op);
1345  SDValue InpV = Op.getOperand(0);
1346  assert(ResTy == ty(InpV));
1347 
1348  // Calculate the vectors of 1 and bitwidth(x).
1349  MVT ElemTy = ty(InpV).getVectorElementType();
1350  unsigned ElemWidth = ElemTy.getSizeInBits();
1351  // Using uint64_t because a shift by 32 can happen.
1352  uint64_t Splat1 = 0, SplatW = 0;
1353  assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32);
1354  for (unsigned i = 0; i != 32/ElemWidth; ++i) {
1355  Splat1 = (Splat1 << ElemWidth) | 1;
1356  SplatW = (SplatW << ElemWidth) | ElemWidth;
1357  }
1358  SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1359  DAG.getConstant(uint32_t(Splat1), dl, MVT::i32));
1360  SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1361  DAG.getConstant(uint32_t(SplatW), dl, MVT::i32));
1362  SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1363  DAG.getConstant(-1, dl, MVT::i32));
1364  // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
1365  // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
1366  // it separately in custom combine or selection).
1367  SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1368  {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1369  DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1370  return DAG.getNode(ISD::SUB, dl, ResTy,
1371  {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1372 }
1373 
1374 SDValue
1375 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1376  MVT ResTy = ty(Op);
1377  assert(ResTy.isVector() && isHvxSingleTy(ResTy));
1378  const SDLoc &dl(Op);
1379  SmallVector<int,256> ShuffMask;
1380 
1381  MVT ElemTy = ResTy.getVectorElementType();
1382  unsigned VecLen = ResTy.getVectorNumElements();
1383  SDValue Vs = Op.getOperand(0);
1384  SDValue Vt = Op.getOperand(1);
1385 
1386  switch (ElemTy.SimpleTy) {
1387  case MVT::i8: {
1388  // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1389  // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1390  // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1391  MVT ExtTy = typeExtElem(ResTy, 2);
1392  unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
1393  : Hexagon::V6_vmpyhv;
1394  SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1395 
1396  // Discard high halves of the resulting values, collect the low halves.
1397  for (unsigned I = 0; I < VecLen; I += 2) {
1398  ShuffMask.push_back(I); // Pick even element.
1399  ShuffMask.push_back(I+VecLen); // Pick odd element.
1400  }
1401  VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1402  SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1403  return DAG.getBitcast(ResTy, BS);
1404  }
1405  case MVT::i16:
1406  // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode.
1407  // (There is also V6_vmpyhv, which behaves in an analogous way to
1408  // V6_vmpybv.)
1409  return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG);
1410  case MVT::i32: {
1411  // Use the following sequence for signed word multiply:
1412  // T0 = V6_vmpyiowh Vs, Vt
1413  // T1 = V6_vaslw T0, 16
1414  // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
1415  SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1416  SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1417  SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1418  SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1419  {T1, Vs, Vt}, DAG);
1420  return T2;
1421  }
1422  default:
1423  break;
1424  }
1425  return SDValue();
1426 }
1427 
1428 SDValue
1429 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1430  MVT ResTy = ty(Op);
1431  assert(ResTy.isVector());
1432  const SDLoc &dl(Op);
1433  SmallVector<int,256> ShuffMask;
1434 
1435  MVT ElemTy = ResTy.getVectorElementType();
1436  unsigned VecLen = ResTy.getVectorNumElements();
1437  SDValue Vs = Op.getOperand(0);
1438  SDValue Vt = Op.getOperand(1);
1439  bool IsSigned = Op.getOpcode() == ISD::MULHS;
1440 
1441  if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
1442  // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1443  // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1444  // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1445  // For i16, use V6_vmpyhv, which behaves in an analogous way to
1446  // V6_vmpybv: results Lo and Hi are products of even/odd elements
1447  // respectively.
1448  MVT ExtTy = typeExtElem(ResTy, 2);
1449  unsigned MpyOpc = ElemTy == MVT::i8
1450  ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1451  : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1452  SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1453 
1454  // Discard low halves of the resulting values, collect the high halves.
1455  for (unsigned I = 0; I < VecLen; I += 2) {
1456  ShuffMask.push_back(I+1); // Pick even element.
1457  ShuffMask.push_back(I+VecLen+1); // Pick odd element.
1458  }
1459  VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1460  SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1461  return DAG.getBitcast(ResTy, BS);
1462  }
1463 
1464  assert(ElemTy == MVT::i32);
1465  SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1466 
1467  if (IsSigned) {
1468  // mulhs(Vs,Vt) =
1469  // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1470  // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
1471  // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1472  // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
1473  // + Lo(Vs) *us Vt] >> 32
1474  // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
1475  // anything, so it cannot produce any carry over to higher bits),
1476  // so everything in [] can be shifted by 16 without loss of precision.
1477  // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
1478  // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
1479  // Denote Hi(Vs) = Vs':
1480  // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
1481  // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
1482  SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1483  // Get Vs':
1484  SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1485  SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1486  {T0, S0, Vt}, DAG);
1487  // Shift by 16:
1488  SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1489  // Get Vs'*Hi(Vt):
1490  SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1491  // Add:
1492  SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1493  return T3;
1494  }
1495 
1496  // Unsigned mulhw. (Would expansion using signed mulhw be better?)
1497 
1498  auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
1499  return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
1500  };
1501  auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
1502  return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
1503  };
1504 
1505  MVT PairTy = typeJoin({ResTy, ResTy});
1506  SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1507  {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
1508  // Multiply-unsigned halfwords:
1509  // LoVec = Vs.uh[2i] * Vt.uh[2i],
1510  // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
1511  SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1512  // The low halves in the LoVec of the pair can be discarded. They are
1513  // not added to anything (in the full-precision product), so they cannot
1514  // produce a carry into the higher bits.
1515  SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1516  // Swap low and high halves in Vt, and do the halfword multiplication
1517  // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
1518  SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1519  SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1520  // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
1521  // These products are words, but cannot be added directly because the
1522  // sums could overflow. Add these products, by halfwords, where each sum
1523  // of a pair of halfwords gives a word.
1524  SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1525  {LoVec(T2), HiVec(T2)}, DAG);
1526  // Add the high halfwords from the products of the low halfwords.
1527  SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1528  SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1529  SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1530  SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1531  return T7;
1532 }
1533 
1534 SDValue
1535 HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
1536  SDValue ValQ = Op.getOperand(0);
1537  MVT ResTy = ty(Op);
1538  MVT VecTy = ty(ValQ);
1539  const SDLoc &dl(Op);
1540 
1541  if (isHvxBoolTy(VecTy) && ResTy.isScalarInteger()) {
1542  unsigned HwLen = Subtarget.getVectorLength();
1543  MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4);
1544  SDValue VQ = compressHvxPred(ValQ, dl, WordTy, DAG);
1545  unsigned BitWidth = ResTy.getSizeInBits();
1546 
1547  if (BitWidth < 64) {
1548  SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
1549  dl, MVT::i32, DAG);
1550  if (BitWidth == 32)
1551  return W0;
1552  assert(BitWidth < 32u);
1553  return DAG.getZExtOrTrunc(W0, dl, ResTy);
1554  }
1555 
1556  // The result is >= 64 bits. The only options are 64 or 128.
1557  assert(BitWidth == 64 || BitWidth == 128);
1558  SmallVector<SDValue,4> Words;
1559  for (unsigned i = 0; i != BitWidth/32; ++i) {
1560  SDValue W = extractHvxElementReg(
1561  VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
1562  Words.push_back(W);
1563  }
1564  SmallVector<SDValue,2> Combines;
1565  assert(Words.size() % 2 == 0);
1566  for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
1567  SDValue C = DAG.getNode(
1568  HexagonISD::COMBINE, dl, MVT::i64, {Words[i+1], Words[i]});
1569  Combines.push_back(C);
1570  }
1571 
1572  if (BitWidth == 64)
1573  return Combines[0];
1574 
1575  return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines);
1576  }
1577 
1578  return Op;
1579 }
1580 
1581 SDValue
1582 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1583  // Sign- and zero-extends are legal.
1585  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op),
1586  Op.getOperand(0));
1587 }
1588 
1589 SDValue
1590 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1591  if (SDValue S = getVectorShiftByInt(Op, DAG))
1592  return S;
1593  return Op;
1594 }
1595 
1596 SDValue
1597 HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
1598  const SDLoc &dl(Op);
1599  MVT ResTy = ty(Op);
1600 
1601  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1602  bool Use64b = Subtarget.useHVX64BOps();
1603  unsigned IntPredCast = Use64b ? Intrinsic::hexagon_V6_pred_typecast
1604  : Intrinsic::hexagon_V6_pred_typecast_128B;
1605  if (IntNo == IntPredCast) {
1606  SDValue Vs = Op.getOperand(1);
1607  MVT OpTy = ty(Vs);
1608  if (isHvxBoolTy(ResTy) && isHvxBoolTy(OpTy)) {
1609  if (ResTy == OpTy)
1610  return Vs;
1611  return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Vs);
1612  }
1613  }
1614 
1615  return Op;
1616 }
1617 
1618 SDValue
1619 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1620  assert(!Op.isMachineOpcode());
1621  SmallVector<SDValue,2> OpsL, OpsH;
1622  const SDLoc &dl(Op);
1623 
1624  auto SplitVTNode = [&DAG,this] (const VTSDNode *N) {
1625  MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
1626  SDValue TV = DAG.getValueType(Ty);
1627  return std::make_pair(TV, TV);
1628  };
1629 
1630  for (SDValue A : Op.getNode()->ops()) {
1631  VectorPair P = Subtarget.isHVXVectorType(ty(A), true)
1632  ? opSplit(A, dl, DAG)
1633  : std::make_pair(A, A);
1634  // Special case for type operand.
1635  if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1636  if (const auto *N = dyn_cast<const VTSDNode>(A.getNode()))
1637  P = SplitVTNode(N);
1638  }
1639  OpsL.push_back(P.first);
1640  OpsH.push_back(P.second);
1641  }
1642 
1643  MVT ResTy = ty(Op);
1644  MVT HalfTy = typeSplit(ResTy).first;
1645  SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1646  SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1647  SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1648  return S;
1649 }
1650 
1651 SDValue
1652 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1653  LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode());
1654  assert(BN->isUnindexed());
1655  MVT MemTy = BN->getMemoryVT().getSimpleVT();
1656  if (!isHvxPairTy(MemTy))
1657  return Op;
1658 
1659  const SDLoc &dl(Op);
1660  unsigned HwLen = Subtarget.getVectorLength();
1661  MVT SingleTy = typeSplit(MemTy).first;
1662  SDValue Chain = BN->getChain();
1663  SDValue Base0 = BN->getBasePtr();
1664  SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1665 
1666  MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
1667  if (MachineMemOperand *MMO = BN->getMemOperand()) {
1668  MachineFunction &MF = DAG.getMachineFunction();
1669  MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen);
1670  MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen);
1671  }
1672 
1673  unsigned MemOpc = BN->getOpcode();
1674  SDValue NewOp;
1675 
1676  if (MemOpc == ISD::LOAD) {
1677  SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1678  SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1679  NewOp = DAG.getMergeValues(
1680  { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
1682  Load0.getValue(1), Load1.getValue(1)) }, dl);
1683  } else {
1684  assert(MemOpc == ISD::STORE);
1685  VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
1686  SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1687  SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1688  NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
1689  }
1690 
1691  return NewOp;
1692 }
1693 
1694 SDValue
1695 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1696  unsigned Opc = Op.getOpcode();
1697  bool IsPairOp = isHvxPairTy(ty(Op)) ||
1698  llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
1699  return isHvxPairTy(ty(V));
1700  });
1701 
1702  if (IsPairOp) {
1703  switch (Opc) {
1704  default:
1705  break;
1706  case ISD::LOAD:
1707  case ISD::STORE:
1708  return SplitHvxMemOp(Op, DAG);
1709  case ISD::CTPOP:
1710  case ISD::CTLZ:
1711  case ISD::CTTZ:
1712  case ISD::MUL:
1713  case ISD::MULHS:
1714  case ISD::MULHU:
1715  case ISD::AND:
1716  case ISD::OR:
1717  case ISD::XOR:
1718  case ISD::SRA:
1719  case ISD::SHL:
1720  case ISD::SRL:
1721  case ISD::SETCC:
1722  case ISD::VSELECT:
1723  case ISD::SIGN_EXTEND:
1724  case ISD::ZERO_EXTEND:
1726  return SplitHvxPairOp(Op, DAG);
1727  }
1728  }
1729 
1730  switch (Opc) {
1731  default:
1732  break;
1733  case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
1734  case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
1735  case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
1736  case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
1737  case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
1738  case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
1739  case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
1740  case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
1741  case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
1742  case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
1743  case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
1744  case ISD::SRA:
1745  case ISD::SHL:
1746  case ISD::SRL: return LowerHvxShift(Op, DAG);
1747  case ISD::MUL: return LowerHvxMul(Op, DAG);
1748  case ISD::MULHS:
1749  case ISD::MULHU: return LowerHvxMulh(Op, DAG);
1750  case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
1751  case ISD::SETCC:
1752  case ISD::INTRINSIC_VOID: return Op;
1753  case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
1754  // Unaligned loads will be handled by the default lowering.
1755  case ISD::LOAD: return SDValue();
1756  }
1757 #ifndef NDEBUG
1758  Op.dumpr(&DAG);
1759 #endif
1760  llvm_unreachable("Unhandled HVX operation");
1761 }
1762 
1763 void
1764 HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
1766 }
1767 
1768 void
1769 HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
1770  SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1771  unsigned Opc = N->getOpcode();
1772  switch (Opc) {
1773  case ISD::BITCAST:
1774  if (isHvxBoolTy(ty(N->getOperand(0)))) {
1775  SDValue Op(N, 0);
1776  SDValue C = LowerHvxBitcast(Op, DAG);
1777  Results.push_back(C);
1778  }
1779  break;
1780  default:
1781  break;
1782  }
1783 }
1784 
1785 SDValue
1786 HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
1787  const {
1788  const SDLoc &dl(N);
1789  SDValue Op(N, 0);
1790 
1791  unsigned Opc = Op.getOpcode();
1792  if (Opc == ISD::VSELECT) {
1793  // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0)
1794  SDValue Cond = Op.getOperand(0);
1795  if (Cond->getOpcode() == ISD::XOR) {
1796  SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
1797  if (C1->getOpcode() == HexagonISD::QTRUE) {
1798  SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
1799  Op.getOperand(2), Op.getOperand(1));
1800  return VSel;
1801  }
1802  }
1803  }
1804  return SDValue();
1805 }
1806 
1807 bool
1808 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
1809  // If the type of the result, or any operand type are HVX vector types,
1810  // this is an HVX operation.
1811  return Subtarget.isHVXVectorType(ty(Op), true) ||
1812  llvm::any_of(Op.getNode()->ops(),
1813  [this] (SDValue V) {
1814  return Subtarget.isHVXVectorType(ty(V), true);
1815  });
1816 }
1817 
1818 bool
1819 HexagonTargetLowering::isHvxOperation(SDNode *N) const {
1820  // If the type of any result, or any operand type are HVX vector types,
1821  // this is an HVX operation.
1822  auto IsHvxTy = [this] (EVT Ty) {
1823  return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true);
1824  };
1825  auto IsHvxOp = [this] (SDValue Op) {
1826  return Subtarget.isHVXVectorType(ty(Op), true);
1827  };
1828  return llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp);
1829 }
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:743
static MVT getIntegerVT(unsigned BitWidth)
BitVector & set()
Definition: BitVector.h:398
EVT getValueType() const
Return the ValueType of the referenced return value.
LLVM_NODISCARD std::enable_if_t< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > dyn_cast(const Y &Val)
Definition: Casting.h:328
bool isUndef() const
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition: ArrayRef.h:219
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:510
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:519
iterator_range< value_iterator > values() const
iterator begin() const
Definition: ArrayRef.h:144
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:697
bool isVector() const
Return true if this is a vector value type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:249
static const MVT LegalW128[]
bool test(unsigned Idx) const
Definition: BitVector.h:502
unsigned getVectorNumElements() const
const SDValue & getChain() const
Function Alias Analysis Results
F(f)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:686
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
static const MVT LegalV64[]
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1...
Definition: ISDOpcodes.h:497
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
A description of a memory reference used in the backend.
Shift and rotation operations.
Definition: ISDOpcodes.h:575
Base class for LoadSDNode and StoreSDNode.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:212
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn&#39;t supported on the target and indicate what to d...
SimpleValueType SimpleTy
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:486
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
This class is used to represent EVT&#39;s, which are used to parameterize some operations.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:458
int64_t getSExtValue() const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:421
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:605
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:222
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:915
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:607
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:675
ArrayRef< SDUse > ops() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements. ...
Definition: SelectionDAG.h:779
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:167
MVT getVectorElementType() const
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified, possibly variable, elements.
Definition: ISDOpcodes.h:455
SmallVector< MachineOperand, 4 > Cond
#define P(N)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:182
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:492
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
void setTargetDAGCombine(ISD::NodeType NT)
Targets should invoke this method for each target independent node that they want to provide a custom...
bool isMachineOpcode() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:156
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
static const MVT LegalW64[]
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:464
#define H(x, y, z)
Definition: MD5.cpp:58
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static const MVT LegalV128[]
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1490
constexpr double e
Definition: MathExtras.h:58
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:147
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
Extended Value Type.
Definition: ValueTypes.h:35
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:52
bool isHVXVectorType(MVT VecTy, bool IncludeBool=false) const
SDValue getMemBasePlusOffset(SDValue Base, int64_t Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:475
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:223
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:883
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:762
iterator end() const
Definition: ArrayRef.h:145
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:584
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:719
Represents one node in the SelectionDAG.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:597
void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:596
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:646
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:649
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array...
Definition: ArrayRef.h:186
Flags
Flags values. These may be or&#39;d together.
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
void dumpr() const
ArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
Definition: ArrayRef.h:226
unsigned getVectorLength() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:550
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:664
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:195
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:816
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
#define I(x, y, z)
Definition: MD5.cpp:59
#define N
uint32_t Size
Definition: Profile.cpp:46
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:483
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getValueType(EVT)
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:619
unsigned getNumOperands() const
Conversion operators.
Definition: ISDOpcodes.h:643
const SDValue & getOperand(unsigned i) const
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:184
const SDValue & getBasePtr() const
LLVMContext * getContext() const
Definition: SelectionDAG.h:431
static Constant * get(ArrayRef< Constant *> V)
Definition: Constants.cpp:1187
#define T1
void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:539