LLVM  9.0.0svn
HexagonISelLoweringHVX.cpp
Go to the documentation of this file.
1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "HexagonISelLowering.h"
10 #include "HexagonRegisterInfo.h"
11 #include "HexagonSubtarget.h"
13 
14 using namespace llvm;
15 
20 
21 
22 void
23 HexagonTargetLowering::initializeHVXLowering() {
24  if (Subtarget.useHVX64BOps()) {
25  addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass);
26  addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
27  addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
28  addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
29  addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
30  addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
31  // These "short" boolean vector types should be legal because
32  // they will appear as results of vector compares. If they were
33  // not legal, type legalization would try to make them legal
34  // and that would require using operations that do not use or
35  // produce such types. That, in turn, would imply using custom
36  // nodes, which would be unoptimizable by the DAG combiner.
37  // The idea is to rely on target-independent operations as much
38  // as possible.
39  addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
40  addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
41  addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
42  addRegisterClass(MVT::v512i1, &Hexagon::HvxQRRegClass);
43  } else if (Subtarget.useHVX128BOps()) {
44  addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass);
45  addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass);
46  addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass);
47  addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass);
48  addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
49  addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass);
50  addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
51  addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
52  addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
53  addRegisterClass(MVT::v1024i1, &Hexagon::HvxQRRegClass);
54  }
55 
56  // Set up operation actions.
57 
58  bool Use64b = Subtarget.useHVX64BOps();
59  ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
60  ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
61  MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
62  MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
63 
64  auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
65  setOperationAction(Opc, FromTy, Promote);
66  AddPromotedToType(Opc, FromTy, ToTy);
67  };
68 
71 
72  for (MVT T : LegalV) {
75 
83  if (T != ByteV) {
87  }
88 
95  // Make concat-vectors custom to handle concats of more than 2 vectors.
104  if (T != ByteV) {
106  // HVX only has shifts of words and halfwords.
110 
111  // Promote all shuffles to operate on vectors of bytes.
112  setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
113  }
114 
122  }
123 
124  for (MVT T : LegalW) {
125  // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
126  // independent) handling of it would convert it to a load, which is
127  // not always the optimal choice.
129  // Make concat-vectors custom to handle concats of more than 2 vectors.
131 
132  // Custom-lower these operations for pairs. Expand them into a concat
133  // of the corresponding operations on individual vectors.
141 
147 
158  if (T != ByteW) {
162 
163  // Promote all shuffles to operate on vectors of bytes.
164  setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
165  }
166  }
167 
168  // Boolean vectors.
169 
170  for (MVT T : LegalW) {
171  // Boolean types for vector pairs will overlap with the boolean
172  // types for single vectors, e.g.
173  // v64i8 -> v64i1 (single)
174  // v64i16 -> v64i1 (pair)
175  // Set these actions first, and allow the single actions to overwrite
176  // any duplicates.
177  MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
182  }
183 
184  for (MVT T : LegalV) {
185  MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
195  }
196 }
197 
198 SDValue
199 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
200  const SDLoc &dl, SelectionDAG &DAG) const {
201  SmallVector<SDValue,4> IntOps;
202  IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
203  for (const SDValue &Op : Ops)
204  IntOps.push_back(Op);
205  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
206 }
207 
208 MVT
209 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
210  assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
211 
212  MVT ElemTy = Tys.first.getVectorElementType();
213  return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
214  Tys.second.getVectorNumElements());
215 }
216 
217 HexagonTargetLowering::TypePair
218 HexagonTargetLowering::typeSplit(MVT VecTy) const {
219  assert(VecTy.isVector());
220  unsigned NumElem = VecTy.getVectorNumElements();
221  assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
222  MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
223  return { HalfTy, HalfTy };
224 }
225 
226 MVT
227 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
228  MVT ElemTy = VecTy.getVectorElementType();
229  MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
230  return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
231 }
232 
233 MVT
234 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
235  MVT ElemTy = VecTy.getVectorElementType();
236  MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
237  return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
238 }
239 
240 SDValue
241 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
242  SelectionDAG &DAG) const {
243  if (ty(Vec).getVectorElementType() == ElemTy)
244  return Vec;
245  MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
246  return DAG.getBitcast(CastTy, Vec);
247 }
248 
249 SDValue
250 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
251  SelectionDAG &DAG) const {
252  return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
253  Ops.second, Ops.first);
254 }
255 
256 HexagonTargetLowering::VectorPair
257 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
258  SelectionDAG &DAG) const {
259  TypePair Tys = typeSplit(ty(Vec));
260  if (Vec.getOpcode() == HexagonISD::QCAT)
261  return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
262  return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
263 }
264 
265 bool
266 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
267  return Subtarget.isHVXVectorType(Ty) &&
268  Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
269 }
270 
271 bool
272 HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
273  return Subtarget.isHVXVectorType(Ty) &&
274  Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
275 }
276 
277 SDValue
278 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
279  SelectionDAG &DAG) const {
280  if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
281  ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
282 
283  unsigned ElemWidth = ElemTy.getSizeInBits();
284  if (ElemWidth == 8)
285  return ElemIdx;
286 
287  unsigned L = Log2_32(ElemWidth/8);
288  const SDLoc &dl(ElemIdx);
289  return DAG.getNode(ISD::SHL, dl, MVT::i32,
290  {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
291 }
292 
293 SDValue
294 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
295  SelectionDAG &DAG) const {
296  unsigned ElemWidth = ElemTy.getSizeInBits();
297  assert(ElemWidth >= 8 && ElemWidth <= 32);
298  if (ElemWidth == 32)
299  return Idx;
300 
301  if (ty(Idx) != MVT::i32)
302  Idx = DAG.getBitcast(MVT::i32, Idx);
303  const SDLoc &dl(Idx);
304  SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
305  SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
306  return SubIdx;
307 }
308 
309 SDValue
310 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
312  SelectionDAG &DAG) const {
313  MVT OpTy = ty(Op0);
314  assert(OpTy == ty(Op1));
315 
316  MVT ElemTy = OpTy.getVectorElementType();
317  if (ElemTy == MVT::i8)
318  return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
319  assert(ElemTy.getSizeInBits() >= 8);
320 
321  MVT ResTy = tyVector(OpTy, MVT::i8);
322  unsigned ElemSize = ElemTy.getSizeInBits() / 8;
323 
324  SmallVector<int,128> ByteMask;
325  for (int M : Mask) {
326  if (M < 0) {
327  for (unsigned I = 0; I != ElemSize; ++I)
328  ByteMask.push_back(-1);
329  } else {
330  int NewM = M*ElemSize;
331  for (unsigned I = 0; I != ElemSize; ++I)
332  ByteMask.push_back(NewM+I);
333  }
334  }
335  assert(ResTy.getVectorNumElements() == ByteMask.size());
336  return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
337  opCastElem(Op1, MVT::i8, DAG), ByteMask);
338 }
339 
340 SDValue
341 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
342  const SDLoc &dl, MVT VecTy,
343  SelectionDAG &DAG) const {
344  unsigned VecLen = Values.size();
346  MVT ElemTy = VecTy.getVectorElementType();
347  unsigned ElemWidth = ElemTy.getSizeInBits();
348  unsigned HwLen = Subtarget.getVectorLength();
349 
350  unsigned ElemSize = ElemWidth / 8;
351  assert(ElemSize*VecLen == HwLen);
353 
354  if (VecTy.getVectorElementType() != MVT::i32) {
355  assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
356  unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
357  MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
358  for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
359  SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
360  Words.push_back(DAG.getBitcast(MVT::i32, W));
361  }
362  } else {
363  Words.assign(Values.begin(), Values.end());
364  }
365 
366  unsigned NumWords = Words.size();
367  bool IsSplat = true, IsUndef = true;
368  SDValue SplatV;
369  for (unsigned i = 0; i != NumWords && IsSplat; ++i) {
370  if (isUndef(Words[i]))
371  continue;
372  IsUndef = false;
373  if (!SplatV.getNode())
374  SplatV = Words[i];
375  else if (SplatV != Words[i])
376  IsSplat = false;
377  }
378  if (IsUndef)
379  return DAG.getUNDEF(VecTy);
380  if (IsSplat) {
381  assert(SplatV.getNode());
382  auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
383  if (IdxN && IdxN->isNullValue())
384  return getZero(dl, VecTy, DAG);
385  return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV);
386  }
387 
388  // Delay recognizing constant vectors until here, so that we can generate
389  // a vsplat.
390  SmallVector<ConstantInt*, 128> Consts(VecLen);
391  bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
392  if (AllConst) {
393  ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
394  (Constant**)Consts.end());
395  Constant *CV = ConstantVector::get(Tmp);
396  unsigned Align = HwLen;
397  SDValue CP = LowerConstantPool(DAG.getConstantPool(CV, VecTy, Align), DAG);
398  return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
400  }
401 
402  // A special case is a situation where the vector is built entirely from
403  // elements extracted from another vector. This could be done via a shuffle
404  // more efficiently, but typically, the size of the source vector will not
405  // match the size of the vector being built (which precludes the use of a
406  // shuffle directly).
407  // This only handles a single source vector, and the vector being built
408  // should be of a sub-vector type of the source vector type.
409  auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
410  SmallVectorImpl<int> &SrcIdx) {
411  SDValue Vec;
412  for (SDValue V : Values) {
413  if (isUndef(V)) {
414  SrcIdx.push_back(-1);
415  continue;
416  }
417  if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
418  return false;
419  // All extracts should come from the same vector.
420  SDValue T = V.getOperand(0);
421  if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
422  return false;
423  Vec = T;
424  ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
425  if (C == nullptr)
426  return false;
427  int I = C->getSExtValue();
428  assert(I >= 0 && "Negative element index");
429  SrcIdx.push_back(I);
430  }
431  SrcVec = Vec;
432  return true;
433  };
434 
435  SmallVector<int,128> ExtIdx;
436  SDValue ExtVec;
437  if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
438  MVT ExtTy = ty(ExtVec);
439  unsigned ExtLen = ExtTy.getVectorNumElements();
440  if (ExtLen == VecLen || ExtLen == 2*VecLen) {
441  // Construct a new shuffle mask that will produce a vector with the same
442  // number of elements as the input vector, and such that the vector we
443  // want will be the initial subvector of it.
445  BitVector Used(ExtLen);
446 
447  for (int M : ExtIdx) {
448  Mask.push_back(M);
449  if (M >= 0)
450  Used.set(M);
451  }
452  // Fill the rest of the mask with the unused elements of ExtVec in hopes
453  // that it will result in a permutation of ExtVec's elements. It's still
454  // fine if it doesn't (e.g. if undefs are present, or elements are
455  // repeated), but permutations can always be done efficiently via vdelta
456  // and vrdelta.
457  for (unsigned I = 0; I != ExtLen; ++I) {
458  if (Mask.size() == ExtLen)
459  break;
460  if (!Used.test(I))
461  Mask.push_back(I);
462  }
463 
464  SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
465  DAG.getUNDEF(ExtTy), Mask);
466  if (ExtLen == VecLen)
467  return S;
468  return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, VecTy, S);
469  }
470  }
471 
472  // Construct two halves in parallel, then or them together.
473  assert(4*Words.size() == Subtarget.getVectorLength());
474  SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
475  SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
476  SDValue S = DAG.getConstant(4, dl, MVT::i32);
477  for (unsigned i = 0; i != NumWords/2; ++i) {
478  SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
479  {HalfV0, Words[i]});
480  SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
481  {HalfV1, Words[i+NumWords/2]});
482  HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
483  HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
484  }
485 
486  HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
487  {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
488  SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
489  return DstV;
490 }
491 
492 SDValue
493 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
494  unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
495  MVT PredTy = ty(PredV);
496  unsigned HwLen = Subtarget.getVectorLength();
497  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
498 
499  if (Subtarget.isHVXVectorType(PredTy, true)) {
500  // Move the vector predicate SubV to a vector register, and scale it
501  // down to match the representation (bytes per type element) that VecV
502  // uses. The scaling down will pick every 2nd or 4th (every Scale-th
503  // in general) element and put them at the front of the resulting
504  // vector. This subvector will then be inserted into the Q2V of VecV.
505  // To avoid having an operation that generates an illegal type (short
506  // vector), generate a full size vector.
507  //
508  SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
509  SmallVector<int,128> Mask(HwLen);
510  // Scale = BitBytes(PredV) / Given BitBytes.
511  unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
512  unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
513 
514  for (unsigned i = 0; i != HwLen; ++i) {
515  unsigned Num = i % Scale;
516  unsigned Off = i / Scale;
517  Mask[BlockLen*Num + Off] = i;
518  }
519  SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
520  if (!ZeroFill)
521  return S;
522  // Fill the bytes beyond BlockLen with 0s.
523  MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
524  SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
525  {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
526  SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
527  return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
528  }
529 
530  // Make sure that this is a valid scalar predicate.
531  assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
532 
533  unsigned Bytes = 8 / PredTy.getVectorNumElements();
534  SmallVector<SDValue,4> Words[2];
535  unsigned IdxW = 0;
536 
537  auto Lo32 = [&DAG, &dl] (SDValue P) {
538  return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P);
539  };
540  auto Hi32 = [&DAG, &dl] (SDValue P) {
541  return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P);
542  };
543 
544  SDValue W0 = isUndef(PredV)
545  ? DAG.getUNDEF(MVT::i64)
546  : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
547  Words[IdxW].push_back(Hi32(W0));
548  Words[IdxW].push_back(Lo32(W0));
549 
550  while (Bytes < BitBytes) {
551  IdxW ^= 1;
552  Words[IdxW].clear();
553 
554  if (Bytes < 4) {
555  for (const SDValue &W : Words[IdxW ^ 1]) {
556  SDValue T = expandPredicate(W, dl, DAG);
557  Words[IdxW].push_back(Hi32(T));
558  Words[IdxW].push_back(Lo32(T));
559  }
560  } else {
561  for (const SDValue &W : Words[IdxW ^ 1]) {
562  Words[IdxW].push_back(W);
563  Words[IdxW].push_back(W);
564  }
565  }
566  Bytes *= 2;
567  }
568 
569  assert(Bytes == BitBytes);
570 
571  SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
572  SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
573  for (const SDValue &W : Words[IdxW]) {
574  Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
575  Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
576  }
577 
578  return Vec;
579 }
580 
581 SDValue
582 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
583  const SDLoc &dl, MVT VecTy,
584  SelectionDAG &DAG) const {
585  // Construct a vector V of bytes, such that a comparison V >u 0 would
586  // produce the required vector predicate.
587  unsigned VecLen = Values.size();
588  unsigned HwLen = Subtarget.getVectorLength();
589  assert(VecLen <= HwLen || VecLen == 8*HwLen);
591  bool AllT = true, AllF = true;
592 
593  auto IsTrue = [] (SDValue V) {
594  if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
595  return !N->isNullValue();
596  return false;
597  };
598  auto IsFalse = [] (SDValue V) {
599  if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
600  return N->isNullValue();
601  return false;
602  };
603 
604  if (VecLen <= HwLen) {
605  // In the hardware, each bit of a vector predicate corresponds to a byte
606  // of a vector register. Calculate how many bytes does a bit of VecTy
607  // correspond to.
608  assert(HwLen % VecLen == 0);
609  unsigned BitBytes = HwLen / VecLen;
610  for (SDValue V : Values) {
611  AllT &= IsTrue(V);
612  AllF &= IsFalse(V);
613 
614  SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
615  : DAG.getUNDEF(MVT::i8);
616  for (unsigned B = 0; B != BitBytes; ++B)
617  Bytes.push_back(Ext);
618  }
619  } else {
620  // There are as many i1 values, as there are bits in a vector register.
621  // Divide the values into groups of 8 and check that each group consists
622  // of the same value (ignoring undefs).
623  for (unsigned I = 0; I != VecLen; I += 8) {
624  unsigned B = 0;
625  // Find the first non-undef value in this group.
626  for (; B != 8; ++B) {
627  if (!Values[I+B].isUndef())
628  break;
629  }
630  SDValue F = Values[I+B];
631  AllT &= IsTrue(F);
632  AllF &= IsFalse(F);
633 
634  SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
635  : DAG.getUNDEF(MVT::i8);
636  Bytes.push_back(Ext);
637  // Verify that the rest of values in the group are the same as the
638  // first.
639  for (; B != 8; ++B)
640  assert(Values[I+B].isUndef() || Values[I+B] == F);
641  }
642  }
643 
644  if (AllT)
645  return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
646  if (AllF)
647  return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
648 
649  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
650  SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
651  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
652 }
653 
654 SDValue
655 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
656  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
657  MVT ElemTy = ty(VecV).getVectorElementType();
658 
659  unsigned ElemWidth = ElemTy.getSizeInBits();
660  assert(ElemWidth >= 8 && ElemWidth <= 32);
661  (void)ElemWidth;
662 
663  SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
664  SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
665  {VecV, ByteIdx});
666  if (ElemTy == MVT::i32)
667  return ExWord;
668 
669  // Have an extracted word, need to extract the smaller element out of it.
670  // 1. Extract the bits of (the original) IdxV that correspond to the index
671  // of the desired element in the 32-bit word.
672  SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
673  // 2. Extract the element from the word.
674  SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
675  return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
676 }
677 
678 SDValue
679 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
680  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
681  // Implement other return types if necessary.
682  assert(ResTy == MVT::i1);
683 
684  unsigned HwLen = Subtarget.getVectorLength();
685  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
686  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
687 
688  unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
689  SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
690  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
691 
692  SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
693  SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
694  return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
695 }
696 
697 SDValue
698 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
699  SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
700  MVT ElemTy = ty(VecV).getVectorElementType();
701 
702  unsigned ElemWidth = ElemTy.getSizeInBits();
703  assert(ElemWidth >= 8 && ElemWidth <= 32);
704  (void)ElemWidth;
705 
706  auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
707  SDValue ByteIdxV) {
708  MVT VecTy = ty(VecV);
709  unsigned HwLen = Subtarget.getVectorLength();
710  SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
711  {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
712  SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
713  SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
714  SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
715  {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
716  SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
717  return TorV;
718  };
719 
720  SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
721  if (ElemTy == MVT::i32)
722  return InsertWord(VecV, ValV, ByteIdx);
723 
724  // If this is not inserting a 32-bit word, convert it into such a thing.
725  // 1. Extract the existing word from the target vector.
726  SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
727  {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
728  SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
729  dl, MVT::i32, DAG);
730 
731  // 2. Treating the extracted word as a 32-bit vector, insert the given
732  // value into it.
733  SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
734  MVT SubVecTy = tyVector(ty(Ext), ElemTy);
735  SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
736  ValV, SubIdx, dl, ElemTy, DAG);
737 
738  // 3. Insert the 32-bit word back into the original vector.
739  return InsertWord(VecV, Ins, ByteIdx);
740 }
741 
742 SDValue
743 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
744  SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
745  unsigned HwLen = Subtarget.getVectorLength();
746  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
747  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
748 
749  unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
750  SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
751  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
752  ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
753 
754  SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
755  return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
756 }
757 
758 SDValue
759 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
760  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
761  MVT VecTy = ty(VecV);
762  unsigned HwLen = Subtarget.getVectorLength();
763  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
764  MVT ElemTy = VecTy.getVectorElementType();
765  unsigned ElemWidth = ElemTy.getSizeInBits();
766 
767  // If the source vector is a vector pair, get the single vector containing
768  // the subvector of interest. The subvector will never overlap two single
769  // vectors.
770  if (isHvxPairTy(VecTy)) {
771  unsigned SubIdx;
772  if (Idx * ElemWidth >= 8*HwLen) {
773  SubIdx = Hexagon::vsub_hi;
774  Idx -= VecTy.getVectorNumElements() / 2;
775  } else {
776  SubIdx = Hexagon::vsub_lo;
777  }
778  VecTy = typeSplit(VecTy).first;
779  VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV);
780  if (VecTy == ResTy)
781  return VecV;
782  }
783 
784  // The only meaningful subvectors of a single HVX vector are those that
785  // fit in a scalar register.
786  assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
787 
788  MVT WordTy = tyVector(VecTy, MVT::i32);
789  SDValue WordVec = DAG.getBitcast(WordTy, VecV);
790  unsigned WordIdx = (Idx*ElemWidth) / 32;
791 
792  SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
793  SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
794  if (ResTy.getSizeInBits() == 32)
795  return DAG.getBitcast(ResTy, W0);
796 
797  SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
798  SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
799  SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
800  return DAG.getBitcast(ResTy, WW);
801 }
802 
803 SDValue
804 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
805  const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
806  MVT VecTy = ty(VecV);
807  unsigned HwLen = Subtarget.getVectorLength();
808  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
809  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
810  // IdxV is required to be a constant.
811  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
812 
813  unsigned ResLen = ResTy.getVectorNumElements();
814  unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
815  unsigned Offset = Idx * BitBytes;
816  SDValue Undef = DAG.getUNDEF(ByteTy);
818 
819  if (Subtarget.isHVXVectorType(ResTy, true)) {
820  // Converting between two vector predicates. Since the result is shorter
821  // than the source, it will correspond to a vector predicate with the
822  // relevant bits replicated. The replication count is the ratio of the
823  // source and target vector lengths.
824  unsigned Rep = VecTy.getVectorNumElements() / ResLen;
825  assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
826  for (unsigned i = 0; i != HwLen/Rep; ++i) {
827  for (unsigned j = 0; j != Rep; ++j)
828  Mask.push_back(i + Offset);
829  }
830  SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
831  return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
832  }
833 
834  // Converting between a vector predicate and a scalar predicate. In the
835  // vector predicate, a group of BitBytes bits will correspond to a single
836  // i1 element of the source vector type. Those bits will all have the same
837  // value. The same will be true for ByteVec, where each byte corresponds
838  // to a bit in the vector predicate.
839  // The algorithm is to traverse the ByteVec, going over the i1 values from
840  // the source vector, and generate the corresponding representation in an
841  // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
842  // elements so that the interesting 8 bytes will be in the low end of the
843  // vector.
844  unsigned Rep = 8 / ResLen;
845  // Make sure the output fill the entire vector register, so repeat the
846  // 8-byte groups as many times as necessary.
847  for (unsigned r = 0; r != HwLen/ResLen; ++r) {
848  // This will generate the indexes of the 8 interesting bytes.
849  for (unsigned i = 0; i != ResLen; ++i) {
850  for (unsigned j = 0; j != Rep; ++j)
851  Mask.push_back(Offset + i*BitBytes);
852  }
853  }
854 
855  SDValue Zero = getZero(dl, MVT::i32, DAG);
856  SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
857  // Combine the two low words from ShuffV into a v8i8, and byte-compare
858  // them against 0.
859  SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
861  {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
862  SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
863  return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
864  {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
865 }
866 
867 SDValue
868 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
869  SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
870  MVT VecTy = ty(VecV);
871  MVT SubTy = ty(SubV);
872  unsigned HwLen = Subtarget.getVectorLength();
873  MVT ElemTy = VecTy.getVectorElementType();
874  unsigned ElemWidth = ElemTy.getSizeInBits();
875 
876  bool IsPair = isHvxPairTy(VecTy);
877  MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
878  // The two single vectors that VecV consists of, if it's a pair.
879  SDValue V0, V1;
880  SDValue SingleV = VecV;
881  SDValue PickHi;
882 
883  if (IsPair) {
884  V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV);
885  V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV);
886 
887  SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
888  dl, MVT::i32);
889  PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
890  if (isHvxSingleTy(SubTy)) {
891  if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
892  unsigned Idx = CN->getZExtValue();
893  assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
894  unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
895  return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
896  }
897  // If IdxV is not a constant, generate the two variants: with the
898  // SubV as the high and as the low subregister, and select the right
899  // pair based on the IdxV.
900  SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
901  SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
902  return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
903  }
904  // The subvector being inserted must be entirely contained in one of
905  // the vectors V0 or V1. Set SingleV to the correct one, and update
906  // IdxV to be the index relative to the beginning of that vector.
907  SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
908  IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
909  SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
910  }
911 
912  // The only meaningful subvectors of a single HVX vector are those that
913  // fit in a scalar register.
914  assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
915  // Convert IdxV to be index in bytes.
916  auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
917  if (!IdxN || !IdxN->isNullValue()) {
918  IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
919  DAG.getConstant(ElemWidth/8, dl, MVT::i32));
920  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
921  }
922  // When inserting a single word, the rotation back to the original position
923  // would be by HwLen-Idx, but if two words are inserted, it will need to be
924  // by (HwLen-4)-Idx.
925  unsigned RolBase = HwLen;
926  if (VecTy.getSizeInBits() == 32) {
927  SDValue V = DAG.getBitcast(MVT::i32, SubV);
928  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V);
929  } else {
930  SDValue V = DAG.getBitcast(MVT::i64, SubV);
931  SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
932  SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
933  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
934  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
935  DAG.getConstant(4, dl, MVT::i32));
936  SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
937  RolBase = HwLen-4;
938  }
939  // If the vector wasn't ror'ed, don't ror it back.
940  if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) {
941  SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
942  DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
943  SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
944  }
945 
946  if (IsPair) {
947  SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
948  SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
949  return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
950  }
951  return SingleV;
952 }
953 
954 SDValue
955 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
956  SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
957  MVT VecTy = ty(VecV);
958  MVT SubTy = ty(SubV);
959  assert(Subtarget.isHVXVectorType(VecTy, true));
960  // VecV is an HVX vector predicate. SubV may be either an HVX vector
961  // predicate as well, or it can be a scalar predicate.
962 
963  unsigned VecLen = VecTy.getVectorNumElements();
964  unsigned HwLen = Subtarget.getVectorLength();
965  assert(HwLen % VecLen == 0 && "Unexpected vector type");
966 
967  unsigned Scale = VecLen / SubTy.getVectorNumElements();
968  unsigned BitBytes = HwLen / VecLen;
969  unsigned BlockLen = HwLen / Scale;
970 
971  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
972  SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
973  SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
974  SDValue ByteIdx;
975 
976  auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
977  if (!IdxN || !IdxN->isNullValue()) {
978  ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
979  DAG.getConstant(BitBytes, dl, MVT::i32));
980  ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
981  }
982 
983  // ByteVec is the target vector VecV rotated in such a way that the
984  // subvector should be inserted at index 0. Generate a predicate mask
985  // and use vmux to do the insertion.
986  MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
987  SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
988  {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
989  ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
990  // Rotate ByteVec back, and convert to a vector predicate.
991  if (!IdxN || !IdxN->isNullValue()) {
992  SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
993  SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
994  ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
995  }
996  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
997 }
998 
999 SDValue
1000 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1001  MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1002  // Sign- and any-extending of a vector predicate to a vector register is
1003  // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1004  // a vector of 1s (where the 1s are of type matching the vector type).
1005  assert(Subtarget.isHVXVectorType(ResTy));
1006  if (!ZeroExt)
1007  return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
1008 
1009  assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1010  SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
1011  DAG.getConstant(1, dl, MVT::i32));
1012  SDValue False = getZero(dl, ResTy, DAG);
1013  return DAG.getSelect(dl, ResTy, VecV, True, False);
1014 }
1015 
1016 SDValue
1017 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1018  const {
1019  const SDLoc &dl(Op);
1020  MVT VecTy = ty(Op);
1021 
1022  unsigned Size = Op.getNumOperands();
1024  for (unsigned i = 0; i != Size; ++i)
1025  Ops.push_back(Op.getOperand(i));
1026 
1027  if (VecTy.getVectorElementType() == MVT::i1)
1028  return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1029 
1030  if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
1031  ArrayRef<SDValue> A(Ops);
1032  MVT SingleTy = typeSplit(VecTy).first;
1033  SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
1034  SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
1035  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
1036  }
1037 
1038  return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1039 }
1040 
1041 SDValue
1042 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1043  const {
1044  // Vector concatenation of two integer (non-bool) vectors does not need
1045  // special lowering. Custom-lower concats of bool vectors and expand
1046  // concats of more than 2 vectors.
1047  MVT VecTy = ty(Op);
1048  const SDLoc &dl(Op);
1049  unsigned NumOp = Op.getNumOperands();
1050  if (VecTy.getVectorElementType() != MVT::i1) {
1051  if (NumOp == 2)
1052  return Op;
1053  // Expand the other cases into a build-vector.
1054  SmallVector<SDValue,8> Elems;
1055  for (SDValue V : Op.getNode()->ops())
1056  DAG.ExtractVectorElements(V, Elems);
1057  // A vector of i16 will be broken up into a build_vector of i16's.
1058  // This is a problem, since at the time of operation legalization,
1059  // all operations are expected to be type-legalized, and i16 is not
1060  // a legal type. If any of the extracted elements is not of a valid
1061  // type, sign-extend it to a valid one.
1062  for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
1063  SDValue V = Elems[i];
1064  MVT Ty = ty(V);
1065  if (!isTypeLegal(Ty)) {
1066  EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty);
1067  if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1068  Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1069  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1070  V.getOperand(0), V.getOperand(1)),
1071  DAG.getValueType(Ty));
1072  continue;
1073  }
1074  // A few less complicated cases.
1075  if (V.getOpcode() == ISD::Constant)
1076  Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1077  else if (V.isUndef())
1078  Elems[i] = DAG.getUNDEF(NTy);
1079  else
1080  llvm_unreachable("Unexpected vector element");
1081  }
1082  }
1083  return DAG.getBuildVector(VecTy, dl, Elems);
1084  }
1085 
1086  assert(VecTy.getVectorElementType() == MVT::i1);
1087  unsigned HwLen = Subtarget.getVectorLength();
1088  assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1089 
1090  SDValue Op0 = Op.getOperand(0);
1091 
1092  // If the operands are HVX types (i.e. not scalar predicates), then
1093  // defer the concatenation, and create QCAT instead.
1094  if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1095  if (NumOp == 2)
1096  return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1097 
1098  ArrayRef<SDUse> U(Op.getNode()->ops());
1099  SmallVector<SDValue,4> SV(U.begin(), U.end());
1100  ArrayRef<SDValue> Ops(SV);
1101 
1102  MVT HalfTy = typeSplit(VecTy).first;
1103  SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1104  Ops.take_front(NumOp/2));
1105  SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1106  Ops.take_back(NumOp/2));
1107  return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1108  }
1109 
1110  // Count how many bytes (in a vector register) each bit in VecTy
1111  // corresponds to.
1112  unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1113 
1114  SmallVector<SDValue,8> Prefixes;
1115  for (SDValue V : Op.getNode()->op_values()) {
1116  SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1117  Prefixes.push_back(P);
1118  }
1119 
1120  unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1121  MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1122  SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1123  SDValue Res = getZero(dl, ByteTy, DAG);
1124  for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1125  Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1126  Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1127  }
1128  return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1129 }
1130 
1131 SDValue
1132 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1133  const {
1134  // Change the type of the extracted element to i32.
1135  SDValue VecV = Op.getOperand(0);
1136  MVT ElemTy = ty(VecV).getVectorElementType();
1137  const SDLoc &dl(Op);
1138  SDValue IdxV = Op.getOperand(1);
1139  if (ElemTy == MVT::i1)
1140  return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1141 
1142  return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1143 }
1144 
1145 SDValue
1146 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1147  const {
1148  const SDLoc &dl(Op);
1149  SDValue VecV = Op.getOperand(0);
1150  SDValue ValV = Op.getOperand(1);
1151  SDValue IdxV = Op.getOperand(2);
1152  MVT ElemTy = ty(VecV).getVectorElementType();
1153  if (ElemTy == MVT::i1)
1154  return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1155 
1156  return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1157 }
1158 
1159 SDValue
1160 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1161  const {
1162  SDValue SrcV = Op.getOperand(0);
1163  MVT SrcTy = ty(SrcV);
1164  MVT DstTy = ty(Op);
1165  SDValue IdxV = Op.getOperand(1);
1166  unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
1167  assert(Idx % DstTy.getVectorNumElements() == 0);
1168  (void)Idx;
1169  const SDLoc &dl(Op);
1170 
1171  MVT ElemTy = SrcTy.getVectorElementType();
1172  if (ElemTy == MVT::i1)
1173  return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1174 
1175  return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1176 }
1177 
1178 SDValue
1179 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1180  const {
1181  // Idx does not need to be a constant.
1182  SDValue VecV = Op.getOperand(0);
1183  SDValue ValV = Op.getOperand(1);
1184  SDValue IdxV = Op.getOperand(2);
1185 
1186  const SDLoc &dl(Op);
1187  MVT VecTy = ty(VecV);
1188  MVT ElemTy = VecTy.getVectorElementType();
1189  if (ElemTy == MVT::i1)
1190  return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1191 
1192  return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1193 }
1194 
1195 SDValue
1196 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1197  // Lower any-extends of boolean vectors to sign-extends, since they
1198  // translate directly to Q2V. Zero-extending could also be done equally
1199  // fast, but Q2V is used/recognized in more places.
1200  // For all other vectors, use zero-extend.
1201  MVT ResTy = ty(Op);
1202  SDValue InpV = Op.getOperand(0);
1203  MVT ElemTy = ty(InpV).getVectorElementType();
1204  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1205  return LowerHvxSignExt(Op, DAG);
1206  return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
1207 }
1208 
1209 SDValue
1210 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1211  MVT ResTy = ty(Op);
1212  SDValue InpV = Op.getOperand(0);
1213  MVT ElemTy = ty(InpV).getVectorElementType();
1214  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1215  return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1216  return Op;
1217 }
1218 
1219 SDValue
1220 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1221  MVT ResTy = ty(Op);
1222  SDValue InpV = Op.getOperand(0);
1223  MVT ElemTy = ty(InpV).getVectorElementType();
1224  if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1225  return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1226  return Op;
1227 }
1228 
1229 SDValue
1230 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1231  // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
1232  // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1233  const SDLoc &dl(Op);
1234  MVT ResTy = ty(Op);
1235  SDValue InpV = Op.getOperand(0);
1236  assert(ResTy == ty(InpV));
1237 
1238  // Calculate the vectors of 1 and bitwidth(x).
1239  MVT ElemTy = ty(InpV).getVectorElementType();
1240  unsigned ElemWidth = ElemTy.getSizeInBits();
1241  // Using uint64_t because a shift by 32 can happen.
1242  uint64_t Splat1 = 0, SplatW = 0;
1243  assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32);
1244  for (unsigned i = 0; i != 32/ElemWidth; ++i) {
1245  Splat1 = (Splat1 << ElemWidth) | 1;
1246  SplatW = (SplatW << ElemWidth) | ElemWidth;
1247  }
1248  SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1249  DAG.getConstant(uint32_t(Splat1), dl, MVT::i32));
1250  SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1251  DAG.getConstant(uint32_t(SplatW), dl, MVT::i32));
1252  SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1253  DAG.getConstant(-1, dl, MVT::i32));
1254  // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
1255  // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
1256  // it separately in custom combine or selection).
1257  SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1258  {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1259  DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1260  return DAG.getNode(ISD::SUB, dl, ResTy,
1261  {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1262 }
1263 
1264 SDValue
1265 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1266  MVT ResTy = ty(Op);
1267  assert(ResTy.isVector() && isHvxSingleTy(ResTy));
1268  const SDLoc &dl(Op);
1269  SmallVector<int,256> ShuffMask;
1270 
1271  MVT ElemTy = ResTy.getVectorElementType();
1272  unsigned VecLen = ResTy.getVectorNumElements();
1273  SDValue Vs = Op.getOperand(0);
1274  SDValue Vt = Op.getOperand(1);
1275 
1276  switch (ElemTy.SimpleTy) {
1277  case MVT::i8: {
1278  // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1279  // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1280  // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1281  MVT ExtTy = typeExtElem(ResTy, 2);
1282  unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
1283  : Hexagon::V6_vmpyhv;
1284  SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1285 
1286  // Discard high halves of the resulting values, collect the low halves.
1287  for (unsigned I = 0; I < VecLen; I += 2) {
1288  ShuffMask.push_back(I); // Pick even element.
1289  ShuffMask.push_back(I+VecLen); // Pick odd element.
1290  }
1291  VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1292  SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1293  return DAG.getBitcast(ResTy, BS);
1294  }
1295  case MVT::i16:
1296  // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode.
1297  // (There is also V6_vmpyhv, which behaves in an analogous way to
1298  // V6_vmpybv.)
1299  return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG);
1300  case MVT::i32: {
1301  // Use the following sequence for signed word multiply:
1302  // T0 = V6_vmpyiowh Vs, Vt
1303  // T1 = V6_vaslw T0, 16
1304  // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
1305  SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1306  SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1307  SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1308  SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1309  {T1, Vs, Vt}, DAG);
1310  return T2;
1311  }
1312  default:
1313  break;
1314  }
1315  return SDValue();
1316 }
1317 
1318 SDValue
1319 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1320  MVT ResTy = ty(Op);
1321  assert(ResTy.isVector());
1322  const SDLoc &dl(Op);
1323  SmallVector<int,256> ShuffMask;
1324 
1325  MVT ElemTy = ResTy.getVectorElementType();
1326  unsigned VecLen = ResTy.getVectorNumElements();
1327  SDValue Vs = Op.getOperand(0);
1328  SDValue Vt = Op.getOperand(1);
1329  bool IsSigned = Op.getOpcode() == ISD::MULHS;
1330 
1331  if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
1332  // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1333  // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1334  // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1335  // For i16, use V6_vmpyhv, which behaves in an analogous way to
1336  // V6_vmpybv: results Lo and Hi are products of even/odd elements
1337  // respectively.
1338  MVT ExtTy = typeExtElem(ResTy, 2);
1339  unsigned MpyOpc = ElemTy == MVT::i8
1340  ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1341  : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1342  SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1343 
1344  // Discard low halves of the resulting values, collect the high halves.
1345  for (unsigned I = 0; I < VecLen; I += 2) {
1346  ShuffMask.push_back(I+1); // Pick even element.
1347  ShuffMask.push_back(I+VecLen+1); // Pick odd element.
1348  }
1349  VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1350  SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1351  return DAG.getBitcast(ResTy, BS);
1352  }
1353 
1354  assert(ElemTy == MVT::i32);
1355  SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1356 
1357  if (IsSigned) {
1358  // mulhs(Vs,Vt) =
1359  // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1360  // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
1361  // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1362  // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
1363  // + Lo(Vs) *us Vt] >> 32
1364  // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
1365  // anything, so it cannot produce any carry over to higher bits),
1366  // so everything in [] can be shifted by 16 without loss of precision.
1367  // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
1368  // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
1369  // Denote Hi(Vs) = Vs':
1370  // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
1371  // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
1372  SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1373  // Get Vs':
1374  SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1375  SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1376  {T0, S0, Vt}, DAG);
1377  // Shift by 16:
1378  SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1379  // Get Vs'*Hi(Vt):
1380  SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1381  // Add:
1382  SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1383  return T3;
1384  }
1385 
1386  // Unsigned mulhw. (Would expansion using signed mulhw be better?)
1387 
1388  auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
1389  return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
1390  };
1391  auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
1392  return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
1393  };
1394 
1395  MVT PairTy = typeJoin({ResTy, ResTy});
1396  SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1397  {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
1398  // Multiply-unsigned halfwords:
1399  // LoVec = Vs.uh[2i] * Vt.uh[2i],
1400  // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
1401  SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1402  // The low halves in the LoVec of the pair can be discarded. They are
1403  // not added to anything (in the full-precision product), so they cannot
1404  // produce a carry into the higher bits.
1405  SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1406  // Swap low and high halves in Vt, and do the halfword multiplication
1407  // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
1408  SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1409  SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1410  // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
1411  // These products are words, but cannot be added directly because the
1412  // sums could overflow. Add these products, by halfwords, where each sum
1413  // of a pair of halfwords gives a word.
1414  SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1415  {LoVec(T2), HiVec(T2)}, DAG);
1416  // Add the high halfwords from the products of the low halfwords.
1417  SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1418  SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1419  SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1420  SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1421  return T7;
1422 }
1423 
1424 SDValue
1425 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1426  // Sign- and zero-extends are legal.
1428  return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op),
1429  Op.getOperand(0));
1430 }
1431 
1432 SDValue
1433 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1434  if (SDValue S = getVectorShiftByInt(Op, DAG))
1435  return S;
1436  return Op;
1437 }
1438 
1439 SDValue
1440 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1441  assert(!Op.isMachineOpcode());
1442  SmallVector<SDValue,2> OpsL, OpsH;
1443  const SDLoc &dl(Op);
1444 
1445  auto SplitVTNode = [&DAG,this] (const VTSDNode *N) {
1446  MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
1447  SDValue TV = DAG.getValueType(Ty);
1448  return std::make_pair(TV, TV);
1449  };
1450 
1451  for (SDValue A : Op.getNode()->ops()) {
1452  VectorPair P = Subtarget.isHVXVectorType(ty(A), true)
1453  ? opSplit(A, dl, DAG)
1454  : std::make_pair(A, A);
1455  // Special case for type operand.
1456  if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1457  if (const auto *N = dyn_cast<const VTSDNode>(A.getNode()))
1458  P = SplitVTNode(N);
1459  }
1460  OpsL.push_back(P.first);
1461  OpsH.push_back(P.second);
1462  }
1463 
1464  MVT ResTy = ty(Op);
1465  MVT HalfTy = typeSplit(ResTy).first;
1466  SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1467  SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1468  SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1469  return S;
1470 }
1471 
1472 SDValue
1473 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1474  LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode());
1475  assert(BN->isUnindexed());
1476  MVT MemTy = BN->getMemoryVT().getSimpleVT();
1477  if (!isHvxPairTy(MemTy))
1478  return Op;
1479 
1480  const SDLoc &dl(Op);
1481  unsigned HwLen = Subtarget.getVectorLength();
1482  MVT SingleTy = typeSplit(MemTy).first;
1483  SDValue Chain = BN->getChain();
1484  SDValue Base0 = BN->getBasePtr();
1485  SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1486 
1487  MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
1488  if (MachineMemOperand *MMO = BN->getMemOperand()) {
1489  MachineFunction &MF = DAG.getMachineFunction();
1490  MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen);
1491  MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen);
1492  }
1493 
1494  unsigned MemOpc = BN->getOpcode();
1495  SDValue NewOp;
1496 
1497  if (MemOpc == ISD::LOAD) {
1498  SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1499  SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1500  NewOp = DAG.getMergeValues(
1501  { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
1503  Load0.getValue(1), Load1.getValue(1)) }, dl);
1504  } else {
1505  assert(MemOpc == ISD::STORE);
1506  VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
1507  SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1508  SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1509  NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
1510  }
1511 
1512  return NewOp;
1513 }
1514 
1515 SDValue
1516 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1517  unsigned Opc = Op.getOpcode();
1518  bool IsPairOp = isHvxPairTy(ty(Op)) ||
1519  llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
1520  return isHvxPairTy(ty(V));
1521  });
1522 
1523  if (IsPairOp) {
1524  switch (Opc) {
1525  default:
1526  break;
1527  case ISD::LOAD:
1528  case ISD::STORE:
1529  return SplitHvxMemOp(Op, DAG);
1530  case ISD::CTPOP:
1531  case ISD::CTLZ:
1532  case ISD::CTTZ:
1533  case ISD::MUL:
1534  case ISD::MULHS:
1535  case ISD::MULHU:
1536  case ISD::AND:
1537  case ISD::OR:
1538  case ISD::XOR:
1539  case ISD::SRA:
1540  case ISD::SHL:
1541  case ISD::SRL:
1542  case ISD::SETCC:
1543  case ISD::VSELECT:
1544  case ISD::SIGN_EXTEND:
1545  case ISD::ZERO_EXTEND:
1547  return SplitHvxPairOp(Op, DAG);
1548  }
1549  }
1550 
1551  switch (Opc) {
1552  default:
1553  break;
1554  case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
1555  case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
1556  case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
1557  case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
1558  case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
1559  case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
1560 
1561  case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
1562  case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
1563  case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
1564  case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
1565  case ISD::SRA:
1566  case ISD::SHL:
1567  case ISD::SRL: return LowerHvxShift(Op, DAG);
1568  case ISD::MUL: return LowerHvxMul(Op, DAG);
1569  case ISD::MULHS:
1570  case ISD::MULHU: return LowerHvxMulh(Op, DAG);
1571  case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
1572  case ISD::SETCC:
1573  case ISD::INTRINSIC_VOID: return Op;
1574  // Unaligned loads will be handled by the default lowering.
1575  case ISD::LOAD: return SDValue();
1576  }
1577 #ifndef NDEBUG
1578  Op.dumpr(&DAG);
1579 #endif
1580  llvm_unreachable("Unhandled HVX operation");
1581 }
1582 
1583 bool
1584 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
1585  // If the type of the result, or any operand type are HVX vector types,
1586  // this is an HVX operation.
1587  return Subtarget.isHVXVectorType(ty(Op), true) ||
1588  llvm::any_of(Op.getNode()->ops(),
1589  [this] (SDValue V) {
1590  return Subtarget.isHVXVectorType(ty(V), true);
1591  });
1592 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
static MVT getIntegerVT(unsigned BitWidth)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
BitVector & set()
Definition: BitVector.h:397
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isUndef() const
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition: ArrayRef.h:211
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:957
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:382
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:391
iterator begin() const
Definition: ArrayRef.h:136
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:543
bool isVector() const
Return true if this is a vector value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
static const MVT LegalW128[]
bool test(unsigned Idx) const
Definition: BitVector.h:501
unsigned getVectorNumElements() const
const SDValue & getChain() const
F(f)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:970
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:532
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
static const MVT LegalV64[]
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:377
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, const SDLoc &DL)
Returns sum of the base pointer and offset.
A description of a memory reference used in the backend.
Shift and rotation operations.
Definition: ISDOpcodes.h:434
Base class for LoadSDNode and StoreSDNode.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn&#39;t supported on the target and indicate what to d...
SimpleValueType SimpleTy
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:467
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
This class is used to represent EVT&#39;s, which are used to parameterize some operations.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
unsigned getSizeInBits() const
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:412
int64_t getSExtValue() const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:404
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:453
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:872
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:586
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:521
ArrayRef< SDUse > ops() const
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:150
MVT getVectorElementType() const
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:351
#define P(N)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:165
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
Machine Value Type.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
bool isMachineOpcode() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
static const MVT LegalW64[]
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:356
#define H(x, y, z)
Definition: MD5.cpp:57
static const MVT LegalV128[]
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1199
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
Extended Value Type.
Definition: ValueTypes.h:33
size_t size() const
Definition: SmallVector.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
unsigned first
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:49
bool isHVXVectorType(MVT VecTy, bool IncludeBool=false) const
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:363
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:744
iterator end() const
Definition: ArrayRef.h:137
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:437
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0)
Append the extracted elements from Start to Count out of the vector Op in Args.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:444
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:492
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:495
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array...
Definition: ArrayRef.h:178
void dumpr() const
ArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
Definition: ArrayRef.h:218
unsigned getVectorLength() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:411
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:510
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:187
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:642
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:369
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:467
unsigned getNumOperands() const
Conversion operators.
Definition: ISDOpcodes.h:489
const SDValue & getOperand(unsigned i) const
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
LLVMContext * getContext() const
Definition: SelectionDAG.h:414
static Constant * get(ArrayRef< Constant *> V)
Definition: Constants.cpp:1088
#define T1
void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:404