LLVM  6.0.0svn
LegalizeVectorOps.cpp
Go to the documentation of this file.
1 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the SelectionDAG::LegalizeVectors method.
11 //
12 // The vector legalizer looks for vector operations which might need to be
13 // scalarized and legalizes them. This is a separate step from Legalize because
14 // scalarizing can introduce illegal types. For example, suppose we have an
15 // ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
16 // on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
17 // operation, which introduces nodes with the illegal type i64 which must be
18 // expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
19 // the operation must be unrolled, which introduces nodes with the illegal
20 // type i8 which must be promoted.
21 //
22 // This does not legalize vector manipulations like ISD::BUILD_VECTOR,
23 // or operations that happen to take a vector which are custom-lowered;
24 // the legalization for such operations never produces nodes
25 // with illegal types, so it's okay to put off legalizing them until
26 // SelectionDAG::Legalize runs.
27 //
28 //===----------------------------------------------------------------------===//
29 
30 #include "llvm/ADT/APInt.h"
31 #include "llvm/ADT/DenseMap.h"
32 #include "llvm/ADT/SmallVector.h"
40 #include "llvm/IR/DataLayout.h"
41 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Compiler.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <utility>
49 
50 using namespace llvm;
51 
52 namespace {
53 
54 class VectorLegalizer {
55  SelectionDAG& DAG;
56  const TargetLowering &TLI;
57  bool Changed = false; // Keep track of whether anything changed
58 
59  /// For nodes that are of legal width, and that have more than one use, this
60  /// map indicates what regularized operand to use. This allows us to avoid
61  /// legalizing the same thing more than once.
63 
64  /// \brief Adds a node to the translation cache.
65  void AddLegalizedOperand(SDValue From, SDValue To) {
66  LegalizedNodes.insert(std::make_pair(From, To));
67  // If someone requests legalization of the new node, return itself.
68  if (From != To)
69  LegalizedNodes.insert(std::make_pair(To, To));
70  }
71 
72  /// \brief Legalizes the given node.
73  SDValue LegalizeOp(SDValue Op);
74 
75  /// \brief Assuming the node is legal, "legalize" the results.
76  SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
77 
78  /// \brief Implements unrolling a VSETCC.
79  SDValue UnrollVSETCC(SDValue Op);
80 
81  /// \brief Implement expand-based legalization of vector operations.
82  ///
83  /// This is just a high-level routine to dispatch to specific code paths for
84  /// operations to legalize them.
85  SDValue Expand(SDValue Op);
86 
87  /// \brief Implements expansion for FNEG; falls back to UnrollVectorOp if
88  /// FSUB isn't legal.
89  ///
90  /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
91  /// SINT_TO_FLOAT and SHR on vectors isn't legal.
92  SDValue ExpandUINT_TO_FLOAT(SDValue Op);
93 
94  /// \brief Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
95  SDValue ExpandSEXTINREG(SDValue Op);
96 
97  /// \brief Implement expansion for ANY_EXTEND_VECTOR_INREG.
98  ///
99  /// Shuffles the low lanes of the operand into place and bitcasts to the proper
100  /// type. The contents of the bits in the extended part of each element are
101  /// undef.
102  SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op);
103 
104  /// \brief Implement expansion for SIGN_EXTEND_VECTOR_INREG.
105  ///
106  /// Shuffles the low lanes of the operand into place, bitcasts to the proper
107  /// type, then shifts left and arithmetic shifts right to introduce a sign
108  /// extension.
109  SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op);
110 
111  /// \brief Implement expansion for ZERO_EXTEND_VECTOR_INREG.
112  ///
113  /// Shuffles the low lanes of the operand into place and blends zeros into
114  /// the remaining lanes, finally bitcasting to the proper type.
115  SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op);
116 
117  /// \brief Expand bswap of vectors into a shuffle if legal.
118  SDValue ExpandBSWAP(SDValue Op);
119 
120  /// \brief Implement vselect in terms of XOR, AND, OR when blend is not
121  /// supported by the target.
122  SDValue ExpandVSELECT(SDValue Op);
123  SDValue ExpandSELECT(SDValue Op);
124  SDValue ExpandLoad(SDValue Op);
125  SDValue ExpandStore(SDValue Op);
126  SDValue ExpandFNEG(SDValue Op);
127  SDValue ExpandFSUB(SDValue Op);
128  SDValue ExpandBITREVERSE(SDValue Op);
129  SDValue ExpandCTLZ(SDValue Op);
130  SDValue ExpandCTTZ_ZERO_UNDEF(SDValue Op);
131 
132  /// \brief Implements vector promotion.
133  ///
134  /// This is essentially just bitcasting the operands to a different type and
135  /// bitcasting the result back to the original type.
136  SDValue Promote(SDValue Op);
137 
138  /// \brief Implements [SU]INT_TO_FP vector promotion.
139  ///
140  /// This is a [zs]ext of the input operand to the next size up.
141  SDValue PromoteINT_TO_FP(SDValue Op);
142 
143  /// \brief Implements FP_TO_[SU]INT vector promotion of the result type.
144  ///
145  /// It is promoted to the next size up integer type. The result is then
146  /// truncated back to the original type.
147  SDValue PromoteFP_TO_INT(SDValue Op, bool isSigned);
148 
149 public:
150  VectorLegalizer(SelectionDAG& dag) :
151  DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
152 
153  /// \brief Begin legalizer the vector operations in the DAG.
154  bool Run();
155 };
156 
157 } // end anonymous namespace
158 
159 bool VectorLegalizer::Run() {
160  // Before we start legalizing vector nodes, check if there are any vectors.
161  bool HasVectors = false;
163  E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
164  // Check if the values of the nodes contain vectors. We don't need to check
165  // the operands because we are going to check their values at some point.
166  for (SDNode::value_iterator J = I->value_begin(), E = I->value_end();
167  J != E; ++J)
168  HasVectors |= J->isVector();
169 
170  // If we found a vector node we can start the legalization.
171  if (HasVectors)
172  break;
173  }
174 
175  // If this basic block has no vectors then no need to legalize vectors.
176  if (!HasVectors)
177  return false;
178 
179  // The legalize process is inherently a bottom-up recursive process (users
180  // legalize their uses before themselves). Given infinite stack space, we
181  // could just start legalizing on the root and traverse the whole graph. In
182  // practice however, this causes us to run out of stack space on large basic
183  // blocks. To avoid this problem, compute an ordering of the nodes where each
184  // node is only legalized after all of its operands are legalized.
187  E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
188  LegalizeOp(SDValue(&*I, 0));
189 
190  // Finally, it's possible the root changed. Get the new root.
191  SDValue OldRoot = DAG.getRoot();
192  assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
193  DAG.setRoot(LegalizedNodes[OldRoot]);
194 
195  LegalizedNodes.clear();
196 
197  // Remove dead nodes now.
198  DAG.RemoveDeadNodes();
199 
200  return Changed;
201 }
202 
203 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) {
204  // Generic legalization: just pass the operand through.
205  for (unsigned i = 0, e = Op.getNode()->getNumValues(); i != e; ++i)
206  AddLegalizedOperand(Op.getValue(i), Result.getValue(i));
207  return Result.getValue(Op.getResNo());
208 }
209 
210 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
211  // Note that LegalizeOp may be reentered even from single-use nodes, which
212  // means that we always must cache transformed nodes.
213  DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
214  if (I != LegalizedNodes.end()) return I->second;
215 
216  SDNode* Node = Op.getNode();
217 
218  // Legalize the operands
220  for (const SDValue &Op : Node->op_values())
221  Ops.push_back(LegalizeOp(Op));
222 
223  SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0);
224 
225  bool HasVectorValue = false;
226  if (Op.getOpcode() == ISD::LOAD) {
227  LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
228  ISD::LoadExtType ExtType = LD->getExtensionType();
229  if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD)
230  switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
231  LD->getMemoryVT())) {
232  default: llvm_unreachable("This action is not supported yet!");
234  return TranslateLegalizeResults(Op, Result);
236  if (SDValue Lowered = TLI.LowerOperation(Result, DAG)) {
237  if (Lowered == Result)
238  return TranslateLegalizeResults(Op, Lowered);
239  Changed = true;
240  if (Lowered->getNumValues() != Op->getNumValues()) {
241  // This expanded to something other than the load. Assume the
242  // lowering code took care of any chain values, and just handle the
243  // returned value.
244  assert(Result.getValue(1).use_empty() &&
245  "There are still live users of the old chain!");
246  return LegalizeOp(Lowered);
247  }
248  return TranslateLegalizeResults(Op, Lowered);
249  }
252  Changed = true;
253  return LegalizeOp(ExpandLoad(Op));
254  }
255  } else if (Op.getOpcode() == ISD::STORE) {
256  StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
257  EVT StVT = ST->getMemoryVT();
258  MVT ValVT = ST->getValue().getSimpleValueType();
259  if (StVT.isVector() && ST->isTruncatingStore())
260  switch (TLI.getTruncStoreAction(ValVT, StVT)) {
261  default: llvm_unreachable("This action is not supported yet!");
263  return TranslateLegalizeResults(Op, Result);
264  case TargetLowering::Custom: {
265  SDValue Lowered = TLI.LowerOperation(Result, DAG);
266  Changed = Lowered != Result;
267  return TranslateLegalizeResults(Op, Lowered);
268  }
270  Changed = true;
271  return LegalizeOp(ExpandStore(Op));
272  }
273  } else if (Op.getOpcode() == ISD::MSCATTER || Op.getOpcode() == ISD::MSTORE)
274  HasVectorValue = true;
275 
276  for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end();
277  J != E;
278  ++J)
279  HasVectorValue |= J->isVector();
280  if (!HasVectorValue)
281  return TranslateLegalizeResults(Op, Result);
282 
283  EVT QueryType;
284  switch (Op.getOpcode()) {
285  default:
286  return TranslateLegalizeResults(Op, Result);
287  case ISD::ADD:
288  case ISD::SUB:
289  case ISD::MUL:
290  case ISD::SDIV:
291  case ISD::UDIV:
292  case ISD::SREM:
293  case ISD::UREM:
294  case ISD::SDIVREM:
295  case ISD::UDIVREM:
296  case ISD::FADD:
297  case ISD::FSUB:
298  case ISD::FMUL:
299  case ISD::FDIV:
300  case ISD::FREM:
301  case ISD::AND:
302  case ISD::OR:
303  case ISD::XOR:
304  case ISD::SHL:
305  case ISD::SRA:
306  case ISD::SRL:
307  case ISD::ROTL:
308  case ISD::ROTR:
309  case ISD::BSWAP:
310  case ISD::BITREVERSE:
311  case ISD::CTLZ:
312  case ISD::CTTZ:
315  case ISD::CTPOP:
316  case ISD::SELECT:
317  case ISD::VSELECT:
318  case ISD::SELECT_CC:
319  case ISD::SETCC:
320  case ISD::ZERO_EXTEND:
321  case ISD::ANY_EXTEND:
322  case ISD::TRUNCATE:
323  case ISD::SIGN_EXTEND:
324  case ISD::FP_TO_SINT:
325  case ISD::FP_TO_UINT:
326  case ISD::FNEG:
327  case ISD::FABS:
328  case ISD::FMINNUM:
329  case ISD::FMAXNUM:
330  case ISD::FMINNAN:
331  case ISD::FMAXNAN:
332  case ISD::FCOPYSIGN:
333  case ISD::FSQRT:
334  case ISD::FSIN:
335  case ISD::FCOS:
336  case ISD::FPOWI:
337  case ISD::FPOW:
338  case ISD::FLOG:
339  case ISD::FLOG2:
340  case ISD::FLOG10:
341  case ISD::FEXP:
342  case ISD::FEXP2:
343  case ISD::FCEIL:
344  case ISD::FTRUNC:
345  case ISD::FRINT:
346  case ISD::FNEARBYINT:
347  case ISD::FROUND:
348  case ISD::FFLOOR:
349  case ISD::FP_ROUND:
350  case ISD::FP_EXTEND:
351  case ISD::FMA:
356  case ISD::SMIN:
357  case ISD::SMAX:
358  case ISD::UMIN:
359  case ISD::UMAX:
360  case ISD::SMUL_LOHI:
361  case ISD::UMUL_LOHI:
362  QueryType = Node->getValueType(0);
363  break;
364  case ISD::FP_ROUND_INREG:
365  QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT();
366  break;
367  case ISD::SINT_TO_FP:
368  case ISD::UINT_TO_FP:
369  QueryType = Node->getOperand(0).getValueType();
370  break;
371  case ISD::MSCATTER:
372  QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType();
373  break;
374  case ISD::MSTORE:
375  QueryType = cast<MaskedStoreSDNode>(Node)->getValue().getValueType();
376  break;
377  }
378 
379  switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) {
380  default: llvm_unreachable("This action is not supported yet!");
382  Result = Promote(Op);
383  Changed = true;
384  break;
386  break;
387  case TargetLowering::Custom: {
388  if (SDValue Tmp1 = TLI.LowerOperation(Op, DAG)) {
389  Result = Tmp1;
390  break;
391  }
393  }
395  Result = Expand(Op);
396  }
397 
398  // Make sure that the generated code is itself legal.
399  if (Result != Op) {
400  Result = LegalizeOp(Result);
401  Changed = true;
402  }
403 
404  // Note that LegalizeOp may be reentered even from single-use nodes, which
405  // means that we always must cache transformed nodes.
406  AddLegalizedOperand(Op, Result);
407  return Result;
408 }
409 
410 SDValue VectorLegalizer::Promote(SDValue Op) {
411  // For a few operations there is a specific concept for promotion based on
412  // the operand's type.
413  switch (Op.getOpcode()) {
414  case ISD::SINT_TO_FP:
415  case ISD::UINT_TO_FP:
416  // "Promote" the operation by extending the operand.
417  return PromoteINT_TO_FP(Op);
418  case ISD::FP_TO_UINT:
419  case ISD::FP_TO_SINT:
420  // Promote the operation by extending the operand.
421  return PromoteFP_TO_INT(Op, Op->getOpcode() == ISD::FP_TO_SINT);
422  }
423 
424  // There are currently two cases of vector promotion:
425  // 1) Bitcasting a vector of integers to a different type to a vector of the
426  // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
427  // 2) Extending a vector of floats to a vector of the same number of larger
428  // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
429  MVT VT = Op.getSimpleValueType();
430  assert(Op.getNode()->getNumValues() == 1 &&
431  "Can't promote a vector with multiple results!");
432  MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT);
433  SDLoc dl(Op);
435 
436  for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
437  if (Op.getOperand(j).getValueType().isVector())
438  if (Op.getOperand(j)
439  .getValueType()
441  .isFloatingPoint() &&
442  NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
443  Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op.getOperand(j));
444  else
445  Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Op.getOperand(j));
446  else
447  Operands[j] = Op.getOperand(j);
448  }
449 
450  Op = DAG.getNode(Op.getOpcode(), dl, NVT, Operands, Op.getNode()->getFlags());
451  if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
452  (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
453  NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
454  return DAG.getNode(ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0, dl));
455  else
456  return DAG.getNode(ISD::BITCAST, dl, VT, Op);
457 }
458 
459 SDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) {
460  // INT_TO_FP operations may require the input operand be promoted even
461  // when the type is otherwise legal.
462  EVT VT = Op.getOperand(0).getValueType();
463  assert(Op.getNode()->getNumValues() == 1 &&
464  "Can't promote a vector with multiple results!");
465 
466  // Normal getTypeToPromoteTo() doesn't work here, as that will promote
467  // by widening the vector w/ the same element width and twice the number
468  // of elements. We want the other way around, the same number of elements,
469  // each twice the width.
470  //
471  // Increase the bitwidth of the element to the next pow-of-two
472  // (which is greater than 8 bits).
473 
475  assert(NVT.isSimple() && "Promoting to a non-simple vector type!");
476  SDLoc dl(Op);
478 
479  unsigned Opc = Op.getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND :
481  for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
482  if (Op.getOperand(j).getValueType().isVector())
483  Operands[j] = DAG.getNode(Opc, dl, NVT, Op.getOperand(j));
484  else
485  Operands[j] = Op.getOperand(j);
486  }
487 
488  return DAG.getNode(Op.getOpcode(), dl, Op.getValueType(), Operands);
489 }
490 
491 // For FP_TO_INT we promote the result type to a vector type with wider
492 // elements and then truncate the result. This is different from the default
493 // PromoteVector which uses bitcast to promote thus assumning that the
494 // promoted vector type has the same overall size.
495 SDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op, bool isSigned) {
496  assert(Op.getNode()->getNumValues() == 1 &&
497  "Can't promote a vector with multiple results!");
498  EVT VT = Op.getValueType();
499 
500  EVT NewVT;
501  unsigned NewOpc;
502  while (true) {
503  NewVT = VT.widenIntegerVectorElementType(*DAG.getContext());
504  assert(NewVT.isSimple() && "Promoting to a non-simple vector type!");
505  if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewVT)) {
506  NewOpc = ISD::FP_TO_SINT;
507  break;
508  }
509  if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewVT)) {
510  NewOpc = ISD::FP_TO_UINT;
511  break;
512  }
513  }
514 
515  SDLoc loc(Op);
516  SDValue promoted = DAG.getNode(NewOpc, SDLoc(Op), NewVT, Op.getOperand(0));
517  return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT, promoted);
518 }
519 
520 SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
521  LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
522 
523  EVT SrcVT = LD->getMemoryVT();
524  EVT SrcEltVT = SrcVT.getScalarType();
525  unsigned NumElem = SrcVT.getVectorNumElements();
526 
527  SDValue NewChain;
528  SDValue Value;
529  if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) {
530  SDLoc dl(Op);
531 
533  SmallVector<SDValue, 8> LoadChains;
534 
535  EVT DstEltVT = LD->getValueType(0).getScalarType();
536  SDValue Chain = LD->getChain();
537  SDValue BasePTR = LD->getBasePtr();
538  ISD::LoadExtType ExtType = LD->getExtensionType();
539 
540  // When elements in a vector is not byte-addressable, we cannot directly
541  // load each element by advancing pointer, which could only address bytes.
542  // Instead, we load all significant words, mask bits off, and concatenate
543  // them to form each element. Finally, they are extended to destination
544  // scalar type to build the destination vector.
545  EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());
546 
547  assert(WideVT.isRound() &&
548  "Could not handle the sophisticated case when the widest integer is"
549  " not power of 2.");
550  assert(WideVT.bitsGE(SrcEltVT) &&
551  "Type is not legalized?");
552 
553  unsigned WideBytes = WideVT.getStoreSize();
554  unsigned Offset = 0;
555  unsigned RemainingBytes = SrcVT.getStoreSize();
556  SmallVector<SDValue, 8> LoadVals;
557 
558  while (RemainingBytes > 0) {
559  SDValue ScalarLoad;
560  unsigned LoadBytes = WideBytes;
561 
562  if (RemainingBytes >= LoadBytes) {
563  ScalarLoad =
564  DAG.getLoad(WideVT, dl, Chain, BasePTR,
565  LD->getPointerInfo().getWithOffset(Offset),
566  MinAlign(LD->getAlignment(), Offset),
567  LD->getMemOperand()->getFlags(), LD->getAAInfo());
568  } else {
569  EVT LoadVT = WideVT;
570  while (RemainingBytes < LoadBytes) {
571  LoadBytes >>= 1; // Reduce the load size by half.
572  LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3);
573  }
574  ScalarLoad =
575  DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR,
576  LD->getPointerInfo().getWithOffset(Offset), LoadVT,
577  MinAlign(LD->getAlignment(), Offset),
578  LD->getMemOperand()->getFlags(), LD->getAAInfo());
579  }
580 
581  RemainingBytes -= LoadBytes;
582  Offset += LoadBytes;
583  BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
584  DAG.getConstant(LoadBytes, dl,
585  BasePTR.getValueType()));
586 
587  LoadVals.push_back(ScalarLoad.getValue(0));
588  LoadChains.push_back(ScalarLoad.getValue(1));
589  }
590 
591  // Extract bits, pack and extend/trunc them into destination type.
592  unsigned SrcEltBits = SrcEltVT.getSizeInBits();
593  SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT);
594 
595  unsigned BitOffset = 0;
596  unsigned WideIdx = 0;
597  unsigned WideBits = WideVT.getSizeInBits();
598 
599  for (unsigned Idx = 0; Idx != NumElem; ++Idx) {
600  SDValue Lo, Hi, ShAmt;
601 
602  if (BitOffset < WideBits) {
603  ShAmt = DAG.getConstant(
604  BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
605  Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt);
606  Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask);
607  }
608 
609  BitOffset += SrcEltBits;
610  if (BitOffset >= WideBits) {
611  WideIdx++;
612  BitOffset -= WideBits;
613  if (BitOffset > 0) {
614  ShAmt = DAG.getConstant(
615  SrcEltBits - BitOffset, dl,
616  TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
617  Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt);
618  Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask);
619  }
620  }
621 
622  if (Hi.getNode())
623  Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi);
624 
625  switch (ExtType) {
626  default: llvm_unreachable("Unknown extended-load op!");
627  case ISD::EXTLOAD:
628  Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT);
629  break;
630  case ISD::ZEXTLOAD:
631  Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
632  break;
633  case ISD::SEXTLOAD:
634  ShAmt =
635  DAG.getConstant(WideBits - SrcEltBits, dl,
636  TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
637  Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt);
638  Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt);
639  Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
640  break;
641  }
642  Vals.push_back(Lo);
643  }
644 
645  NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
646  Value = DAG.getBuildVector(Op.getNode()->getValueType(0), dl, Vals);
647  } else {
648  SDValue Scalarized = TLI.scalarizeVectorLoad(LD, DAG);
649 
650  NewChain = Scalarized.getValue(1);
651  Value = Scalarized.getValue(0);
652  }
653 
654  AddLegalizedOperand(Op.getValue(0), Value);
655  AddLegalizedOperand(Op.getValue(1), NewChain);
656 
657  return (Op.getResNo() ? NewChain : Value);
658 }
659 
660 SDValue VectorLegalizer::ExpandStore(SDValue Op) {
661  StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
662 
663  EVT StVT = ST->getMemoryVT();
664  EVT MemSclVT = StVT.getScalarType();
665  unsigned ScalarSize = MemSclVT.getSizeInBits();
666 
667  // Round odd types to the next pow of two.
668  if (!isPowerOf2_32(ScalarSize)) {
669  // FIXME: This is completely broken and inconsistent with ExpandLoad
670  // handling.
671 
672  // For sub-byte element sizes, this ends up with 0 stride between elements,
673  // so the same element just gets re-written to the same location. There seem
674  // to be tests explicitly testing for this broken behavior though. tests
675  // for this broken behavior.
676 
677  LLVMContext &Ctx = *DAG.getContext();
678 
679  EVT NewMemVT
680  = EVT::getVectorVT(Ctx,
681  MemSclVT.getIntegerVT(Ctx, NextPowerOf2(ScalarSize)),
682  StVT.getVectorNumElements());
683 
684  SDValue NewVectorStore = DAG.getTruncStore(
685  ST->getChain(), SDLoc(Op), ST->getValue(), ST->getBasePtr(),
686  ST->getPointerInfo(), NewMemVT, ST->getAlignment(),
687  ST->getMemOperand()->getFlags(), ST->getAAInfo());
688  ST = cast<StoreSDNode>(NewVectorStore.getNode());
689  }
690 
691  SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
692  AddLegalizedOperand(Op, TF);
693  return TF;
694 }
695 
696 SDValue VectorLegalizer::Expand(SDValue Op) {
697  switch (Op->getOpcode()) {
699  return ExpandSEXTINREG(Op);
701  return ExpandANY_EXTEND_VECTOR_INREG(Op);
703  return ExpandSIGN_EXTEND_VECTOR_INREG(Op);
705  return ExpandZERO_EXTEND_VECTOR_INREG(Op);
706  case ISD::BSWAP:
707  return ExpandBSWAP(Op);
708  case ISD::VSELECT:
709  return ExpandVSELECT(Op);
710  case ISD::SELECT:
711  return ExpandSELECT(Op);
712  case ISD::UINT_TO_FP:
713  return ExpandUINT_TO_FLOAT(Op);
714  case ISD::FNEG:
715  return ExpandFNEG(Op);
716  case ISD::FSUB:
717  return ExpandFSUB(Op);
718  case ISD::SETCC:
719  return UnrollVSETCC(Op);
720  case ISD::BITREVERSE:
721  return ExpandBITREVERSE(Op);
722  case ISD::CTLZ:
724  return ExpandCTLZ(Op);
726  return ExpandCTTZ_ZERO_UNDEF(Op);
727  default:
728  return DAG.UnrollVectorOp(Op.getNode());
729  }
730 }
731 
732 SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
733  // Lower a select instruction where the condition is a scalar and the
734  // operands are vectors. Lower this select to VSELECT and implement it
735  // using XOR AND OR. The selector bit is broadcasted.
736  EVT VT = Op.getValueType();
737  SDLoc DL(Op);
738 
739  SDValue Mask = Op.getOperand(0);
740  SDValue Op1 = Op.getOperand(1);
741  SDValue Op2 = Op.getOperand(2);
742 
743  assert(VT.isVector() && !Mask.getValueType().isVector()
744  && Op1.getValueType() == Op2.getValueType() && "Invalid type");
745 
746  // If we can't even use the basic vector operations of
747  // AND,OR,XOR, we will have to scalarize the op.
748  // Notice that the operation may be 'promoted' which means that it is
749  // 'bitcasted' to another type which is handled.
750  // Also, we need to be able to construct a splat vector using BUILD_VECTOR.
755  return DAG.UnrollVectorOp(Op.getNode());
756 
757  // Generate a mask operand.
759 
760  // What is the size of each element in the vector mask.
761  EVT BitTy = MaskTy.getScalarType();
762 
763  Mask = DAG.getSelect(DL, BitTy, Mask,
765  BitTy),
766  DAG.getConstant(0, DL, BitTy));
767 
768  // Broadcast the mask so that the entire vector is all-one or all zero.
769  Mask = DAG.getSplatBuildVector(MaskTy, DL, Mask);
770 
771  // Bitcast the operands to be the same type as the mask.
772  // This is needed when we select between FP types because
773  // the mask is a vector of integers.
774  Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
775  Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
776 
777  SDValue AllOnes = DAG.getConstant(
778  APInt::getAllOnesValue(BitTy.getSizeInBits()), DL, MaskTy);
779  SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes);
780 
781  Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
782  Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
783  SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
784  return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val);
785 }
786 
787 SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) {
788  EVT VT = Op.getValueType();
789 
790  // Make sure that the SRA and SHL instructions are available.
793  return DAG.UnrollVectorOp(Op.getNode());
794 
795  SDLoc DL(Op);
796  EVT OrigTy = cast<VTSDNode>(Op->getOperand(1))->getVT();
797 
798  unsigned BW = VT.getScalarSizeInBits();
799  unsigned OrigBW = OrigTy.getScalarSizeInBits();
800  SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
801 
802  Op = Op.getOperand(0);
803  Op = DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz);
804  return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
805 }
806 
807 // Generically expand a vector anyext in register to a shuffle of the relevant
808 // lanes into the appropriate locations, with other lanes left undef.
809 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) {
810  SDLoc DL(Op);
811  EVT VT = Op.getValueType();
812  int NumElements = VT.getVectorNumElements();
813  SDValue Src = Op.getOperand(0);
814  EVT SrcVT = Src.getValueType();
815  int NumSrcElements = SrcVT.getVectorNumElements();
816 
817  // Build a base mask of undef shuffles.
818  SmallVector<int, 16> ShuffleMask;
819  ShuffleMask.resize(NumSrcElements, -1);
820 
821  // Place the extended lanes into the correct locations.
822  int ExtLaneScale = NumSrcElements / NumElements;
823  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
824  for (int i = 0; i < NumElements; ++i)
825  ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
826 
827  return DAG.getNode(
828  ISD::BITCAST, DL, VT,
829  DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
830 }
831 
832 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) {
833  SDLoc DL(Op);
834  EVT VT = Op.getValueType();
835  SDValue Src = Op.getOperand(0);
836  EVT SrcVT = Src.getValueType();
837 
838  // First build an any-extend node which can be legalized above when we
839  // recurse through it.
840  Op = DAG.getAnyExtendVectorInReg(Src, DL, VT);
841 
842  // Now we need sign extend. Do this by shifting the elements. Even if these
843  // aren't legal operations, they have a better chance of being legalized
844  // without full scalarization than the sign extension does.
845  unsigned EltWidth = VT.getScalarSizeInBits();
846  unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
847  SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
848  return DAG.getNode(ISD::SRA, DL, VT,
849  DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
850  ShiftAmount);
851 }
852 
853 // Generically expand a vector zext in register to a shuffle of the relevant
854 // lanes into the appropriate locations, a blend of zero into the high bits,
855 // and a bitcast to the wider element type.
856 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) {
857  SDLoc DL(Op);
858  EVT VT = Op.getValueType();
859  int NumElements = VT.getVectorNumElements();
860  SDValue Src = Op.getOperand(0);
861  EVT SrcVT = Src.getValueType();
862  int NumSrcElements = SrcVT.getVectorNumElements();
863 
864  // Build up a zero vector to blend into this one.
865  SDValue Zero = DAG.getConstant(0, DL, SrcVT);
866 
867  // Shuffle the incoming lanes into the correct position, and pull all other
868  // lanes from the zero vector.
869  SmallVector<int, 16> ShuffleMask;
870  ShuffleMask.reserve(NumSrcElements);
871  for (int i = 0; i < NumSrcElements; ++i)
872  ShuffleMask.push_back(i);
873 
874  int ExtLaneScale = NumSrcElements / NumElements;
875  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
876  for (int i = 0; i < NumElements; ++i)
877  ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
878 
879  return DAG.getNode(ISD::BITCAST, DL, VT,
880  DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
881 }
882 
883 static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
884  int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
885  for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
886  for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
887  ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
888 }
889 
890 SDValue VectorLegalizer::ExpandBSWAP(SDValue Op) {
891  EVT VT = Op.getValueType();
892 
893  // Generate a byte wise shuffle mask for the BSWAP.
894  SmallVector<int, 16> ShuffleMask;
895  createBSWAPShuffleMask(VT, ShuffleMask);
896  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
897 
898  // Only emit a shuffle if the mask is legal.
899  if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT))
900  return DAG.UnrollVectorOp(Op.getNode());
901 
902  SDLoc DL(Op);
903  Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Op.getOperand(0));
904  Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
905  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
906 }
907 
908 SDValue VectorLegalizer::ExpandBITREVERSE(SDValue Op) {
909  EVT VT = Op.getValueType();
910 
911  // If we have the scalar operation, it's probably cheaper to unroll it.
913  return DAG.UnrollVectorOp(Op.getNode());
914 
915  // If the vector element width is a whole number of bytes, test if its legal
916  // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
917  // vector. This greatly reduces the number of bit shifts necessary.
918  unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
919  if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
920  SmallVector<int, 16> BSWAPMask;
921  createBSWAPShuffleMask(VT, BSWAPMask);
922 
923  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
924  if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
926  (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
927  TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
930  SDLoc DL(Op);
931  Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Op.getOperand(0));
932  Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
933  BSWAPMask);
934  Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
935  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
936  }
937  }
938 
939  // If we have the appropriate vector bit operations, it is better to use them
940  // than unrolling and expanding each component.
941  if (!TLI.isOperationLegalOrCustom(ISD::SHL, VT) ||
945  return DAG.UnrollVectorOp(Op.getNode());
946 
947  // Let LegalizeDAG handle this later.
948  return Op;
949 }
950 
951 SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
952  // Implement VSELECT in terms of XOR, AND, OR
953  // on platforms which do not support blend natively.
954  SDLoc DL(Op);
955 
956  SDValue Mask = Op.getOperand(0);
957  SDValue Op1 = Op.getOperand(1);
958  SDValue Op2 = Op.getOperand(2);
959 
960  EVT VT = Mask.getValueType();
961 
962  // If we can't even use the basic vector operations of
963  // AND,OR,XOR, we will have to scalarize the op.
964  // Notice that the operation may be 'promoted' which means that it is
965  // 'bitcasted' to another type which is handled.
966  // This operation also isn't safe with AND, OR, XOR when the boolean
967  // type is 0/1 as we need an all ones vector constant to mask with.
968  // FIXME: Sign extend 1 to all ones if thats legal on the target.
972  TLI.getBooleanContents(Op1.getValueType()) !=
974  return DAG.UnrollVectorOp(Op.getNode());
975 
976  // If the mask and the type are different sizes, unroll the vector op. This
977  // can occur when getSetCCResultType returns something that is different in
978  // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
979  if (VT.getSizeInBits() != Op1.getValueSizeInBits())
980  return DAG.UnrollVectorOp(Op.getNode());
981 
982  // Bitcast the operands to be the same type as the mask.
983  // This is needed when we select between FP types because
984  // the mask is a vector of integers.
985  Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
986  Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
987 
988  SDValue AllOnes = DAG.getConstant(
990  SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes);
991 
992  Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
993  Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
994  SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
995  return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val);
996 }
997 
998 SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
999  EVT VT = Op.getOperand(0).getValueType();
1000  SDLoc DL(Op);
1001 
1002  // Make sure that the SINT_TO_FP and SRL instructions are available.
1005  return DAG.UnrollVectorOp(Op.getNode());
1006 
1007  unsigned BW = VT.getScalarSizeInBits();
1008  assert((BW == 64 || BW == 32) &&
1009  "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1010 
1011  SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1012 
1013  // Constants to clear the upper part of the word.
1014  // Notice that we can also use SHL+SHR, but using a constant is slightly
1015  // faster on x86.
1016  uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1017  SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1018 
1019  // Two to the power of half-word-size.
1020  SDValue TWOHW = DAG.getConstantFP(1 << (BW / 2), DL, Op.getValueType());
1021 
1022  // Clear upper part of LO, lower HI
1023  SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord);
1024  SDValue LO = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), HalfWordMask);
1025 
1026  // Convert hi and lo to floats
1027  // Convert the hi part back to the upper values
1028  // TODO: Can any fast-math-flags be set on these nodes?
1029  SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), HI);
1030  fHI = DAG.getNode(ISD::FMUL, DL, Op.getValueType(), fHI, TWOHW);
1031  SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), LO);
1032 
1033  // Add the two halves
1034  return DAG.getNode(ISD::FADD, DL, Op.getValueType(), fHI, fLO);
1035 }
1036 
1037 SDValue VectorLegalizer::ExpandFNEG(SDValue Op) {
1039  SDLoc DL(Op);
1040  SDValue Zero = DAG.getConstantFP(-0.0, DL, Op.getValueType());
1041  // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1042  return DAG.getNode(ISD::FSUB, DL, Op.getValueType(),
1043  Zero, Op.getOperand(0));
1044  }
1045  return DAG.UnrollVectorOp(Op.getNode());
1046 }
1047 
1048 SDValue VectorLegalizer::ExpandFSUB(SDValue Op) {
1049  // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1050  // we can defer this to operation legalization where it will be lowered as
1051  // a+(-b).
1052  EVT VT = Op.getValueType();
1053  if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1055  return Op; // Defer to LegalizeDAG
1056 
1057  return DAG.UnrollVectorOp(Op.getNode());
1058 }
1059 
1060 SDValue VectorLegalizer::ExpandCTLZ(SDValue Op) {
1061  EVT VT = Op.getValueType();
1062  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
1063 
1064  // If the non-ZERO_UNDEF version is supported we can use that instead.
1065  if (Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
1067  SDLoc DL(Op);
1068  return DAG.getNode(ISD::CTLZ, DL, Op.getValueType(), Op.getOperand(0));
1069  }
1070 
1071  // If CTPOP is available we can lower with a CTPOP based method:
1072  // u16 ctlz(u16 x) {
1073  // x |= (x >> 1);
1074  // x |= (x >> 2);
1075  // x |= (x >> 4);
1076  // x |= (x >> 8);
1077  // return ctpop(~x);
1078  // }
1079  // Ref: "Hacker's Delight" by Henry Warren
1080  if (isPowerOf2_32(NumBitsPerElt) &&
1085  SDLoc DL(Op);
1086  SDValue Res = Op.getOperand(0);
1087  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
1088 
1089  for (unsigned i = 1; i != NumBitsPerElt; i *= 2)
1090  Res = DAG.getNode(
1091  ISD::OR, DL, VT, Res,
1092  DAG.getNode(ISD::SRL, DL, VT, Res, DAG.getConstant(i, DL, ShiftTy)));
1093 
1094  Res = DAG.getNOT(DL, Res, VT);
1095  return DAG.getNode(ISD::CTPOP, DL, VT, Res);
1096  }
1097 
1098  // Otherwise go ahead and unroll.
1099  return DAG.UnrollVectorOp(Op.getNode());
1100 }
1101 
1102 SDValue VectorLegalizer::ExpandCTTZ_ZERO_UNDEF(SDValue Op) {
1103  // If the non-ZERO_UNDEF version is supported we can use that instead.
1105  SDLoc DL(Op);
1106  return DAG.getNode(ISD::CTTZ, DL, Op.getValueType(), Op.getOperand(0));
1107  }
1108 
1109  // Otherwise go ahead and unroll.
1110  return DAG.UnrollVectorOp(Op.getNode());
1111 }
1112 
1113 SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
1114  EVT VT = Op.getValueType();
1115  unsigned NumElems = VT.getVectorNumElements();
1116  EVT EltVT = VT.getVectorElementType();
1117  SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1), CC = Op.getOperand(2);
1118  EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1119  SDLoc dl(Op);
1120  SmallVector<SDValue, 8> Ops(NumElems);
1121  for (unsigned i = 0; i < NumElems; ++i) {
1122  SDValue LHSElem = DAG.getNode(
1123  ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1124  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1125  SDValue RHSElem = DAG.getNode(
1126  ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1127  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1128  Ops[i] = DAG.getNode(ISD::SETCC, dl,
1130  *DAG.getContext(), TmpEltVT),
1131  LHSElem, RHSElem, CC);
1132  Ops[i] = DAG.getSelect(dl, EltVT, Ops[i],
1134  (EltVT.getSizeInBits()), dl, EltVT),
1135  DAG.getConstant(0, dl, EltVT));
1136  }
1137  return DAG.getBuildVector(VT, dl, Ops);
1138 }
1139 
1141  return VectorLegalizer(*this).Run();
1142 }
LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return how this load with extension should be treated: either it is legal, needs to be promoted to a ...
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:545
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:512
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:569
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:555
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:494
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
const SDValue & getValue() const
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
const SDValue & getChain() const
unsigned getAlignment() const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:911
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:485
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:359
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:452
void reserve(size_type N)
Definition: SmallVector.h:380
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:404
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Shift and rotation operations.
Definition: ISDOpcodes.h:379
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:388
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This file implements a class to represent arbitrary precision integral constant values and operations...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:455
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:398
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:830
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:472
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements. ...
Definition: SelectionDAG.h:729
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:498
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:302
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:602
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getBasePtr() const
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:235
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
Machine Value Type.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
Simple binary floating point operators.
Definition: ISDOpcodes.h:259
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
iterator_range< value_op_iterator > op_values() const
bool isRound() const
Return true if the size is a power-of-two number of bytes.
Definition: ValueTypes.h:217
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:891
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:385
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:530
Extended Value Type.
Definition: ValueTypes.h:34
uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:632
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
Iterator for intrusive lists based on ilist_node.
SDValue scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:314
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:209
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:527
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:712
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:96
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:382
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
Definition: ValueTypes.h:342
Represents one node in the SelectionDAG.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:423
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT) const
Return how this store with truncation should be treated: either it is legal, needs to be promoted to ...
MachinePointerInfo getWithOffset(int64_t O) const
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:389
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:445
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:448
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:287
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Definition: ISDOpcodes.h:572
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to...
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:424
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:362
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:463
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:581
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
#define I(x, y, z)
Definition: MD5.cpp:58
Flags getFlags() const
Return the raw flags of the source value,.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:141
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:443
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
const MachinePointerInfo & getPointerInfo() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Definition: Value.h:73
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:277
unsigned getResNo() const
get the index which selects a specific result in the SDNode
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:235
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:412
unsigned getNumOperands() const
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations, those with specific masks.
Conversion operators.
Definition: ISDOpcodes.h:442
const SDValue & getOperand(unsigned i) const
bool isBigEndian() const
Definition: DataLayout.h:217
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:451
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:126
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:561
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
LLVMContext * getContext() const
Definition: SelectionDAG.h:393
This file describes how to lower LLVM code to machine code.
void resize(size_type N)
Definition: SmallVector.h:355
This class is used to represent ISD::LOAD nodes.