LLVM  16.0.0git
LegalizeVectorOps.cpp
Go to the documentation of this file.
1 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::LegalizeVectors method.
10 //
11 // The vector legalizer looks for vector operations which might need to be
12 // scalarized and legalizes them. This is a separate step from Legalize because
13 // scalarizing can introduce illegal types. For example, suppose we have an
14 // ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15 // on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16 // operation, which introduces nodes with the illegal type i64 which must be
17 // expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18 // the operation must be unrolled, which introduces nodes with the illegal
19 // type i8 which must be promoted.
20 //
21 // This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22 // or operations that happen to take a vector which are custom-lowered;
23 // the legalization for such operations never produces nodes
24 // with illegal types, so it's okay to put off legalizing them until
25 // SelectionDAG::Legalize runs.
26 //
27 //===----------------------------------------------------------------------===//
28 
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/SmallVector.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/Support/Casting.h"
38 #include "llvm/Support/Compiler.h"
39 #include "llvm/Support/Debug.h"
42 #include <cassert>
43 #include <cstdint>
44 #include <iterator>
45 #include <utility>
46 
47 using namespace llvm;
48 
49 #define DEBUG_TYPE "legalizevectorops"
50 
51 namespace {
52 
53 class VectorLegalizer {
54  SelectionDAG& DAG;
55  const TargetLowering &TLI;
56  bool Changed = false; // Keep track of whether anything changed
57 
58  /// For nodes that are of legal width, and that have more than one use, this
59  /// map indicates what regularized operand to use. This allows us to avoid
60  /// legalizing the same thing more than once.
62 
63  /// Adds a node to the translation cache.
64  void AddLegalizedOperand(SDValue From, SDValue To) {
65  LegalizedNodes.insert(std::make_pair(From, To));
66  // If someone requests legalization of the new node, return itself.
67  if (From != To)
68  LegalizedNodes.insert(std::make_pair(To, To));
69  }
70 
71  /// Legalizes the given node.
72  SDValue LegalizeOp(SDValue Op);
73 
74  /// Assuming the node is legal, "legalize" the results.
75  SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
76 
77  /// Make sure Results are legal and update the translation cache.
78  SDValue RecursivelyLegalizeResults(SDValue Op,
80 
81  /// Wrapper to interface LowerOperation with a vector of Results.
82  /// Returns false if the target wants to use default expansion. Otherwise
83  /// returns true. If return is true and the Results are empty, then the
84  /// target wants to keep the input node as is.
85  bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
86 
87  /// Implements unrolling a VSETCC.
88  SDValue UnrollVSETCC(SDNode *Node);
89 
90  /// Implement expand-based legalization of vector operations.
91  ///
92  /// This is just a high-level routine to dispatch to specific code paths for
93  /// operations to legalize them.
94  void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
95 
96  /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
97  /// FP_TO_SINT isn't legal.
98  void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
99 
100  /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
101  /// SINT_TO_FLOAT and SHR on vectors isn't legal.
102  void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
103 
104  /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
105  SDValue ExpandSEXTINREG(SDNode *Node);
106 
107  /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
108  ///
109  /// Shuffles the low lanes of the operand into place and bitcasts to the proper
110  /// type. The contents of the bits in the extended part of each element are
111  /// undef.
112  SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
113 
114  /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
115  ///
116  /// Shuffles the low lanes of the operand into place, bitcasts to the proper
117  /// type, then shifts left and arithmetic shifts right to introduce a sign
118  /// extension.
119  SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
120 
121  /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
122  ///
123  /// Shuffles the low lanes of the operand into place and blends zeros into
124  /// the remaining lanes, finally bitcasting to the proper type.
125  SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
126 
127  /// Expand bswap of vectors into a shuffle if legal.
128  SDValue ExpandBSWAP(SDNode *Node);
129 
130  /// Implement vselect in terms of XOR, AND, OR when blend is not
131  /// supported by the target.
132  SDValue ExpandVSELECT(SDNode *Node);
133  SDValue ExpandVP_SELECT(SDNode *Node);
134  SDValue ExpandVP_MERGE(SDNode *Node);
135  SDValue ExpandVP_REM(SDNode *Node);
136  SDValue ExpandSELECT(SDNode *Node);
137  std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
138  SDValue ExpandStore(SDNode *N);
139  SDValue ExpandFNEG(SDNode *Node);
140  void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141  void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142  void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143  void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144  void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145  void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146  void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147  void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148  void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 
150  void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 
152  /// Implements vector promotion.
153  ///
154  /// This is essentially just bitcasting the operands to a different type and
155  /// bitcasting the result back to the original type.
156  void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 
158  /// Implements [SU]INT_TO_FP vector promotion.
159  ///
160  /// This is a [zs]ext of the input operand to a larger integer type.
161  void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 
163  /// Implements FP_TO_[SU]INT vector promotion of the result type.
164  ///
165  /// It is promoted to a larger integer type. The result is then
166  /// truncated back to the original type.
167  void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
168 
169 public:
170  VectorLegalizer(SelectionDAG& dag) :
171  DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
172 
173  /// Begin legalizer the vector operations in the DAG.
174  bool Run();
175 };
176 
177 } // end anonymous namespace
178 
179 bool VectorLegalizer::Run() {
180  // Before we start legalizing vector nodes, check if there are any vectors.
181  bool HasVectors = false;
182  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
183  E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
184  // Check if the values of the nodes contain vectors. We don't need to check
185  // the operands because we are going to check their values at some point.
186  HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
187 
188  // If we found a vector node we can start the legalization.
189  if (HasVectors)
190  break;
191  }
192 
193  // If this basic block has no vectors then no need to legalize vectors.
194  if (!HasVectors)
195  return false;
196 
197  // The legalize process is inherently a bottom-up recursive process (users
198  // legalize their uses before themselves). Given infinite stack space, we
199  // could just start legalizing on the root and traverse the whole graph. In
200  // practice however, this causes us to run out of stack space on large basic
201  // blocks. To avoid this problem, compute an ordering of the nodes where each
202  // node is only legalized after all of its operands are legalized.
203  DAG.AssignTopologicalOrder();
204  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
205  E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
206  LegalizeOp(SDValue(&*I, 0));
207 
208  // Finally, it's possible the root changed. Get the new root.
209  SDValue OldRoot = DAG.getRoot();
210  assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
211  DAG.setRoot(LegalizedNodes[OldRoot]);
212 
213  LegalizedNodes.clear();
214 
215  // Remove dead nodes now.
216  DAG.RemoveDeadNodes();
217 
218  return Changed;
219 }
220 
221 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
222  assert(Op->getNumValues() == Result->getNumValues() &&
223  "Unexpected number of results");
224  // Generic legalization: just pass the operand through.
225  for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
226  AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
227  return SDValue(Result, Op.getResNo());
228 }
229 
230 SDValue
231 VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
233  assert(Results.size() == Op->getNumValues() &&
234  "Unexpected number of results");
235  // Make sure that the generated code is itself legal.
236  for (unsigned i = 0, e = Results.size(); i != e; ++i) {
237  Results[i] = LegalizeOp(Results[i]);
238  AddLegalizedOperand(Op.getValue(i), Results[i]);
239  }
240 
241  return Results[Op.getResNo()];
242 }
243 
244 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
245  // Note that LegalizeOp may be reentered even from single-use nodes, which
246  // means that we always must cache transformed nodes.
247  DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
248  if (I != LegalizedNodes.end()) return I->second;
249 
250  // Legalize the operands
252  for (const SDValue &Oper : Op->op_values())
253  Ops.push_back(LegalizeOp(Oper));
254 
255  SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
256 
257  bool HasVectorValueOrOp =
258  llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
259  llvm::any_of(Node->op_values(),
260  [](SDValue O) { return O.getValueType().isVector(); });
261  if (!HasVectorValueOrOp)
262  return TranslateLegalizeResults(Op, Node);
263 
265  EVT ValVT;
266  switch (Op.getOpcode()) {
267  default:
268  return TranslateLegalizeResults(Op, Node);
269  case ISD::LOAD: {
270  LoadSDNode *LD = cast<LoadSDNode>(Node);
271  ISD::LoadExtType ExtType = LD->getExtensionType();
272  EVT LoadedVT = LD->getMemoryVT();
273  if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
274  Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
275  break;
276  }
277  case ISD::STORE: {
278  StoreSDNode *ST = cast<StoreSDNode>(Node);
279  EVT StVT = ST->getMemoryVT();
280  MVT ValVT = ST->getValue().getSimpleValueType();
281  if (StVT.isVector() && ST->isTruncatingStore())
282  Action = TLI.getTruncStoreAction(ValVT, StVT);
283  break;
284  }
285  case ISD::MERGE_VALUES:
286  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
287  // This operation lies about being legal: when it claims to be legal,
288  // it should actually be expanded.
289  if (Action == TargetLowering::Legal)
290  Action = TargetLowering::Expand;
291  break;
292 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
293  case ISD::STRICT_##DAGN:
294 #include "llvm/IR/ConstrainedOps.def"
295  ValVT = Node->getValueType(0);
296  if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
297  Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
298  ValVT = Node->getOperand(1).getValueType();
299  Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
300  // If we're asked to expand a strict vector floating-point operation,
301  // by default we're going to simply unroll it. That is usually the
302  // best approach, except in the case where the resulting strict (scalar)
303  // operations would themselves use the fallback mutation to non-strict.
304  // In that specific case, just do the fallback on the vector op.
305  if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
306  TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
308  EVT EltVT = ValVT.getVectorElementType();
309  if (TLI.getOperationAction(Node->getOpcode(), EltVT)
311  TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
313  Action = TargetLowering::Legal;
314  }
315  break;
316  case ISD::ADD:
317  case ISD::SUB:
318  case ISD::MUL:
319  case ISD::MULHS:
320  case ISD::MULHU:
321  case ISD::SDIV:
322  case ISD::UDIV:
323  case ISD::SREM:
324  case ISD::UREM:
325  case ISD::SDIVREM:
326  case ISD::UDIVREM:
327  case ISD::FADD:
328  case ISD::FSUB:
329  case ISD::FMUL:
330  case ISD::FDIV:
331  case ISD::FREM:
332  case ISD::AND:
333  case ISD::OR:
334  case ISD::XOR:
335  case ISD::SHL:
336  case ISD::SRA:
337  case ISD::SRL:
338  case ISD::FSHL:
339  case ISD::FSHR:
340  case ISD::ROTL:
341  case ISD::ROTR:
342  case ISD::ABS:
343  case ISD::BSWAP:
344  case ISD::BITREVERSE:
345  case ISD::CTLZ:
346  case ISD::CTTZ:
349  case ISD::CTPOP:
350  case ISD::SELECT:
351  case ISD::VSELECT:
352  case ISD::SELECT_CC:
353  case ISD::ZERO_EXTEND:
354  case ISD::ANY_EXTEND:
355  case ISD::TRUNCATE:
356  case ISD::SIGN_EXTEND:
357  case ISD::FP_TO_SINT:
358  case ISD::FP_TO_UINT:
359  case ISD::FNEG:
360  case ISD::FABS:
361  case ISD::FMINNUM:
362  case ISD::FMAXNUM:
363  case ISD::FMINNUM_IEEE:
364  case ISD::FMAXNUM_IEEE:
365  case ISD::FMINIMUM:
366  case ISD::FMAXIMUM:
367  case ISD::FCOPYSIGN:
368  case ISD::FSQRT:
369  case ISD::FSIN:
370  case ISD::FCOS:
371  case ISD::FPOWI:
372  case ISD::FPOW:
373  case ISD::FLOG:
374  case ISD::FLOG2:
375  case ISD::FLOG10:
376  case ISD::FEXP:
377  case ISD::FEXP2:
378  case ISD::FCEIL:
379  case ISD::FTRUNC:
380  case ISD::FRINT:
381  case ISD::FNEARBYINT:
382  case ISD::FROUND:
383  case ISD::FROUNDEVEN:
384  case ISD::FFLOOR:
385  case ISD::FP_ROUND:
386  case ISD::FP_EXTEND:
387  case ISD::FMA:
392  case ISD::SMIN:
393  case ISD::SMAX:
394  case ISD::UMIN:
395  case ISD::UMAX:
396  case ISD::SMUL_LOHI:
397  case ISD::UMUL_LOHI:
398  case ISD::SADDO:
399  case ISD::UADDO:
400  case ISD::SSUBO:
401  case ISD::USUBO:
402  case ISD::SMULO:
403  case ISD::UMULO:
404  case ISD::FCANONICALIZE:
405  case ISD::SADDSAT:
406  case ISD::UADDSAT:
407  case ISD::SSUBSAT:
408  case ISD::USUBSAT:
409  case ISD::SSHLSAT:
410  case ISD::USHLSAT:
411  case ISD::FP_TO_SINT_SAT:
412  case ISD::FP_TO_UINT_SAT:
413  case ISD::MGATHER:
414  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
415  break;
416  case ISD::SMULFIX:
417  case ISD::SMULFIXSAT:
418  case ISD::UMULFIX:
419  case ISD::UMULFIXSAT:
420  case ISD::SDIVFIX:
421  case ISD::SDIVFIXSAT:
422  case ISD::UDIVFIX:
423  case ISD::UDIVFIXSAT: {
424  unsigned Scale = Node->getConstantOperandVal(2);
425  Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
426  Node->getValueType(0), Scale);
427  break;
428  }
429  case ISD::SINT_TO_FP:
430  case ISD::UINT_TO_FP:
431  case ISD::VECREDUCE_ADD:
432  case ISD::VECREDUCE_MUL:
433  case ISD::VECREDUCE_AND:
434  case ISD::VECREDUCE_OR:
435  case ISD::VECREDUCE_XOR:
436  case ISD::VECREDUCE_SMAX:
437  case ISD::VECREDUCE_SMIN:
438  case ISD::VECREDUCE_UMAX:
439  case ISD::VECREDUCE_UMIN:
440  case ISD::VECREDUCE_FADD:
441  case ISD::VECREDUCE_FMUL:
442  case ISD::VECREDUCE_FMAX:
443  case ISD::VECREDUCE_FMIN:
444  Action = TLI.getOperationAction(Node->getOpcode(),
445  Node->getOperand(0).getValueType());
446  break;
449  Action = TLI.getOperationAction(Node->getOpcode(),
450  Node->getOperand(1).getValueType());
451  break;
452  case ISD::SETCC: {
453  MVT OpVT = Node->getOperand(0).getSimpleValueType();
454  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
455  Action = TLI.getCondCodeAction(CCCode, OpVT);
456  if (Action == TargetLowering::Legal)
457  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
458  break;
459  }
460 
461 #define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
462  case ISD::VPID: { \
463  EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
464  : Node->getOperand(LEGALPOS).getValueType(); \
465  if (ISD::VPID == ISD::VP_SETCC) { \
466  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
467  Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
468  if (Action != TargetLowering::Legal) \
469  break; \
470  } \
471  Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
472  } break;
473 #include "llvm/IR/VPIntrinsics.def"
474  }
475 
476  LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
477 
478  SmallVector<SDValue, 8> ResultVals;
479  switch (Action) {
480  default: llvm_unreachable("This action is not supported yet!");
482  assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
483  "This action is not supported yet!");
484  LLVM_DEBUG(dbgs() << "Promoting\n");
485  Promote(Node, ResultVals);
486  assert(!ResultVals.empty() && "No results for promotion?");
487  break;
489  LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
490  break;
492  LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
493  if (LowerOperationWrapper(Node, ResultVals))
494  break;
495  LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
496  [[fallthrough]];
498  LLVM_DEBUG(dbgs() << "Expanding\n");
499  Expand(Node, ResultVals);
500  break;
501  }
502 
503  if (ResultVals.empty())
504  return TranslateLegalizeResults(Op, Node);
505 
506  Changed = true;
507  return RecursivelyLegalizeResults(Op, ResultVals);
508 }
509 
510 // FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
511 // merge them somehow?
512 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
514  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
515 
516  if (!Res.getNode())
517  return false;
518 
519  if (Res == SDValue(Node, 0))
520  return true;
521 
522  // If the original node has one result, take the return value from
523  // LowerOperation as is. It might not be result number 0.
524  if (Node->getNumValues() == 1) {
525  Results.push_back(Res);
526  return true;
527  }
528 
529  // If the original node has multiple results, then the return node should
530  // have the same number of results.
531  assert((Node->getNumValues() == Res->getNumValues()) &&
532  "Lowering returned the wrong number of results!");
533 
534  // Places new result values base on N result number.
535  for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
536  Results.push_back(Res.getValue(I));
537 
538  return true;
539 }
540 
541 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
542  // For a few operations there is a specific concept for promotion based on
543  // the operand's type.
544  switch (Node->getOpcode()) {
545  case ISD::SINT_TO_FP:
546  case ISD::UINT_TO_FP:
549  // "Promote" the operation by extending the operand.
550  PromoteINT_TO_FP(Node, Results);
551  return;
552  case ISD::FP_TO_UINT:
553  case ISD::FP_TO_SINT:
556  // Promote the operation by extending the operand.
557  PromoteFP_TO_INT(Node, Results);
558  return;
559  case ISD::FP_ROUND:
560  case ISD::FP_EXTEND:
561  // These operations are used to do promotion so they can't be promoted
562  // themselves.
563  llvm_unreachable("Don't know how to promote this operation!");
564  }
565 
566  // There are currently two cases of vector promotion:
567  // 1) Bitcasting a vector of integers to a different type to a vector of the
568  // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
569  // 2) Extending a vector of floats to a vector of the same number of larger
570  // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
571  assert(Node->getNumValues() == 1 &&
572  "Can't promote a vector with multiple results!");
573  MVT VT = Node->getSimpleValueType(0);
574  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
575  SDLoc dl(Node);
576  SmallVector<SDValue, 4> Operands(Node->getNumOperands());
577 
578  for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
579  if (Node->getOperand(j).getValueType().isVector())
580  if (Node->getOperand(j)
581  .getValueType()
582  .getVectorElementType()
583  .isFloatingPoint() &&
585  Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
586  else
587  Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
588  else
589  Operands[j] = Node->getOperand(j);
590  }
591 
592  SDValue Res =
593  DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
594 
595  if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
598  Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res,
599  DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
600  else
601  Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
602 
603  Results.push_back(Res);
604 }
605 
606 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
608  // INT_TO_FP operations may require the input operand be promoted even
609  // when the type is otherwise legal.
610  bool IsStrict = Node->isStrictFPOpcode();
611  MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
612  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
614  "Vectors have different number of elements!");
615 
616  SDLoc dl(Node);
617  SmallVector<SDValue, 4> Operands(Node->getNumOperands());
618 
619  unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
620  Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
623  for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
624  if (Node->getOperand(j).getValueType().isVector())
625  Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
626  else
627  Operands[j] = Node->getOperand(j);
628  }
629 
630  if (IsStrict) {
631  SDValue Res = DAG.getNode(Node->getOpcode(), dl,
632  {Node->getValueType(0), MVT::Other}, Operands);
633  Results.push_back(Res);
634  Results.push_back(Res.getValue(1));
635  return;
636  }
637 
638  SDValue Res =
639  DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
640  Results.push_back(Res);
641 }
642 
643 // For FP_TO_INT we promote the result type to a vector type with wider
644 // elements and then truncate the result. This is different from the default
645 // PromoteVector which uses bitcast to promote thus assumning that the
646 // promoted vector type has the same overall size.
647 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
649  MVT VT = Node->getSimpleValueType(0);
650  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
651  bool IsStrict = Node->isStrictFPOpcode();
653  "Vectors have different number of elements!");
654 
655  unsigned NewOpc = Node->getOpcode();
656  // Change FP_TO_UINT to FP_TO_SINT if possible.
657  // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
658  if (NewOpc == ISD::FP_TO_UINT &&
659  TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
660  NewOpc = ISD::FP_TO_SINT;
661 
662  if (NewOpc == ISD::STRICT_FP_TO_UINT &&
663  TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
664  NewOpc = ISD::STRICT_FP_TO_SINT;
665 
666  SDLoc dl(Node);
667  SDValue Promoted, Chain;
668  if (IsStrict) {
669  Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
670  {Node->getOperand(0), Node->getOperand(1)});
671  Chain = Promoted.getValue(1);
672  } else
673  Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
674 
675  // Assert that the converted value fits in the original type. If it doesn't
676  // (eg: because the value being converted is too big), then the result of the
677  // original operation was undefined anyway, so the assert is still correct.
678  if (Node->getOpcode() == ISD::FP_TO_UINT ||
679  Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
680  NewOpc = ISD::AssertZext;
681  else
682  NewOpc = ISD::AssertSext;
683 
684  Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
685  DAG.getValueType(VT.getScalarType()));
686  Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
687  Results.push_back(Promoted);
688  if (IsStrict)
689  Results.push_back(Chain);
690 }
691 
692 std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
693  LoadSDNode *LD = cast<LoadSDNode>(N);
694  return TLI.scalarizeVectorLoad(LD, DAG);
695 }
696 
697 SDValue VectorLegalizer::ExpandStore(SDNode *N) {
698  StoreSDNode *ST = cast<StoreSDNode>(N);
699  SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
700  return TF;
701 }
702 
703 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
704  switch (Node->getOpcode()) {
705  case ISD::LOAD: {
706  std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
707  Results.push_back(Tmp.first);
708  Results.push_back(Tmp.second);
709  return;
710  }
711  case ISD::STORE:
712  Results.push_back(ExpandStore(Node));
713  return;
714  case ISD::MERGE_VALUES:
715  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
716  Results.push_back(Node->getOperand(i));
717  return;
719  Results.push_back(ExpandSEXTINREG(Node));
720  return;
722  Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
723  return;
725  Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
726  return;
728  Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
729  return;
730  case ISD::BSWAP:
731  Results.push_back(ExpandBSWAP(Node));
732  return;
733  case ISD::VP_BSWAP:
734  Results.push_back(TLI.expandVPBSWAP(Node, DAG));
735  return;
736  case ISD::VSELECT:
737  Results.push_back(ExpandVSELECT(Node));
738  return;
739  case ISD::VP_SELECT:
740  Results.push_back(ExpandVP_SELECT(Node));
741  return;
742  case ISD::VP_SREM:
743  case ISD::VP_UREM:
744  if (SDValue Expanded = ExpandVP_REM(Node)) {
745  Results.push_back(Expanded);
746  return;
747  }
748  break;
749  case ISD::SELECT:
750  Results.push_back(ExpandSELECT(Node));
751  return;
752  case ISD::SELECT_CC: {
753  if (Node->getValueType(0).isScalableVector()) {
754  EVT CondVT = TLI.getSetCCResultType(
755  DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
756  SDValue SetCC =
757  DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),
758  Node->getOperand(1), Node->getOperand(4));
759  Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,
760  Node->getOperand(2),
761  Node->getOperand(3)));
762  return;
763  }
764  break;
765  }
766  case ISD::FP_TO_UINT:
767  ExpandFP_TO_UINT(Node, Results);
768  return;
769  case ISD::UINT_TO_FP:
770  ExpandUINT_TO_FLOAT(Node, Results);
771  return;
772  case ISD::FNEG:
773  Results.push_back(ExpandFNEG(Node));
774  return;
775  case ISD::FSUB:
776  ExpandFSUB(Node, Results);
777  return;
778  case ISD::SETCC:
779  case ISD::VP_SETCC:
780  ExpandSETCC(Node, Results);
781  return;
782  case ISD::ABS:
783  if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
784  Results.push_back(Expanded);
785  return;
786  }
787  break;
788  case ISD::BITREVERSE:
789  ExpandBITREVERSE(Node, Results);
790  return;
791  case ISD::CTPOP:
792  if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
793  Results.push_back(Expanded);
794  return;
795  }
796  break;
797  case ISD::CTLZ:
799  if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
800  Results.push_back(Expanded);
801  return;
802  }
803  break;
804  case ISD::CTTZ:
806  if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
807  Results.push_back(Expanded);
808  return;
809  }
810  break;
811  case ISD::FSHL:
812  case ISD::FSHR:
813  if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
814  Results.push_back(Expanded);
815  return;
816  }
817  break;
818  case ISD::ROTL:
819  case ISD::ROTR:
820  if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
821  Results.push_back(Expanded);
822  return;
823  }
824  break;
825  case ISD::FMINNUM:
826  case ISD::FMAXNUM:
827  if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
828  Results.push_back(Expanded);
829  return;
830  }
831  break;
832  case ISD::SMIN:
833  case ISD::SMAX:
834  case ISD::UMIN:
835  case ISD::UMAX:
836  if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
837  Results.push_back(Expanded);
838  return;
839  }
840  break;
841  case ISD::UADDO:
842  case ISD::USUBO:
843  ExpandUADDSUBO(Node, Results);
844  return;
845  case ISD::SADDO:
846  case ISD::SSUBO:
847  ExpandSADDSUBO(Node, Results);
848  return;
849  case ISD::UMULO:
850  case ISD::SMULO:
851  ExpandMULO(Node, Results);
852  return;
853  case ISD::USUBSAT:
854  case ISD::SSUBSAT:
855  case ISD::UADDSAT:
856  case ISD::SADDSAT:
857  if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
858  Results.push_back(Expanded);
859  return;
860  }
861  break;
862  case ISD::USHLSAT:
863  case ISD::SSHLSAT:
864  if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
865  Results.push_back(Expanded);
866  return;
867  }
868  break;
869  case ISD::FP_TO_SINT_SAT:
870  case ISD::FP_TO_UINT_SAT:
871  // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
872  if (Node->getValueType(0).isScalableVector()) {
873  if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {
874  Results.push_back(Expanded);
875  return;
876  }
877  }
878  break;
879  case ISD::SMULFIX:
880  case ISD::UMULFIX:
881  if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
882  Results.push_back(Expanded);
883  return;
884  }
885  break;
886  case ISD::SMULFIXSAT:
887  case ISD::UMULFIXSAT:
888  // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
889  // why. Maybe it results in worse codegen compared to the unroll for some
890  // targets? This should probably be investigated. And if we still prefer to
891  // unroll an explanation could be helpful.
892  break;
893  case ISD::SDIVFIX:
894  case ISD::UDIVFIX:
895  ExpandFixedPointDiv(Node, Results);
896  return;
897  case ISD::SDIVFIXSAT:
898  case ISD::UDIVFIXSAT:
899  break;
900 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
901  case ISD::STRICT_##DAGN:
902 #include "llvm/IR/ConstrainedOps.def"
903  ExpandStrictFPOp(Node, Results);
904  return;
905  case ISD::VECREDUCE_ADD:
906  case ISD::VECREDUCE_MUL:
907  case ISD::VECREDUCE_AND:
908  case ISD::VECREDUCE_OR:
909  case ISD::VECREDUCE_XOR:
910  case ISD::VECREDUCE_SMAX:
911  case ISD::VECREDUCE_SMIN:
912  case ISD::VECREDUCE_UMAX:
913  case ISD::VECREDUCE_UMIN:
914  case ISD::VECREDUCE_FADD:
915  case ISD::VECREDUCE_FMUL:
916  case ISD::VECREDUCE_FMAX:
917  case ISD::VECREDUCE_FMIN:
918  Results.push_back(TLI.expandVecReduce(Node, DAG));
919  return;
922  Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
923  return;
924  case ISD::SREM:
925  case ISD::UREM:
926  ExpandREM(Node, Results);
927  return;
928  case ISD::VP_MERGE:
929  Results.push_back(ExpandVP_MERGE(Node));
930  return;
931  }
932 
933  Results.push_back(DAG.UnrollVectorOp(Node));
934 }
935 
936 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
937  // Lower a select instruction where the condition is a scalar and the
938  // operands are vectors. Lower this select to VSELECT and implement it
939  // using XOR AND OR. The selector bit is broadcasted.
940  EVT VT = Node->getValueType(0);
941  SDLoc DL(Node);
942 
943  SDValue Mask = Node->getOperand(0);
944  SDValue Op1 = Node->getOperand(1);
945  SDValue Op2 = Node->getOperand(2);
946 
947  assert(VT.isVector() && !Mask.getValueType().isVector()
948  && Op1.getValueType() == Op2.getValueType() && "Invalid type");
949 
950  // If we can't even use the basic vector operations of
951  // AND,OR,XOR, we will have to scalarize the op.
952  // Notice that the operation may be 'promoted' which means that it is
953  // 'bitcasted' to another type which is handled.
954  // Also, we need to be able to construct a splat vector using either
955  // BUILD_VECTOR or SPLAT_VECTOR.
956  // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
957  // BUILD_VECTOR?
958  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
959  TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
960  TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
961  TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
963  VT) == TargetLowering::Expand)
964  return DAG.UnrollVectorOp(Node);
965 
966  // Generate a mask operand.
968 
969  // What is the size of each element in the vector mask.
970  EVT BitTy = MaskTy.getScalarType();
971 
972  Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
973  DAG.getConstant(0, DL, BitTy));
974 
975  // Broadcast the mask so that the entire vector is all one or all zero.
976  Mask = DAG.getSplat(MaskTy, DL, Mask);
977 
978  // Bitcast the operands to be the same type as the mask.
979  // This is needed when we select between FP types because
980  // the mask is a vector of integers.
981  Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
982  Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
983 
984  SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
985 
986  Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
987  Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
988  SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
989  return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
990 }
991 
992 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
993  EVT VT = Node->getValueType(0);
994 
995  // Make sure that the SRA and SHL instructions are available.
996  if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
997  TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
998  return DAG.UnrollVectorOp(Node);
999 
1000  SDLoc DL(Node);
1001  EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
1002 
1003  unsigned BW = VT.getScalarSizeInBits();
1004  unsigned OrigBW = OrigTy.getScalarSizeInBits();
1005  SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
1006 
1007  SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
1008  return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
1009 }
1010 
1011 // Generically expand a vector anyext in register to a shuffle of the relevant
1012 // lanes into the appropriate locations, with other lanes left undef.
1013 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1014  SDLoc DL(Node);
1015  EVT VT = Node->getValueType(0);
1016  int NumElements = VT.getVectorNumElements();
1017  SDValue Src = Node->getOperand(0);
1018  EVT SrcVT = Src.getValueType();
1019  int NumSrcElements = SrcVT.getVectorNumElements();
1020 
1021  // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1022  // into a larger vector type.
1023  if (SrcVT.bitsLE(VT)) {
1024  assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1025  "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1026  NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1027  SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1028  NumSrcElements);
1029  Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1030  Src, DAG.getVectorIdxConstant(0, DL));
1031  }
1032 
1033  // Build a base mask of undef shuffles.
1034  SmallVector<int, 16> ShuffleMask;
1035  ShuffleMask.resize(NumSrcElements, -1);
1036 
1037  // Place the extended lanes into the correct locations.
1038  int ExtLaneScale = NumSrcElements / NumElements;
1039  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1040  for (int i = 0; i < NumElements; ++i)
1041  ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1042 
1043  return DAG.getNode(
1044  ISD::BITCAST, DL, VT,
1045  DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1046 }
1047 
1048 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1049  SDLoc DL(Node);
1050  EVT VT = Node->getValueType(0);
1051  SDValue Src = Node->getOperand(0);
1052  EVT SrcVT = Src.getValueType();
1053 
1054  // First build an any-extend node which can be legalized above when we
1055  // recurse through it.
1056  SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1057 
1058  // Now we need sign extend. Do this by shifting the elements. Even if these
1059  // aren't legal operations, they have a better chance of being legalized
1060  // without full scalarization than the sign extension does.
1061  unsigned EltWidth = VT.getScalarSizeInBits();
1062  unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1063  SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1064  return DAG.getNode(ISD::SRA, DL, VT,
1065  DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1066  ShiftAmount);
1067 }
1068 
1069 // Generically expand a vector zext in register to a shuffle of the relevant
1070 // lanes into the appropriate locations, a blend of zero into the high bits,
1071 // and a bitcast to the wider element type.
1072 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1073  SDLoc DL(Node);
1074  EVT VT = Node->getValueType(0);
1075  int NumElements = VT.getVectorNumElements();
1076  SDValue Src = Node->getOperand(0);
1077  EVT SrcVT = Src.getValueType();
1078  int NumSrcElements = SrcVT.getVectorNumElements();
1079 
1080  // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1081  // into a larger vector type.
1082  if (SrcVT.bitsLE(VT)) {
1083  assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1084  "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1085  NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1086  SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1087  NumSrcElements);
1088  Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1089  Src, DAG.getVectorIdxConstant(0, DL));
1090  }
1091 
1092  // Build up a zero vector to blend into this one.
1093  SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1094 
1095  // Shuffle the incoming lanes into the correct position, and pull all other
1096  // lanes from the zero vector.
1097  auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1098 
1099  int ExtLaneScale = NumSrcElements / NumElements;
1100  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1101  for (int i = 0; i < NumElements; ++i)
1102  ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1103 
1104  return DAG.getNode(ISD::BITCAST, DL, VT,
1105  DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1106 }
1107 
1108 static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1109  int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1110  for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1111  for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1112  ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1113 }
1114 
1115 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1116  EVT VT = Node->getValueType(0);
1117 
1118  // Scalable vectors can't use shuffle expansion.
1119  if (VT.isScalableVector())
1120  return TLI.expandBSWAP(Node, DAG);
1121 
1122  // Generate a byte wise shuffle mask for the BSWAP.
1123  SmallVector<int, 16> ShuffleMask;
1124  createBSWAPShuffleMask(VT, ShuffleMask);
1125  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1126 
1127  // Only emit a shuffle if the mask is legal.
1128  if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1129  SDLoc DL(Node);
1130  SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1131  Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1132  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1133  }
1134 
1135  // If we have the appropriate vector bit operations, it is better to use them
1136  // than unrolling and expanding each component.
1137  if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1138  TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1139  TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1140  TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1141  return TLI.expandBSWAP(Node, DAG);
1142 
1143  // Otherwise unroll.
1144  return DAG.UnrollVectorOp(Node);
1145 }
1146 
1147 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1149  EVT VT = Node->getValueType(0);
1150 
1151  // We can't unroll or use shuffles for scalable vectors.
1152  if (VT.isScalableVector()) {
1153  Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1154  return;
1155  }
1156 
1157  // If we have the scalar operation, it's probably cheaper to unroll it.
1158  if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1159  SDValue Tmp = DAG.UnrollVectorOp(Node);
1160  Results.push_back(Tmp);
1161  return;
1162  }
1163 
1164  // If the vector element width is a whole number of bytes, test if its legal
1165  // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1166  // vector. This greatly reduces the number of bit shifts necessary.
1167  unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1168  if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1169  SmallVector<int, 16> BSWAPMask;
1170  createBSWAPShuffleMask(VT, BSWAPMask);
1171 
1172  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1173  if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1174  (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1175  (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1176  TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1177  TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1178  TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1179  SDLoc DL(Node);
1180  SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1181  Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1182  BSWAPMask);
1183  Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1184  Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1185  Results.push_back(Op);
1186  return;
1187  }
1188  }
1189 
1190  // If we have the appropriate vector bit operations, it is better to use them
1191  // than unrolling and expanding each component.
1192  if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1193  TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1194  TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1195  TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1196  Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1197  return;
1198  }
1199 
1200  // Otherwise unroll.
1201  SDValue Tmp = DAG.UnrollVectorOp(Node);
1202  Results.push_back(Tmp);
1203 }
1204 
1205 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1206  // Implement VSELECT in terms of XOR, AND, OR
1207  // on platforms which do not support blend natively.
1208  SDLoc DL(Node);
1209 
1210  SDValue Mask = Node->getOperand(0);
1211  SDValue Op1 = Node->getOperand(1);
1212  SDValue Op2 = Node->getOperand(2);
1213 
1214  EVT VT = Mask.getValueType();
1215 
1216  // If we can't even use the basic vector operations of
1217  // AND,OR,XOR, we will have to scalarize the op.
1218  // Notice that the operation may be 'promoted' which means that it is
1219  // 'bitcasted' to another type which is handled.
1220  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1221  TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1222  TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1223  return DAG.UnrollVectorOp(Node);
1224 
1225  // This operation also isn't safe with AND, OR, XOR when the boolean type is
1226  // 0/1 and the select operands aren't also booleans, as we need an all-ones
1227  // vector constant to mask with.
1228  // FIXME: Sign extend 1 to all ones if that's legal on the target.
1229  auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1231  !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1233  return DAG.UnrollVectorOp(Node);
1234 
1235  // If the mask and the type are different sizes, unroll the vector op. This
1236  // can occur when getSetCCResultType returns something that is different in
1237  // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1238  if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1239  return DAG.UnrollVectorOp(Node);
1240 
1241  // Bitcast the operands to be the same type as the mask.
1242  // This is needed when we select between FP types because
1243  // the mask is a vector of integers.
1244  Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1245  Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1246 
1247  SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1248 
1249  Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1250  Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1251  SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1252  return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1253 }
1254 
1255 SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1256  // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1257  // do not support it natively.
1258  SDLoc DL(Node);
1259 
1260  SDValue Mask = Node->getOperand(0);
1261  SDValue Op1 = Node->getOperand(1);
1262  SDValue Op2 = Node->getOperand(2);
1263  SDValue EVL = Node->getOperand(3);
1264 
1265  EVT VT = Mask.getValueType();
1266 
1267  // If we can't even use the basic vector operations of
1268  // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1269  if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1270  TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1271  TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1272  return DAG.UnrollVectorOp(Node);
1273 
1274  // This operation also isn't safe when the operands aren't also booleans.
1275  if (Op1.getValueType().getVectorElementType() != MVT::i1)
1276  return DAG.UnrollVectorOp(Node);
1277 
1278  SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1279  SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Mask, EVL);
1280 
1281  Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Mask, EVL);
1282  Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Mask, EVL);
1283  return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Mask, EVL);
1284 }
1285 
1286 SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1287  // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1288  // indices less than the EVL/pivot are true. Combine that with the original
1289  // mask for a full-length mask. Use a full-length VSELECT to select between
1290  // the true and false values.
1291  SDLoc DL(Node);
1292 
1293  SDValue Mask = Node->getOperand(0);
1294  SDValue Op1 = Node->getOperand(1);
1295  SDValue Op2 = Node->getOperand(2);
1296  SDValue EVL = Node->getOperand(3);
1297 
1298  EVT MaskVT = Mask.getValueType();
1299  bool IsFixedLen = MaskVT.isFixedLengthVector();
1300 
1301  EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1302  MaskVT.getVectorElementCount());
1303 
1304  // If we can't construct the EVL mask efficiently, it's better to unroll.
1305  if ((IsFixedLen &&
1306  !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1307  (!IsFixedLen &&
1308  (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1309  !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1310  return DAG.UnrollVectorOp(Node);
1311 
1312  // If using a SETCC would result in a different type than the mask type,
1313  // unroll.
1314  if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1315  EVLVecVT) != MaskVT)
1316  return DAG.UnrollVectorOp(Node);
1317 
1318  SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1319  SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);
1320  SDValue EVLMask =
1321  DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1322 
1323  SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1324  return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1325 }
1326 
1327 SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1328  // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.
1329  EVT VT = Node->getValueType(0);
1330 
1331  unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1332 
1333  if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1334  !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1335  !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1336  return SDValue();
1337 
1338  SDLoc DL(Node);
1339 
1340  SDValue Dividend = Node->getOperand(0);
1341  SDValue Divisor = Node->getOperand(1);
1342  SDValue Mask = Node->getOperand(2);
1343  SDValue EVL = Node->getOperand(3);
1344 
1345  // X % Y -> X-X/Y*Y
1346  SDValue Div = DAG.getNode(DivOpc, DL, VT, Dividend, Divisor, Mask, EVL);
1347  SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL);
1348  return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL);
1349 }
1350 
1351 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1353  // Attempt to expand using TargetLowering.
1354  SDValue Result, Chain;
1355  if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1356  Results.push_back(Result);
1357  if (Node->isStrictFPOpcode())
1358  Results.push_back(Chain);
1359  return;
1360  }
1361 
1362  // Otherwise go ahead and unroll.
1363  if (Node->isStrictFPOpcode()) {
1364  UnrollStrictFPOp(Node, Results);
1365  return;
1366  }
1367 
1368  Results.push_back(DAG.UnrollVectorOp(Node));
1369 }
1370 
1371 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1373  bool IsStrict = Node->isStrictFPOpcode();
1374  unsigned OpNo = IsStrict ? 1 : 0;
1375  SDValue Src = Node->getOperand(OpNo);
1376  EVT VT = Src.getValueType();
1377  SDLoc DL(Node);
1378 
1379  // Attempt to expand using TargetLowering.
1380  SDValue Result;
1381  SDValue Chain;
1382  if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1383  Results.push_back(Result);
1384  if (IsStrict)
1385  Results.push_back(Chain);
1386  return;
1387  }
1388 
1389  // Make sure that the SINT_TO_FP and SRL instructions are available.
1390  if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1392  (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1394  TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1395  if (IsStrict) {
1396  UnrollStrictFPOp(Node, Results);
1397  return;
1398  }
1399 
1400  Results.push_back(DAG.UnrollVectorOp(Node));
1401  return;
1402  }
1403 
1404  unsigned BW = VT.getScalarSizeInBits();
1405  assert((BW == 64 || BW == 32) &&
1406  "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1407 
1408  SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1409 
1410  // Constants to clear the upper part of the word.
1411  // Notice that we can also use SHL+SHR, but using a constant is slightly
1412  // faster on x86.
1413  uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1414  SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1415 
1416  // Two to the power of half-word-size.
1417  SDValue TWOHW =
1418  DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1419 
1420  // Clear upper part of LO, lower HI
1421  SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1422  SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1423 
1424  if (IsStrict) {
1425  // Convert hi and lo to floats
1426  // Convert the hi part back to the upper values
1427  // TODO: Can any fast-math-flags be set on these nodes?
1429  {Node->getValueType(0), MVT::Other},
1430  {Node->getOperand(0), HI});
1431  fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1432  {fHI.getValue(1), fHI, TWOHW});
1434  {Node->getValueType(0), MVT::Other},
1435  {Node->getOperand(0), LO});
1436 
1438  fLO.getValue(1));
1439 
1440  // Add the two halves
1441  SDValue Result =
1442  DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1443  {TF, fHI, fLO});
1444 
1445  Results.push_back(Result);
1446  Results.push_back(Result.getValue(1));
1447  return;
1448  }
1449 
1450  // Convert hi and lo to floats
1451  // Convert the hi part back to the upper values
1452  // TODO: Can any fast-math-flags be set on these nodes?
1453  SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1454  fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1455  SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1456 
1457  // Add the two halves
1458  Results.push_back(
1459  DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1460 }
1461 
1462 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1463  if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1464  SDLoc DL(Node);
1465  SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1466  // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1467  return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1468  Node->getOperand(0));
1469  }
1470  return DAG.UnrollVectorOp(Node);
1471 }
1472 
1473 void VectorLegalizer::ExpandFSUB(SDNode *Node,
1475  // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1476  // we can defer this to operation legalization where it will be lowered as
1477  // a+(-b).
1478  EVT VT = Node->getValueType(0);
1479  if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1480  TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1481  return; // Defer to LegalizeDAG
1482 
1483  SDValue Tmp = DAG.UnrollVectorOp(Node);
1484  Results.push_back(Tmp);
1485 }
1486 
1487 void VectorLegalizer::ExpandSETCC(SDNode *Node,
1489  bool NeedInvert = false;
1490  bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
1491  SDLoc dl(Node);
1492  MVT OpVT = Node->getOperand(0).getSimpleValueType();
1493  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
1494 
1495  if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1496  Results.push_back(UnrollVSETCC(Node));
1497  return;
1498  }
1499 
1500  SDValue Chain;
1501  SDValue LHS = Node->getOperand(0);
1502  SDValue RHS = Node->getOperand(1);
1503  SDValue CC = Node->getOperand(2);
1504  SDValue Mask, EVL;
1505  if (IsVP) {
1506  Mask = Node->getOperand(3);
1507  EVL = Node->getOperand(4);
1508  }
1509 
1510  bool Legalized =
1511  TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
1512  EVL, NeedInvert, dl, Chain);
1513 
1514  if (Legalized) {
1515  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1516  // condition code, create a new SETCC node.
1517  if (CC.getNode()) {
1518  if (!IsVP)
1519  LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1520  Node->getFlags());
1521  else
1522  LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
1523  {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
1524  }
1525 
1526  // If we expanded the SETCC by inverting the condition code, then wrap
1527  // the existing SETCC in a NOT to restore the intended condition.
1528  if (NeedInvert) {
1529  if (!IsVP)
1530  LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1531  else
1532  LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
1533  }
1534  } else {
1535  // Otherwise, SETCC for the given comparison type must be completely
1536  // illegal; expand it into a SELECT_CC.
1537  EVT VT = Node->getValueType(0);
1538  LHS =
1539  DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1540  DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1541  DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1542  LHS->setFlags(Node->getFlags());
1543  }
1544 
1545  Results.push_back(LHS);
1546 }
1547 
1548 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1550  SDValue Result, Overflow;
1551  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1552  Results.push_back(Result);
1553  Results.push_back(Overflow);
1554 }
1555 
1556 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1558  SDValue Result, Overflow;
1559  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1560  Results.push_back(Result);
1561  Results.push_back(Overflow);
1562 }
1563 
1564 void VectorLegalizer::ExpandMULO(SDNode *Node,
1566  SDValue Result, Overflow;
1567  if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1568  std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1569 
1570  Results.push_back(Result);
1571  Results.push_back(Overflow);
1572 }
1573 
1574 void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1576  SDNode *N = Node;
1577  if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1578  N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1579  Results.push_back(Expanded);
1580 }
1581 
1582 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1584  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1585  ExpandUINT_TO_FLOAT(Node, Results);
1586  return;
1587  }
1588  if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1589  ExpandFP_TO_UINT(Node, Results);
1590  return;
1591  }
1592 
1593  UnrollStrictFPOp(Node, Results);
1594 }
1595 
1596 void VectorLegalizer::ExpandREM(SDNode *Node,
1598  assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1599  "Expected REM node");
1600 
1601  SDValue Result;
1602  if (!TLI.expandREM(Node, Result, DAG))
1603  Result = DAG.UnrollVectorOp(Node);
1604  Results.push_back(Result);
1605 }
1606 
1607 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1609  EVT VT = Node->getValueType(0);
1610  EVT EltVT = VT.getVectorElementType();
1611  unsigned NumElems = VT.getVectorNumElements();
1612  unsigned NumOpers = Node->getNumOperands();
1613  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1614 
1615  EVT TmpEltVT = EltVT;
1616  if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1617  Node->getOpcode() == ISD::STRICT_FSETCCS)
1618  TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1619  *DAG.getContext(), TmpEltVT);
1620 
1621  EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1622  SDValue Chain = Node->getOperand(0);
1623  SDLoc dl(Node);
1624 
1625  SmallVector<SDValue, 32> OpValues;
1626  SmallVector<SDValue, 32> OpChains;
1627  for (unsigned i = 0; i < NumElems; ++i) {
1629  SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1630 
1631  // The Chain is the first operand.
1632  Opers.push_back(Chain);
1633 
1634  // Now process the remaining operands.
1635  for (unsigned j = 1; j < NumOpers; ++j) {
1636  SDValue Oper = Node->getOperand(j);
1637  EVT OperVT = Oper.getValueType();
1638 
1639  if (OperVT.isVector())
1640  Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1641  OperVT.getVectorElementType(), Oper, Idx);
1642 
1643  Opers.push_back(Oper);
1644  }
1645 
1646  SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1647  SDValue ScalarResult = ScalarOp.getValue(0);
1648  SDValue ScalarChain = ScalarOp.getValue(1);
1649 
1650  if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1651  Node->getOpcode() == ISD::STRICT_FSETCCS)
1652  ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1653  DAG.getAllOnesConstant(dl, EltVT),
1654  DAG.getConstant(0, dl, EltVT));
1655 
1656  OpValues.push_back(ScalarResult);
1657  OpChains.push_back(ScalarChain);
1658  }
1659 
1660  SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1661  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1662 
1663  Results.push_back(Result);
1664  Results.push_back(NewChain);
1665 }
1666 
1667 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1668  EVT VT = Node->getValueType(0);
1669  unsigned NumElems = VT.getVectorNumElements();
1670  EVT EltVT = VT.getVectorElementType();
1671  SDValue LHS = Node->getOperand(0);
1672  SDValue RHS = Node->getOperand(1);
1673  SDValue CC = Node->getOperand(2);
1674  EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1675  SDLoc dl(Node);
1676  SmallVector<SDValue, 8> Ops(NumElems);
1677  for (unsigned i = 0; i < NumElems; ++i) {
1678  SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1679  DAG.getVectorIdxConstant(i, dl));
1680  SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1681  DAG.getVectorIdxConstant(i, dl));
1682  Ops[i] = DAG.getNode(ISD::SETCC, dl,
1683  TLI.getSetCCResultType(DAG.getDataLayout(),
1684  *DAG.getContext(), TmpEltVT),
1685  LHSElem, RHSElem, CC);
1686  Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1687  DAG.getConstant(0, dl, EltVT));
1688  }
1689  return DAG.getBuildVector(VT, dl, Ops);
1690 }
1691 
1693  return VectorLegalizer(*this).Run();
1694 }
llvm::ISD::SUB
@ SUB
Definition: ISDOpcodes.h:240
llvm::ISD::FPOWI
@ FPOWI
Definition: ISDOpcodes.h:917
llvm::ISD::FROUNDEVEN
@ FROUNDEVEN
Definition: ISDOpcodes.h:929
i
i
Definition: README.txt:29
ValueTypes.h
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
llvm::AArch64CC::LO
@ LO
Definition: AArch64BaseInfo.h:258
llvm::MVT::getVectorElementType
MVT getVectorElementType() const
Definition: MachineValueType.h:542
llvm::AArch64CC::HI
@ HI
Definition: AArch64BaseInfo.h:263
llvm::ISD::UMULO
@ UMULO
Definition: ISDOpcodes.h:332
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::TargetLoweringBase::Legal
@ Legal
Definition: TargetLowering.h:196
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:886
llvm::EVT::getVectorElementCount
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:322
llvm::ISD::NON_EXTLOAD
@ NON_EXTLOAD
Definition: ISDOpcodes.h:1404
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:943
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::ISD::UDIVFIXSAT
@ UDIVFIXSAT
Definition: ISDOpcodes.h:387
T
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:700
llvm::ISD::UDIV
@ UDIV
Definition: ISDOpcodes.h:243
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
llvm::ISD::UADDO
@ UADDO
Definition: ISDOpcodes.h:324
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:708
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ISD::FSHL
@ FSHL
Definition: ISDOpcodes.h:696
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:839
llvm::MVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: MachineValueType.h:386
ErrorHandling.h
llvm::ISD::SDIVFIX
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
llvm::ISD::MGATHER
@ MGATHER
Definition: ISDOpcodes.h:1219
llvm::ISD::STEP_VECTOR
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:632
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::ISD::FLOG2
@ FLOG2
Definition: ISDOpcodes.h:920
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
llvm::ISD::USUBSAT
@ USUBSAT
Definition: ISDOpcodes.h:350
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:482
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:819
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2344
llvm::ISD::USHLSAT
@ USHLSAT
Definition: ISDOpcodes.h:360
llvm::EVT::isScalableVector
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:160
DenseMap.h
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
llvm::EVT::getVectorVT
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1275
llvm::ISD::FMAXNUM_IEEE
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:951
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:772
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
llvm::ISD::SIGN_EXTEND_VECTOR_INREG
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:803
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1259
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::TargetLoweringBase::LegalizeAction
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
Definition: TargetLowering.h:195
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:280
SelectionDAG.h
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:442
llvm::ISD::SMAX
@ SMAX
Definition: ISDOpcodes.h:661
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::ISD::FABS
@ FABS
Definition: ISDOpcodes.h:912
MachineValueType.h
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:694
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ISD::VECREDUCE_UMAX
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1287
llvm::ISD::FFLOOR
@ FFLOOR
Definition: ISDOpcodes.h:930
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::ISD::LoadExtType
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1404
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
TargetLowering.h
llvm::ISD::FSHR
@ FSHR
Definition: ISDOpcodes.h:697
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:441
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
llvm::ISD::CTLZ
@ CTLZ
Definition: ISDOpcodes.h:702
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:28
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
SelectionDAGNodes.h
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:713
llvm::ISD::SMULFIXSAT
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:763
llvm::ISD::ABS
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:674
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::ISD::SMULFIX
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:781
llvm::SelectionDAG::getTargetLoweringInfo
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:475
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::ISD::ANY_EXTEND_VECTOR_INREG
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:792
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3506
llvm::ISD::FROUND
@ FROUND
Definition: ISDOpcodes.h:928
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:769
llvm::ISD::SRA
@ SRA
Definition: ISDOpcodes.h:692
llvm::TargetLoweringBase::ZeroOrNegativeOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
Definition: TargetLowering.h:233
llvm::ISD::FMINNUM_IEEE
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:950
llvm::ISD::UDIVREM
@ UDIVREM
Definition: ISDOpcodes.h:256
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:773
llvm::ISD::FNEARBYINT
@ FNEARBYINT
Definition: ISDOpcodes.h:927
llvm::ISD::FRINT
@ FRINT
Definition: ISDOpcodes.h:926
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:642
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::ISD::SMULO
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
llvm::ISD::SPLAT_VECTOR
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:613
llvm::EVT::changeVectorElementTypeToInteger
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:199
llvm::ISD::USUBO
@ USUBO
Definition: ISDOpcodes.h:328
llvm::MVT::getScalarType
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Definition: MachineValueType.h:538
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1273
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::MVT::isFloatingPoint
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: MachineValueType.h:360
llvm::ISD::FPOW
@ FPOW
Definition: ISDOpcodes.h:918
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
llvm::ISD::SMIN
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:660
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:956
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:264
llvm::ISD::SADDO
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
llvm::ISD::FLOG10
@ FLOG10
Definition: ISDOpcodes.h:921
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1276
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
uint64_t
llvm::ISD::FP_TO_UINT
@ FP_TO_UINT
Definition: ISDOpcodes.h:820
llvm::ISD::VECREDUCE_ADD
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1280
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:966
llvm::ISD::VECREDUCE_SMAX
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1285
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::TargetLoweringBase::Promote
@ Promote
Definition: TargetLowering.h:197
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::DenseMap
Definition: DenseMap.h:714
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1272
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:709
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1260
llvm::ISD::UADDSAT
@ UADDSAT
Definition: ISDOpcodes.h:341
llvm::ISD::SSUBSAT
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:492
llvm::MVT::getVectorNumElements
unsigned getVectorNumElements() const
Definition: MachineValueType.h:905
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2372
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:46
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::TargetLoweringBase::getSetCCResultType
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
Definition: TargetLoweringBase.cpp:1532
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::ISD::MULHS
@ MULHS
Definition: ISDOpcodes.h:638
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::ISD::SETULT
@ SETULT
Definition: ISDOpcodes.h:1438
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1424
llvm::ISD::SDIVFIXSAT
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
llvm::ISD::VECREDUCE_AND
@ VECREDUCE_AND
Definition: ISDOpcodes.h:1282
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::ISD::FMAXIMUM
@ FMAXIMUM
Definition: ISDOpcodes.h:957
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:154
llvm::ISD::UMAX
@ UMAX
Definition: ISDOpcodes.h:663
Mul
BinaryOperator * Mul
Definition: X86PartialReduction.cpp:70
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
DataLayout.h
llvm::EVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:352
llvm::logicalview::LVAttributeKind::Zero
@ Zero
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:448
llvm::ISD::SREM
@ SREM
Definition: ISDOpcodes.h:244
llvm::ISD::UMUL_LOHI
@ UMUL_LOHI
Definition: ISDOpcodes.h:251
Compiler.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ISD::FEXP
@ FEXP
Definition: ISDOpcodes.h:922
CC
auto CC
Definition: RISCVRedundantCopyElimination.cpp:79
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
llvm::ISD::UMULFIX
@ UMULFIX
Definition: ISDOpcodes.h:368
llvm::ISD::FEXP2
@ FEXP2
Definition: ISDOpcodes.h:923
llvm::ISD::VECREDUCE_XOR
@ VECREDUCE_XOR
Definition: ISDOpcodes.h:1284
llvm::ISD::FMUL
@ FMUL
Definition: ISDOpcodes.h:392
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::ISD::SSHLSAT
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
createBSWAPShuffleMask
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
Definition: LegalizeVectorOps.cpp:1108
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:668
llvm::ISD::FSQRT
@ FSQRT
Definition: ISDOpcodes.h:913
llvm::ISD::INSERT_SUBVECTOR
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:558
j
return j(j<< 16)
llvm::ISD::STRICT_FMUL
@ STRICT_FMUL
Definition: ISDOpcodes.h:402
llvm::ISD::FMAXNUM
@ FMAXNUM
Definition: ISDOpcodes.h:944
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:838
llvm::ISD::VECREDUCE_MUL
@ VECREDUCE_MUL
Definition: ISDOpcodes.h:1281
llvm::EVT::getScalarType
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:295
llvm::ISD::UMULFIXSAT
@ UMULFIXSAT
Definition: ISDOpcodes.h:374
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::ISD::FCOS
@ FCOS
Definition: ISDOpcodes.h:916
llvm::ISD::FCEIL
@ FCEIL
Definition: ISDOpcodes.h:924
llvm::ISD::FSIN
@ FSIN
Definition: ISDOpcodes.h:915
ISDOpcodes.h
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
Casting.h
llvm::TargetLoweringBase::Custom
@ Custom
Definition: TargetLowering.h:200
llvm::ISD::VECREDUCE_OR
@ VECREDUCE_OR
Definition: ISDOpcodes.h:1283
llvm::ISD::SDIV
@ SDIV
Definition: ISDOpcodes.h:242
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:983
llvm::SelectionDAG::LegalizeVectors
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
Definition: LegalizeVectorOps.cpp:1692
llvm::TargetLoweringBase::ZeroOrOneBooleanContent
@ ZeroOrOneBooleanContent
Definition: TargetLowering.h:232
llvm::ISD::STORE
@ STORE
Definition: ISDOpcodes.h:967
llvm::ISD::UINT_TO_FP
@ UINT_TO_FP
Definition: ISDOpcodes.h:774
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::ISD::SSUBO
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:871
llvm::ISD::FSUB
@ FSUB
Definition: ISDOpcodes.h:391
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:691
SmallVector.h
llvm::ISD::FREM
@ FREM
Definition: ISDOpcodes.h:394
llvm::ISD::MUL
@ MUL
Definition: ISDOpcodes.h:241
llvm::ISD::UREM
@ UREM
Definition: ISDOpcodes.h:245
llvm::TargetLoweringBase::Expand
@ Expand
Definition: TargetLowering.h:198
llvm::ISD::ZERO_EXTEND_VECTOR_INREG
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:814
N
#define N
llvm::ISD::BITREVERSE
@ BITREVERSE
Definition: ISDOpcodes.h:704
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:693
llvm::ISD::CTTZ
@ CTTZ
Definition: ISDOpcodes.h:701
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
llvm::ISD::UMIN
@ UMIN
Definition: ISDOpcodes.h:662
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:637
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:911
llvm::ISD::UDIVFIX
@ UDIVFIX
Definition: ISDOpcodes.h:381
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::ISD::VECREDUCE_SMIN
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1286
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:760
llvm::ISD::VECREDUCE_UMIN
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1288
llvm::ISD::FTRUNC
@ FTRUNC
Definition: ISDOpcodes.h:925
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:499
llvm::ISD::FLOG
@ FLOG
Definition: ISDOpcodes.h:919
llvm::EVT::isFixedLengthVector
bool isFixedLengthVector() const
Definition: ValueTypes.h:164
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:852
llvm::ISD::ROTR
@ ROTR
Definition: ISDOpcodes.h:695
Debug.h
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:703
llvm::ISD::SADDSAT
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::Function::size
size_t size() const
Definition: Function.h:712
llvm::ISD::VSELECT
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:722
llvm::ISD::FDIV
@ FDIV
Definition: ISDOpcodes.h:393