LLVM 19.0.0git
LegalizeVectorOps.cpp
Go to the documentation of this file.
1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
37#include "llvm/IR/DataLayout.h"
40#include "llvm/Support/Debug.h"
42#include <cassert>
43#include <cstdint>
44#include <iterator>
45#include <utility>
46
47using namespace llvm;
48
49#define DEBUG_TYPE "legalizevectorops"
50
51namespace {
52
53class VectorLegalizer {
54 SelectionDAG& DAG;
55 const TargetLowering &TLI;
56 bool Changed = false; // Keep track of whether anything changed
57
58 /// For nodes that are of legal width, and that have more than one use, this
59 /// map indicates what regularized operand to use. This allows us to avoid
60 /// legalizing the same thing more than once.
62
63 /// Adds a node to the translation cache.
64 void AddLegalizedOperand(SDValue From, SDValue To) {
65 LegalizedNodes.insert(std::make_pair(From, To));
66 // If someone requests legalization of the new node, return itself.
67 if (From != To)
68 LegalizedNodes.insert(std::make_pair(To, To));
69 }
70
71 /// Legalizes the given node.
72 SDValue LegalizeOp(SDValue Op);
73
74 /// Assuming the node is legal, "legalize" the results.
75 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
76
77 /// Make sure Results are legal and update the translation cache.
78 SDValue RecursivelyLegalizeResults(SDValue Op,
80
81 /// Wrapper to interface LowerOperation with a vector of Results.
82 /// Returns false if the target wants to use default expansion. Otherwise
83 /// returns true. If return is true and the Results are empty, then the
84 /// target wants to keep the input node as is.
85 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
86
87 /// Implements unrolling a VSETCC.
88 SDValue UnrollVSETCC(SDNode *Node);
89
90 /// Implement expand-based legalization of vector operations.
91 ///
92 /// This is just a high-level routine to dispatch to specific code paths for
93 /// operations to legalize them.
95
96 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
97 /// FP_TO_SINT isn't legal.
98 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
99
100 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
101 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
102 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
103
104 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
105 SDValue ExpandSEXTINREG(SDNode *Node);
106
107 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
108 ///
109 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
110 /// type. The contents of the bits in the extended part of each element are
111 /// undef.
112 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
113
114 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
115 ///
116 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
117 /// type, then shifts left and arithmetic shifts right to introduce a sign
118 /// extension.
119 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
120
121 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
122 ///
123 /// Shuffles the low lanes of the operand into place and blends zeros into
124 /// the remaining lanes, finally bitcasting to the proper type.
125 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
126
127 /// Expand bswap of vectors into a shuffle if legal.
128 SDValue ExpandBSWAP(SDNode *Node);
129
130 /// Implement vselect in terms of XOR, AND, OR when blend is not
131 /// supported by the target.
132 SDValue ExpandVSELECT(SDNode *Node);
133 SDValue ExpandVP_SELECT(SDNode *Node);
134 SDValue ExpandVP_MERGE(SDNode *Node);
135 SDValue ExpandVP_REM(SDNode *Node);
136 SDValue ExpandSELECT(SDNode *Node);
137 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
138 SDValue ExpandStore(SDNode *N);
139 SDValue ExpandFNEG(SDNode *Node);
140 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149
150 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151
152 /// Implements vector promotion.
153 ///
154 /// This is essentially just bitcasting the operands to a different type and
155 /// bitcasting the result back to the original type.
157
158 /// Implements [SU]INT_TO_FP vector promotion.
159 ///
160 /// This is a [zs]ext of the input operand to a larger integer type.
161 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162
163 /// Implements FP_TO_[SU]INT vector promotion of the result type.
164 ///
165 /// It is promoted to a larger integer type. The result is then
166 /// truncated back to the original type.
167 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
168
169 /// Implements vector reduce operation promotion.
170 ///
171 /// All vector operands are promoted to a vector type with larger element
172 /// type, and the start value is promoted to a larger scalar type. Then the
173 /// result is truncated back to the original scalar type.
174 void PromoteReduction(SDNode *Node, SmallVectorImpl<SDValue> &Results);
175
176 /// Implements vector setcc operation promotion.
177 ///
178 /// All vector operands are promoted to a vector type with larger element
179 /// type.
180 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
181
182 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
183
184public:
185 VectorLegalizer(SelectionDAG& dag) :
186 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
187
188 /// Begin legalizer the vector operations in the DAG.
189 bool Run();
190};
191
192} // end anonymous namespace
193
194bool VectorLegalizer::Run() {
195 // Before we start legalizing vector nodes, check if there are any vectors.
196 bool HasVectors = false;
197 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
198 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
199 // Check if the values of the nodes contain vectors. We don't need to check
200 // the operands because we are going to check their values at some point.
201 HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
202
203 // If we found a vector node we can start the legalization.
204 if (HasVectors)
205 break;
206 }
207
208 // If this basic block has no vectors then no need to legalize vectors.
209 if (!HasVectors)
210 return false;
211
212 // The legalize process is inherently a bottom-up recursive process (users
213 // legalize their uses before themselves). Given infinite stack space, we
214 // could just start legalizing on the root and traverse the whole graph. In
215 // practice however, this causes us to run out of stack space on large basic
216 // blocks. To avoid this problem, compute an ordering of the nodes where each
217 // node is only legalized after all of its operands are legalized.
218 DAG.AssignTopologicalOrder();
219 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
220 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
221 LegalizeOp(SDValue(&*I, 0));
222
223 // Finally, it's possible the root changed. Get the new root.
224 SDValue OldRoot = DAG.getRoot();
225 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
226 DAG.setRoot(LegalizedNodes[OldRoot]);
227
228 LegalizedNodes.clear();
229
230 // Remove dead nodes now.
231 DAG.RemoveDeadNodes();
232
233 return Changed;
234}
235
236SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
237 assert(Op->getNumValues() == Result->getNumValues() &&
238 "Unexpected number of results");
239 // Generic legalization: just pass the operand through.
240 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
241 AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
242 return SDValue(Result, Op.getResNo());
243}
244
246VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
248 assert(Results.size() == Op->getNumValues() &&
249 "Unexpected number of results");
250 // Make sure that the generated code is itself legal.
251 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
252 Results[i] = LegalizeOp(Results[i]);
253 AddLegalizedOperand(Op.getValue(i), Results[i]);
254 }
255
256 return Results[Op.getResNo()];
257}
258
259SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
260 // Note that LegalizeOp may be reentered even from single-use nodes, which
261 // means that we always must cache transformed nodes.
262 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
263 if (I != LegalizedNodes.end()) return I->second;
264
265 // Legalize the operands
267 for (const SDValue &Oper : Op->op_values())
268 Ops.push_back(LegalizeOp(Oper));
269
270 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
271
272 bool HasVectorValueOrOp =
273 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
274 llvm::any_of(Node->op_values(),
275 [](SDValue O) { return O.getValueType().isVector(); });
276 if (!HasVectorValueOrOp)
277 return TranslateLegalizeResults(Op, Node);
278
279 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
280 EVT ValVT;
281 switch (Op.getOpcode()) {
282 default:
283 return TranslateLegalizeResults(Op, Node);
284 case ISD::LOAD: {
285 LoadSDNode *LD = cast<LoadSDNode>(Node);
286 ISD::LoadExtType ExtType = LD->getExtensionType();
287 EVT LoadedVT = LD->getMemoryVT();
288 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
289 Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
290 break;
291 }
292 case ISD::STORE: {
293 StoreSDNode *ST = cast<StoreSDNode>(Node);
294 EVT StVT = ST->getMemoryVT();
295 MVT ValVT = ST->getValue().getSimpleValueType();
296 if (StVT.isVector() && ST->isTruncatingStore())
297 Action = TLI.getTruncStoreAction(ValVT, StVT);
298 break;
299 }
301 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
302 // This operation lies about being legal: when it claims to be legal,
303 // it should actually be expanded.
304 if (Action == TargetLowering::Legal)
305 Action = TargetLowering::Expand;
306 break;
307#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
308 case ISD::STRICT_##DAGN:
309#include "llvm/IR/ConstrainedOps.def"
310 ValVT = Node->getValueType(0);
311 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
312 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
313 ValVT = Node->getOperand(1).getValueType();
314 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
315 Op.getOpcode() == ISD::STRICT_FSETCCS) {
316 MVT OpVT = Node->getOperand(1).getSimpleValueType();
317 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get();
318 Action = TLI.getCondCodeAction(CCCode, OpVT);
319 if (Action == TargetLowering::Legal)
320 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
321 } else {
322 Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
323 }
324 // If we're asked to expand a strict vector floating-point operation,
325 // by default we're going to simply unroll it. That is usually the
326 // best approach, except in the case where the resulting strict (scalar)
327 // operations would themselves use the fallback mutation to non-strict.
328 // In that specific case, just do the fallback on the vector op.
329 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
330 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
331 TargetLowering::Legal) {
332 EVT EltVT = ValVT.getVectorElementType();
333 if (TLI.getOperationAction(Node->getOpcode(), EltVT)
334 == TargetLowering::Expand &&
335 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
336 == TargetLowering::Legal)
337 Action = TargetLowering::Legal;
338 }
339 break;
340 case ISD::ADD:
341 case ISD::SUB:
342 case ISD::MUL:
343 case ISD::MULHS:
344 case ISD::MULHU:
345 case ISD::SDIV:
346 case ISD::UDIV:
347 case ISD::SREM:
348 case ISD::UREM:
349 case ISD::SDIVREM:
350 case ISD::UDIVREM:
351 case ISD::FADD:
352 case ISD::FSUB:
353 case ISD::FMUL:
354 case ISD::FDIV:
355 case ISD::FREM:
356 case ISD::AND:
357 case ISD::OR:
358 case ISD::XOR:
359 case ISD::SHL:
360 case ISD::SRA:
361 case ISD::SRL:
362 case ISD::FSHL:
363 case ISD::FSHR:
364 case ISD::ROTL:
365 case ISD::ROTR:
366 case ISD::ABS:
367 case ISD::BSWAP:
368 case ISD::BITREVERSE:
369 case ISD::CTLZ:
370 case ISD::CTTZ:
373 case ISD::CTPOP:
374 case ISD::SELECT:
375 case ISD::VSELECT:
376 case ISD::SELECT_CC:
377 case ISD::ZERO_EXTEND:
378 case ISD::ANY_EXTEND:
379 case ISD::TRUNCATE:
380 case ISD::SIGN_EXTEND:
381 case ISD::FP_TO_SINT:
382 case ISD::FP_TO_UINT:
383 case ISD::FNEG:
384 case ISD::FABS:
385 case ISD::FMINNUM:
386 case ISD::FMAXNUM:
389 case ISD::FMINIMUM:
390 case ISD::FMAXIMUM:
391 case ISD::FCOPYSIGN:
392 case ISD::FSQRT:
393 case ISD::FSIN:
394 case ISD::FCOS:
395 case ISD::FLDEXP:
396 case ISD::FPOWI:
397 case ISD::FPOW:
398 case ISD::FLOG:
399 case ISD::FLOG2:
400 case ISD::FLOG10:
401 case ISD::FEXP:
402 case ISD::FEXP2:
403 case ISD::FEXP10:
404 case ISD::FCEIL:
405 case ISD::FTRUNC:
406 case ISD::FRINT:
407 case ISD::FNEARBYINT:
408 case ISD::FROUND:
409 case ISD::FROUNDEVEN:
410 case ISD::FFLOOR:
411 case ISD::FP_ROUND:
412 case ISD::FP_EXTEND:
413 case ISD::FMA:
418 case ISD::SMIN:
419 case ISD::SMAX:
420 case ISD::UMIN:
421 case ISD::UMAX:
422 case ISD::SMUL_LOHI:
423 case ISD::UMUL_LOHI:
424 case ISD::SADDO:
425 case ISD::UADDO:
426 case ISD::SSUBO:
427 case ISD::USUBO:
428 case ISD::SMULO:
429 case ISD::UMULO:
431 case ISD::FFREXP:
432 case ISD::SADDSAT:
433 case ISD::UADDSAT:
434 case ISD::SSUBSAT:
435 case ISD::USUBSAT:
436 case ISD::SSHLSAT:
437 case ISD::USHLSAT:
440 case ISD::MGATHER:
441 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
442 break;
443 case ISD::SMULFIX:
444 case ISD::SMULFIXSAT:
445 case ISD::UMULFIX:
446 case ISD::UMULFIXSAT:
447 case ISD::SDIVFIX:
448 case ISD::SDIVFIXSAT:
449 case ISD::UDIVFIX:
450 case ISD::UDIVFIXSAT: {
451 unsigned Scale = Node->getConstantOperandVal(2);
452 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
453 Node->getValueType(0), Scale);
454 break;
455 }
456 case ISD::LRINT:
457 case ISD::LLRINT:
458 case ISD::SINT_TO_FP:
459 case ISD::UINT_TO_FP:
475 Action = TLI.getOperationAction(Node->getOpcode(),
476 Node->getOperand(0).getValueType());
477 break;
480 Action = TLI.getOperationAction(Node->getOpcode(),
481 Node->getOperand(1).getValueType());
482 break;
483 case ISD::SETCC: {
484 MVT OpVT = Node->getOperand(0).getSimpleValueType();
485 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
486 Action = TLI.getCondCodeAction(CCCode, OpVT);
487 if (Action == TargetLowering::Legal)
488 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
489 break;
490 }
491
492#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
493 case ISD::VPID: { \
494 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
495 : Node->getOperand(LEGALPOS).getValueType(); \
496 if (ISD::VPID == ISD::VP_SETCC) { \
497 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
498 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
499 if (Action != TargetLowering::Legal) \
500 break; \
501 } \
502 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
503 } break;
504#include "llvm/IR/VPIntrinsics.def"
505 }
506
507 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
508
509 SmallVector<SDValue, 8> ResultVals;
510 switch (Action) {
511 default: llvm_unreachable("This action is not supported yet!");
512 case TargetLowering::Promote:
513 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
514 "This action is not supported yet!");
515 LLVM_DEBUG(dbgs() << "Promoting\n");
516 Promote(Node, ResultVals);
517 assert(!ResultVals.empty() && "No results for promotion?");
518 break;
519 case TargetLowering::Legal:
520 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
521 break;
522 case TargetLowering::Custom:
523 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
524 if (LowerOperationWrapper(Node, ResultVals))
525 break;
526 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
527 [[fallthrough]];
528 case TargetLowering::Expand:
529 LLVM_DEBUG(dbgs() << "Expanding\n");
530 Expand(Node, ResultVals);
531 break;
532 }
533
534 if (ResultVals.empty())
535 return TranslateLegalizeResults(Op, Node);
536
537 Changed = true;
538 return RecursivelyLegalizeResults(Op, ResultVals);
539}
540
541// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
542// merge them somehow?
543bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
545 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
546
547 if (!Res.getNode())
548 return false;
549
550 if (Res == SDValue(Node, 0))
551 return true;
552
553 // If the original node has one result, take the return value from
554 // LowerOperation as is. It might not be result number 0.
555 if (Node->getNumValues() == 1) {
556 Results.push_back(Res);
557 return true;
558 }
559
560 // If the original node has multiple results, then the return node should
561 // have the same number of results.
562 assert((Node->getNumValues() == Res->getNumValues()) &&
563 "Lowering returned the wrong number of results!");
564
565 // Places new result values base on N result number.
566 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
567 Results.push_back(Res.getValue(I));
568
569 return true;
570}
571
572void VectorLegalizer::PromoteReduction(SDNode *Node,
574 MVT VecVT = Node->getOperand(1).getSimpleValueType();
575 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
576 MVT ScalarVT = Node->getSimpleValueType(0);
577 MVT NewScalarVT = NewVecVT.getVectorElementType();
578
579 SDLoc DL(Node);
580 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
581
582 // promote the initial value.
583 if (Node->getOperand(0).getValueType().isFloatingPoint())
584 Operands[0] =
585 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0));
586 else
587 Operands[0] =
588 DAG.getNode(ISD::ANY_EXTEND, DL, NewScalarVT, Node->getOperand(0));
589
590 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
591 if (Node->getOperand(j).getValueType().isVector() &&
592 !(ISD::isVPOpcode(Node->getOpcode()) &&
593 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand.
594 // promote the vector operand.
595 if (Node->getOperand(j).getValueType().isFloatingPoint())
596 Operands[j] =
597 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j));
598 else
599 Operands[j] =
600 DAG.getNode(ISD::ANY_EXTEND, DL, NewVecVT, Node->getOperand(j));
601 else
602 Operands[j] = Node->getOperand(j); // Skip VL operand.
603
604 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands,
605 Node->getFlags());
606
607 if (ScalarVT.isFloatingPoint())
608 Res = DAG.getNode(ISD::FP_ROUND, DL, ScalarVT, Res,
609 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
610 else
611 Res = DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, Res);
612
613 Results.push_back(Res);
614}
615
616void VectorLegalizer::PromoteSETCC(SDNode *Node,
618 MVT VecVT = Node->getOperand(0).getSimpleValueType();
619 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
620
621 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
622
623 SDLoc DL(Node);
624 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
625
626 Operands[0] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(0));
627 Operands[1] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(1));
628 Operands[2] = Node->getOperand(2);
629
630 if (Node->getOpcode() == ISD::VP_SETCC) {
631 Operands[3] = Node->getOperand(3); // mask
632 Operands[4] = Node->getOperand(4); // evl
633 }
634
635 SDValue Res = DAG.getNode(Node->getOpcode(), DL, Node->getSimpleValueType(0),
636 Operands, Node->getFlags());
637
638 Results.push_back(Res);
639}
640
641void VectorLegalizer::PromoteSTRICT(SDNode *Node,
643 MVT VecVT = Node->getOperand(1).getSimpleValueType();
644 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
645
646 assert(VecVT.isFloatingPoint());
647
648 SDLoc DL(Node);
649 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
651
652 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
653 if (Node->getOperand(j).getValueType().isVector() &&
654 !(ISD::isVPOpcode(Node->getOpcode()) &&
655 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand.
656 {
657 // promote the vector operand.
658 SDValue Ext =
659 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVecVT, MVT::Other},
660 {Node->getOperand(0), Node->getOperand(j)});
661 Operands[j] = Ext.getValue(0);
662 Chains.push_back(Ext.getValue(1));
663 } else
664 Operands[j] = Node->getOperand(j); // Skip no vector operand.
665
666 SDVTList VTs = DAG.getVTList(NewVecVT, Node->getValueType(1));
667
668 Operands[0] = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
669
670 SDValue Res =
671 DAG.getNode(Node->getOpcode(), DL, VTs, Operands, Node->getFlags());
672
673 SDValue Round =
674 DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VecVT, MVT::Other},
675 {Res.getValue(1), Res.getValue(0),
676 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
677
678 Results.push_back(Round.getValue(0));
679 Results.push_back(Round.getValue(1));
680}
681
682void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
683 // For a few operations there is a specific concept for promotion based on
684 // the operand's type.
685 switch (Node->getOpcode()) {
686 case ISD::SINT_TO_FP:
687 case ISD::UINT_TO_FP:
690 // "Promote" the operation by extending the operand.
691 PromoteINT_TO_FP(Node, Results);
692 return;
693 case ISD::FP_TO_UINT:
694 case ISD::FP_TO_SINT:
697 // Promote the operation by extending the operand.
698 PromoteFP_TO_INT(Node, Results);
699 return;
700 case ISD::VP_REDUCE_ADD:
701 case ISD::VP_REDUCE_MUL:
702 case ISD::VP_REDUCE_AND:
703 case ISD::VP_REDUCE_OR:
704 case ISD::VP_REDUCE_XOR:
705 case ISD::VP_REDUCE_SMAX:
706 case ISD::VP_REDUCE_SMIN:
707 case ISD::VP_REDUCE_UMAX:
708 case ISD::VP_REDUCE_UMIN:
709 case ISD::VP_REDUCE_FADD:
710 case ISD::VP_REDUCE_FMUL:
711 case ISD::VP_REDUCE_FMAX:
712 case ISD::VP_REDUCE_FMIN:
713 case ISD::VP_REDUCE_SEQ_FADD:
714 // Promote the operation by extending the operand.
715 PromoteReduction(Node, Results);
716 return;
717 case ISD::VP_SETCC:
718 case ISD::SETCC:
719 // Promote the operation by extending the operand.
720 PromoteSETCC(Node, Results);
721 return;
722 case ISD::STRICT_FADD:
723 case ISD::STRICT_FSUB:
724 case ISD::STRICT_FMUL:
725 case ISD::STRICT_FDIV:
727 case ISD::STRICT_FMA:
728 PromoteSTRICT(Node, Results);
729 return;
730 case ISD::FP_ROUND:
731 case ISD::FP_EXTEND:
732 // These operations are used to do promotion so they can't be promoted
733 // themselves.
734 llvm_unreachable("Don't know how to promote this operation!");
735 }
736
737 // There are currently two cases of vector promotion:
738 // 1) Bitcasting a vector of integers to a different type to a vector of the
739 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
740 // 2) Extending a vector of floats to a vector of the same number of larger
741 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
742 assert(Node->getNumValues() == 1 &&
743 "Can't promote a vector with multiple results!");
744 MVT VT = Node->getSimpleValueType(0);
745 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
746 SDLoc dl(Node);
747 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
748
749 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
750 // Do not promote the mask operand of a VP OP.
751 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) &&
752 ISD::getVPMaskIdx(Node->getOpcode()) == j;
753 if (Node->getOperand(j).getValueType().isVector() && !SkipPromote)
754 if (Node->getOperand(j)
755 .getValueType()
756 .getVectorElementType()
757 .isFloatingPoint() &&
759 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
760 else
761 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
762 else
763 Operands[j] = Node->getOperand(j);
764 }
765
766 SDValue Res =
767 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
768
769 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
772 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res,
773 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
774 else
775 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
776
777 Results.push_back(Res);
778}
779
780void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
782 // INT_TO_FP operations may require the input operand be promoted even
783 // when the type is otherwise legal.
784 bool IsStrict = Node->isStrictFPOpcode();
785 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
786 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
788 "Vectors have different number of elements!");
789
790 SDLoc dl(Node);
791 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
792
793 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
794 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
797 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
798 if (Node->getOperand(j).getValueType().isVector())
799 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
800 else
801 Operands[j] = Node->getOperand(j);
802 }
803
804 if (IsStrict) {
805 SDValue Res = DAG.getNode(Node->getOpcode(), dl,
806 {Node->getValueType(0), MVT::Other}, Operands);
807 Results.push_back(Res);
808 Results.push_back(Res.getValue(1));
809 return;
810 }
811
812 SDValue Res =
813 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
814 Results.push_back(Res);
815}
816
817// For FP_TO_INT we promote the result type to a vector type with wider
818// elements and then truncate the result. This is different from the default
819// PromoteVector which uses bitcast to promote thus assumning that the
820// promoted vector type has the same overall size.
821void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
823 MVT VT = Node->getSimpleValueType(0);
824 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
825 bool IsStrict = Node->isStrictFPOpcode();
827 "Vectors have different number of elements!");
828
829 unsigned NewOpc = Node->getOpcode();
830 // Change FP_TO_UINT to FP_TO_SINT if possible.
831 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
832 if (NewOpc == ISD::FP_TO_UINT &&
833 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
834 NewOpc = ISD::FP_TO_SINT;
835
836 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
837 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
838 NewOpc = ISD::STRICT_FP_TO_SINT;
839
840 SDLoc dl(Node);
841 SDValue Promoted, Chain;
842 if (IsStrict) {
843 Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
844 {Node->getOperand(0), Node->getOperand(1)});
845 Chain = Promoted.getValue(1);
846 } else
847 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
848
849 // Assert that the converted value fits in the original type. If it doesn't
850 // (eg: because the value being converted is too big), then the result of the
851 // original operation was undefined anyway, so the assert is still correct.
852 if (Node->getOpcode() == ISD::FP_TO_UINT ||
853 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
854 NewOpc = ISD::AssertZext;
855 else
856 NewOpc = ISD::AssertSext;
857
858 Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
859 DAG.getValueType(VT.getScalarType()));
860 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
861 Results.push_back(Promoted);
862 if (IsStrict)
863 Results.push_back(Chain);
864}
865
866std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
867 LoadSDNode *LD = cast<LoadSDNode>(N);
868 return TLI.scalarizeVectorLoad(LD, DAG);
869}
870
871SDValue VectorLegalizer::ExpandStore(SDNode *N) {
872 StoreSDNode *ST = cast<StoreSDNode>(N);
873 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
874 return TF;
875}
876
877void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
878 switch (Node->getOpcode()) {
879 case ISD::LOAD: {
880 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
881 Results.push_back(Tmp.first);
882 Results.push_back(Tmp.second);
883 return;
884 }
885 case ISD::STORE:
886 Results.push_back(ExpandStore(Node));
887 return;
889 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
890 Results.push_back(Node->getOperand(i));
891 return;
893 Results.push_back(ExpandSEXTINREG(Node));
894 return;
896 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
897 return;
899 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
900 return;
902 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
903 return;
904 case ISD::BSWAP:
905 Results.push_back(ExpandBSWAP(Node));
906 return;
907 case ISD::VP_BSWAP:
908 Results.push_back(TLI.expandVPBSWAP(Node, DAG));
909 return;
910 case ISD::VSELECT:
911 Results.push_back(ExpandVSELECT(Node));
912 return;
913 case ISD::VP_SELECT:
914 Results.push_back(ExpandVP_SELECT(Node));
915 return;
916 case ISD::VP_SREM:
917 case ISD::VP_UREM:
918 if (SDValue Expanded = ExpandVP_REM(Node)) {
919 Results.push_back(Expanded);
920 return;
921 }
922 break;
923 case ISD::SELECT:
924 Results.push_back(ExpandSELECT(Node));
925 return;
926 case ISD::SELECT_CC: {
927 if (Node->getValueType(0).isScalableVector()) {
928 EVT CondVT = TLI.getSetCCResultType(
929 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
930 SDValue SetCC =
931 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),
932 Node->getOperand(1), Node->getOperand(4));
933 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,
934 Node->getOperand(2),
935 Node->getOperand(3)));
936 return;
937 }
938 break;
939 }
940 case ISD::FP_TO_UINT:
941 ExpandFP_TO_UINT(Node, Results);
942 return;
943 case ISD::UINT_TO_FP:
944 ExpandUINT_TO_FLOAT(Node, Results);
945 return;
946 case ISD::FNEG:
947 Results.push_back(ExpandFNEG(Node));
948 return;
949 case ISD::FSUB:
950 ExpandFSUB(Node, Results);
951 return;
952 case ISD::SETCC:
953 case ISD::VP_SETCC:
954 ExpandSETCC(Node, Results);
955 return;
956 case ISD::ABS:
957 if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
958 Results.push_back(Expanded);
959 return;
960 }
961 break;
962 case ISD::ABDS:
963 case ISD::ABDU:
964 if (SDValue Expanded = TLI.expandABD(Node, DAG)) {
965 Results.push_back(Expanded);
966 return;
967 }
968 break;
969 case ISD::BITREVERSE:
970 ExpandBITREVERSE(Node, Results);
971 return;
972 case ISD::VP_BITREVERSE:
973 if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) {
974 Results.push_back(Expanded);
975 return;
976 }
977 break;
978 case ISD::CTPOP:
979 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
980 Results.push_back(Expanded);
981 return;
982 }
983 break;
984 case ISD::VP_CTPOP:
985 if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) {
986 Results.push_back(Expanded);
987 return;
988 }
989 break;
990 case ISD::CTLZ:
992 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
993 Results.push_back(Expanded);
994 return;
995 }
996 break;
997 case ISD::VP_CTLZ:
998 case ISD::VP_CTLZ_ZERO_UNDEF:
999 if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) {
1000 Results.push_back(Expanded);
1001 return;
1002 }
1003 break;
1004 case ISD::CTTZ:
1006 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
1007 Results.push_back(Expanded);
1008 return;
1009 }
1010 break;
1011 case ISD::VP_CTTZ:
1012 case ISD::VP_CTTZ_ZERO_UNDEF:
1013 if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) {
1014 Results.push_back(Expanded);
1015 return;
1016 }
1017 break;
1018 case ISD::FSHL:
1019 case ISD::VP_FSHL:
1020 case ISD::FSHR:
1021 case ISD::VP_FSHR:
1022 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
1023 Results.push_back(Expanded);
1024 return;
1025 }
1026 break;
1027 case ISD::ROTL:
1028 case ISD::ROTR:
1029 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
1030 Results.push_back(Expanded);
1031 return;
1032 }
1033 break;
1034 case ISD::FMINNUM:
1035 case ISD::FMAXNUM:
1036 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
1037 Results.push_back(Expanded);
1038 return;
1039 }
1040 break;
1041 case ISD::SMIN:
1042 case ISD::SMAX:
1043 case ISD::UMIN:
1044 case ISD::UMAX:
1045 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1046 Results.push_back(Expanded);
1047 return;
1048 }
1049 break;
1050 case ISD::UADDO:
1051 case ISD::USUBO:
1052 ExpandUADDSUBO(Node, Results);
1053 return;
1054 case ISD::SADDO:
1055 case ISD::SSUBO:
1056 ExpandSADDSUBO(Node, Results);
1057 return;
1058 case ISD::UMULO:
1059 case ISD::SMULO:
1060 ExpandMULO(Node, Results);
1061 return;
1062 case ISD::USUBSAT:
1063 case ISD::SSUBSAT:
1064 case ISD::UADDSAT:
1065 case ISD::SADDSAT:
1066 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1067 Results.push_back(Expanded);
1068 return;
1069 }
1070 break;
1071 case ISD::USHLSAT:
1072 case ISD::SSHLSAT:
1073 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1074 Results.push_back(Expanded);
1075 return;
1076 }
1077 break;
1080 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1081 if (Node->getValueType(0).isScalableVector()) {
1082 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {
1083 Results.push_back(Expanded);
1084 return;
1085 }
1086 }
1087 break;
1088 case ISD::SMULFIX:
1089 case ISD::UMULFIX:
1090 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1091 Results.push_back(Expanded);
1092 return;
1093 }
1094 break;
1095 case ISD::SMULFIXSAT:
1096 case ISD::UMULFIXSAT:
1097 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
1098 // why. Maybe it results in worse codegen compared to the unroll for some
1099 // targets? This should probably be investigated. And if we still prefer to
1100 // unroll an explanation could be helpful.
1101 break;
1102 case ISD::SDIVFIX:
1103 case ISD::UDIVFIX:
1104 ExpandFixedPointDiv(Node, Results);
1105 return;
1106 case ISD::SDIVFIXSAT:
1107 case ISD::UDIVFIXSAT:
1108 break;
1109#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1110 case ISD::STRICT_##DAGN:
1111#include "llvm/IR/ConstrainedOps.def"
1112 ExpandStrictFPOp(Node, Results);
1113 return;
1114 case ISD::VECREDUCE_ADD:
1115 case ISD::VECREDUCE_MUL:
1116 case ISD::VECREDUCE_AND:
1117 case ISD::VECREDUCE_OR:
1118 case ISD::VECREDUCE_XOR:
1129 Results.push_back(TLI.expandVecReduce(Node, DAG));
1130 return;
1133 Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
1134 return;
1135 case ISD::SREM:
1136 case ISD::UREM:
1137 ExpandREM(Node, Results);
1138 return;
1139 case ISD::VP_MERGE:
1140 Results.push_back(ExpandVP_MERGE(Node));
1141 return;
1142 }
1143
1144 SDValue Unrolled = DAG.UnrollVectorOp(Node);
1145 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1146 Results.push_back(Unrolled.getValue(I));
1147}
1148
1149SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1150 // Lower a select instruction where the condition is a scalar and the
1151 // operands are vectors. Lower this select to VSELECT and implement it
1152 // using XOR AND OR. The selector bit is broadcasted.
1153 EVT VT = Node->getValueType(0);
1154 SDLoc DL(Node);
1155
1156 SDValue Mask = Node->getOperand(0);
1157 SDValue Op1 = Node->getOperand(1);
1158 SDValue Op2 = Node->getOperand(2);
1159
1160 assert(VT.isVector() && !Mask.getValueType().isVector()
1161 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1162
1163 // If we can't even use the basic vector operations of
1164 // AND,OR,XOR, we will have to scalarize the op.
1165 // Notice that the operation may be 'promoted' which means that it is
1166 // 'bitcasted' to another type which is handled.
1167 // Also, we need to be able to construct a splat vector using either
1168 // BUILD_VECTOR or SPLAT_VECTOR.
1169 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1170 // BUILD_VECTOR?
1171 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1172 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1173 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
1174 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1176 VT) == TargetLowering::Expand)
1177 return DAG.UnrollVectorOp(Node);
1178
1179 // Generate a mask operand.
1181
1182 // What is the size of each element in the vector mask.
1183 EVT BitTy = MaskTy.getScalarType();
1184
1185 Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
1186 DAG.getConstant(0, DL, BitTy));
1187
1188 // Broadcast the mask so that the entire vector is all one or all zero.
1189 Mask = DAG.getSplat(MaskTy, DL, Mask);
1190
1191 // Bitcast the operands to be the same type as the mask.
1192 // This is needed when we select between FP types because
1193 // the mask is a vector of integers.
1194 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
1195 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
1196
1197 SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
1198
1199 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
1200 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
1201 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
1202 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1203}
1204
1205SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1206 EVT VT = Node->getValueType(0);
1207
1208 // Make sure that the SRA and SHL instructions are available.
1209 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
1210 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
1211 return DAG.UnrollVectorOp(Node);
1212
1213 SDLoc DL(Node);
1214 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
1215
1216 unsigned BW = VT.getScalarSizeInBits();
1217 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1218 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
1219
1220 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
1221 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
1222}
1223
1224// Generically expand a vector anyext in register to a shuffle of the relevant
1225// lanes into the appropriate locations, with other lanes left undef.
1226SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1227 SDLoc DL(Node);
1228 EVT VT = Node->getValueType(0);
1229 int NumElements = VT.getVectorNumElements();
1230 SDValue Src = Node->getOperand(0);
1231 EVT SrcVT = Src.getValueType();
1232 int NumSrcElements = SrcVT.getVectorNumElements();
1233
1234 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1235 // into a larger vector type.
1236 if (SrcVT.bitsLE(VT)) {
1237 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1238 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1239 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1240 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1241 NumSrcElements);
1242 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1243 Src, DAG.getVectorIdxConstant(0, DL));
1244 }
1245
1246 // Build a base mask of undef shuffles.
1247 SmallVector<int, 16> ShuffleMask;
1248 ShuffleMask.resize(NumSrcElements, -1);
1249
1250 // Place the extended lanes into the correct locations.
1251 int ExtLaneScale = NumSrcElements / NumElements;
1252 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1253 for (int i = 0; i < NumElements; ++i)
1254 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1255
1256 return DAG.getNode(
1257 ISD::BITCAST, DL, VT,
1258 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1259}
1260
1261SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1262 SDLoc DL(Node);
1263 EVT VT = Node->getValueType(0);
1264 SDValue Src = Node->getOperand(0);
1265 EVT SrcVT = Src.getValueType();
1266
1267 // First build an any-extend node which can be legalized above when we
1268 // recurse through it.
1269 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1270
1271 // Now we need sign extend. Do this by shifting the elements. Even if these
1272 // aren't legal operations, they have a better chance of being legalized
1273 // without full scalarization than the sign extension does.
1274 unsigned EltWidth = VT.getScalarSizeInBits();
1275 unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1276 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1277 return DAG.getNode(ISD::SRA, DL, VT,
1278 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1279 ShiftAmount);
1280}
1281
1282// Generically expand a vector zext in register to a shuffle of the relevant
1283// lanes into the appropriate locations, a blend of zero into the high bits,
1284// and a bitcast to the wider element type.
1285SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1286 SDLoc DL(Node);
1287 EVT VT = Node->getValueType(0);
1288 int NumElements = VT.getVectorNumElements();
1289 SDValue Src = Node->getOperand(0);
1290 EVT SrcVT = Src.getValueType();
1291 int NumSrcElements = SrcVT.getVectorNumElements();
1292
1293 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1294 // into a larger vector type.
1295 if (SrcVT.bitsLE(VT)) {
1296 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1297 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1298 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1299 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1300 NumSrcElements);
1301 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1302 Src, DAG.getVectorIdxConstant(0, DL));
1303 }
1304
1305 // Build up a zero vector to blend into this one.
1306 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1307
1308 // Shuffle the incoming lanes into the correct position, and pull all other
1309 // lanes from the zero vector.
1310 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1311
1312 int ExtLaneScale = NumSrcElements / NumElements;
1313 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1314 for (int i = 0; i < NumElements; ++i)
1315 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1316
1317 return DAG.getNode(ISD::BITCAST, DL, VT,
1318 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1319}
1320
1321static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1322 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1323 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1324 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1325 ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1326}
1327
1328SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1329 EVT VT = Node->getValueType(0);
1330
1331 // Scalable vectors can't use shuffle expansion.
1332 if (VT.isScalableVector())
1333 return TLI.expandBSWAP(Node, DAG);
1334
1335 // Generate a byte wise shuffle mask for the BSWAP.
1336 SmallVector<int, 16> ShuffleMask;
1337 createBSWAPShuffleMask(VT, ShuffleMask);
1338 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1339
1340 // Only emit a shuffle if the mask is legal.
1341 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1342 SDLoc DL(Node);
1343 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1344 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1345 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1346 }
1347
1348 // If we have the appropriate vector bit operations, it is better to use them
1349 // than unrolling and expanding each component.
1350 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1351 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1352 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1353 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1354 return TLI.expandBSWAP(Node, DAG);
1355
1356 // Otherwise unroll.
1357 return DAG.UnrollVectorOp(Node);
1358}
1359
1360void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1362 EVT VT = Node->getValueType(0);
1363
1364 // We can't unroll or use shuffles for scalable vectors.
1365 if (VT.isScalableVector()) {
1366 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1367 return;
1368 }
1369
1370 // If we have the scalar operation, it's probably cheaper to unroll it.
1371 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1372 SDValue Tmp = DAG.UnrollVectorOp(Node);
1373 Results.push_back(Tmp);
1374 return;
1375 }
1376
1377 // If the vector element width is a whole number of bytes, test if its legal
1378 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1379 // vector. This greatly reduces the number of bit shifts necessary.
1380 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1381 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1382 SmallVector<int, 16> BSWAPMask;
1383 createBSWAPShuffleMask(VT, BSWAPMask);
1384
1385 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1386 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1387 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1388 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1389 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1390 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1391 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1392 SDLoc DL(Node);
1393 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1394 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1395 BSWAPMask);
1396 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1397 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1398 Results.push_back(Op);
1399 return;
1400 }
1401 }
1402
1403 // If we have the appropriate vector bit operations, it is better to use them
1404 // than unrolling and expanding each component.
1405 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1406 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1407 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1408 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1409 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1410 return;
1411 }
1412
1413 // Otherwise unroll.
1414 SDValue Tmp = DAG.UnrollVectorOp(Node);
1415 Results.push_back(Tmp);
1416}
1417
1418SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1419 // Implement VSELECT in terms of XOR, AND, OR
1420 // on platforms which do not support blend natively.
1421 SDLoc DL(Node);
1422
1423 SDValue Mask = Node->getOperand(0);
1424 SDValue Op1 = Node->getOperand(1);
1425 SDValue Op2 = Node->getOperand(2);
1426
1427 EVT VT = Mask.getValueType();
1428
1429 // If we can't even use the basic vector operations of
1430 // AND,OR,XOR, we will have to scalarize the op.
1431 // Notice that the operation may be 'promoted' which means that it is
1432 // 'bitcasted' to another type which is handled.
1433 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1434 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1435 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1436 return DAG.UnrollVectorOp(Node);
1437
1438 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1439 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1440 // vector constant to mask with.
1441 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1442 auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1443 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1444 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1445 Op1.getValueType().getVectorElementType() == MVT::i1))
1446 return DAG.UnrollVectorOp(Node);
1447
1448 // If the mask and the type are different sizes, unroll the vector op. This
1449 // can occur when getSetCCResultType returns something that is different in
1450 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1451 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1452 return DAG.UnrollVectorOp(Node);
1453
1454 // Bitcast the operands to be the same type as the mask.
1455 // This is needed when we select between FP types because
1456 // the mask is a vector of integers.
1457 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1458 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1459
1460 SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1461
1462 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1463 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1464 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1465 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1466}
1467
1468SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1469 // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1470 // do not support it natively.
1471 SDLoc DL(Node);
1472
1473 SDValue Mask = Node->getOperand(0);
1474 SDValue Op1 = Node->getOperand(1);
1475 SDValue Op2 = Node->getOperand(2);
1476 SDValue EVL = Node->getOperand(3);
1477
1478 EVT VT = Mask.getValueType();
1479
1480 // If we can't even use the basic vector operations of
1481 // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1482 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1483 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1484 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1485 return DAG.UnrollVectorOp(Node);
1486
1487 // This operation also isn't safe when the operands aren't also booleans.
1488 if (Op1.getValueType().getVectorElementType() != MVT::i1)
1489 return DAG.UnrollVectorOp(Node);
1490
1491 SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1492 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);
1493
1494 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL);
1495 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL);
1496 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL);
1497}
1498
1499SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1500 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1501 // indices less than the EVL/pivot are true. Combine that with the original
1502 // mask for a full-length mask. Use a full-length VSELECT to select between
1503 // the true and false values.
1504 SDLoc DL(Node);
1505
1506 SDValue Mask = Node->getOperand(0);
1507 SDValue Op1 = Node->getOperand(1);
1508 SDValue Op2 = Node->getOperand(2);
1509 SDValue EVL = Node->getOperand(3);
1510
1511 EVT MaskVT = Mask.getValueType();
1512 bool IsFixedLen = MaskVT.isFixedLengthVector();
1513
1514 EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1515 MaskVT.getVectorElementCount());
1516
1517 // If we can't construct the EVL mask efficiently, it's better to unroll.
1518 if ((IsFixedLen &&
1519 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1520 (!IsFixedLen &&
1521 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1522 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1523 return DAG.UnrollVectorOp(Node);
1524
1525 // If using a SETCC would result in a different type than the mask type,
1526 // unroll.
1527 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1528 EVLVecVT) != MaskVT)
1529 return DAG.UnrollVectorOp(Node);
1530
1531 SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1532 SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);
1533 SDValue EVLMask =
1534 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1535
1536 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1537 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1538}
1539
1540SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1541 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.
1542 EVT VT = Node->getValueType(0);
1543
1544 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1545
1546 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1547 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1548 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1549 return SDValue();
1550
1551 SDLoc DL(Node);
1552
1553 SDValue Dividend = Node->getOperand(0);
1554 SDValue Divisor = Node->getOperand(1);
1555 SDValue Mask = Node->getOperand(2);
1556 SDValue EVL = Node->getOperand(3);
1557
1558 // X % Y -> X-X/Y*Y
1559 SDValue Div = DAG.getNode(DivOpc, DL, VT, Dividend, Divisor, Mask, EVL);
1560 SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL);
1561 return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL);
1562}
1563
1564void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1566 // Attempt to expand using TargetLowering.
1567 SDValue Result, Chain;
1568 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1569 Results.push_back(Result);
1570 if (Node->isStrictFPOpcode())
1571 Results.push_back(Chain);
1572 return;
1573 }
1574
1575 // Otherwise go ahead and unroll.
1576 if (Node->isStrictFPOpcode()) {
1577 UnrollStrictFPOp(Node, Results);
1578 return;
1579 }
1580
1581 Results.push_back(DAG.UnrollVectorOp(Node));
1582}
1583
1584void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1586 bool IsStrict = Node->isStrictFPOpcode();
1587 unsigned OpNo = IsStrict ? 1 : 0;
1588 SDValue Src = Node->getOperand(OpNo);
1589 EVT VT = Src.getValueType();
1590 SDLoc DL(Node);
1591
1592 // Attempt to expand using TargetLowering.
1594 SDValue Chain;
1595 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1596 Results.push_back(Result);
1597 if (IsStrict)
1598 Results.push_back(Chain);
1599 return;
1600 }
1601
1602 // Make sure that the SINT_TO_FP and SRL instructions are available.
1603 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1604 TargetLowering::Expand) ||
1605 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1606 TargetLowering::Expand)) ||
1607 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1608 if (IsStrict) {
1609 UnrollStrictFPOp(Node, Results);
1610 return;
1611 }
1612
1613 Results.push_back(DAG.UnrollVectorOp(Node));
1614 return;
1615 }
1616
1617 unsigned BW = VT.getScalarSizeInBits();
1618 assert((BW == 64 || BW == 32) &&
1619 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1620
1621 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1622
1623 // Constants to clear the upper part of the word.
1624 // Notice that we can also use SHL+SHR, but using a constant is slightly
1625 // faster on x86.
1626 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1627 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1628
1629 // Two to the power of half-word-size.
1630 SDValue TWOHW =
1631 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1632
1633 // Clear upper part of LO, lower HI
1634 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1635 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1636
1637 if (IsStrict) {
1638 // Convert hi and lo to floats
1639 // Convert the hi part back to the upper values
1640 // TODO: Can any fast-math-flags be set on these nodes?
1642 {Node->getValueType(0), MVT::Other},
1643 {Node->getOperand(0), HI});
1644 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1645 {fHI.getValue(1), fHI, TWOHW});
1647 {Node->getValueType(0), MVT::Other},
1648 {Node->getOperand(0), LO});
1649
1650 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1651 fLO.getValue(1));
1652
1653 // Add the two halves
1654 SDValue Result =
1655 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1656 {TF, fHI, fLO});
1657
1658 Results.push_back(Result);
1659 Results.push_back(Result.getValue(1));
1660 return;
1661 }
1662
1663 // Convert hi and lo to floats
1664 // Convert the hi part back to the upper values
1665 // TODO: Can any fast-math-flags be set on these nodes?
1666 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1667 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1668 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1669
1670 // Add the two halves
1671 Results.push_back(
1672 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1673}
1674
1675SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1676 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1677 SDLoc DL(Node);
1678 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1679 // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1680 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1681 Node->getOperand(0));
1682 }
1683 return DAG.UnrollVectorOp(Node);
1684}
1685
1686void VectorLegalizer::ExpandFSUB(SDNode *Node,
1688 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1689 // we can defer this to operation legalization where it will be lowered as
1690 // a+(-b).
1691 EVT VT = Node->getValueType(0);
1692 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1693 TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1694 return; // Defer to LegalizeDAG
1695
1696 SDValue Tmp = DAG.UnrollVectorOp(Node);
1697 Results.push_back(Tmp);
1698}
1699
1700void VectorLegalizer::ExpandSETCC(SDNode *Node,
1702 bool NeedInvert = false;
1703 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
1704 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
1705 Node->getOpcode() == ISD::STRICT_FSETCCS;
1706 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
1707 unsigned Offset = IsStrict ? 1 : 0;
1708
1709 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
1710 SDValue LHS = Node->getOperand(0 + Offset);
1711 SDValue RHS = Node->getOperand(1 + Offset);
1712 SDValue CC = Node->getOperand(2 + Offset);
1713
1714 MVT OpVT = LHS.getSimpleValueType();
1715 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1716
1717 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1718 if (IsStrict) {
1719 UnrollStrictFPOp(Node, Results);
1720 return;
1721 }
1722 Results.push_back(UnrollVSETCC(Node));
1723 return;
1724 }
1725
1726 SDValue Mask, EVL;
1727 if (IsVP) {
1728 Mask = Node->getOperand(3 + Offset);
1729 EVL = Node->getOperand(4 + Offset);
1730 }
1731
1732 SDLoc dl(Node);
1733 bool Legalized =
1734 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
1735 EVL, NeedInvert, dl, Chain, IsSignaling);
1736
1737 if (Legalized) {
1738 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1739 // condition code, create a new SETCC node.
1740 if (CC.getNode()) {
1741 if (IsStrict) {
1742 LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
1743 {Chain, LHS, RHS, CC}, Node->getFlags());
1744 Chain = LHS.getValue(1);
1745 } else if (IsVP) {
1746 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
1747 {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
1748 } else {
1749 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1750 Node->getFlags());
1751 }
1752 }
1753
1754 // If we expanded the SETCC by inverting the condition code, then wrap
1755 // the existing SETCC in a NOT to restore the intended condition.
1756 if (NeedInvert) {
1757 if (!IsVP)
1758 LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1759 else
1760 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
1761 }
1762 } else {
1763 assert(!IsStrict && "Don't know how to expand for strict nodes.");
1764
1765 // Otherwise, SETCC for the given comparison type must be completely
1766 // illegal; expand it into a SELECT_CC.
1767 EVT VT = Node->getValueType(0);
1768 LHS =
1769 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1770 DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1771 DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1772 LHS->setFlags(Node->getFlags());
1773 }
1774
1775 Results.push_back(LHS);
1776 if (IsStrict)
1777 Results.push_back(Chain);
1778}
1779
1780void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1782 SDValue Result, Overflow;
1783 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1784 Results.push_back(Result);
1785 Results.push_back(Overflow);
1786}
1787
1788void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1790 SDValue Result, Overflow;
1791 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1792 Results.push_back(Result);
1793 Results.push_back(Overflow);
1794}
1795
1796void VectorLegalizer::ExpandMULO(SDNode *Node,
1798 SDValue Result, Overflow;
1799 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1800 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1801
1802 Results.push_back(Result);
1803 Results.push_back(Overflow);
1804}
1805
1806void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1808 SDNode *N = Node;
1809 if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1810 N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1811 Results.push_back(Expanded);
1812}
1813
1814void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1816 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1817 ExpandUINT_TO_FLOAT(Node, Results);
1818 return;
1819 }
1820 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1821 ExpandFP_TO_UINT(Node, Results);
1822 return;
1823 }
1824
1825 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1826 Node->getOpcode() == ISD::STRICT_FSETCCS) {
1827 ExpandSETCC(Node, Results);
1828 return;
1829 }
1830
1831 UnrollStrictFPOp(Node, Results);
1832}
1833
1834void VectorLegalizer::ExpandREM(SDNode *Node,
1836 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1837 "Expected REM node");
1838
1840 if (!TLI.expandREM(Node, Result, DAG))
1841 Result = DAG.UnrollVectorOp(Node);
1842 Results.push_back(Result);
1843}
1844
1845void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1847 EVT VT = Node->getValueType(0);
1848 EVT EltVT = VT.getVectorElementType();
1849 unsigned NumElems = VT.getVectorNumElements();
1850 unsigned NumOpers = Node->getNumOperands();
1851 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1852
1853 EVT TmpEltVT = EltVT;
1854 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1855 Node->getOpcode() == ISD::STRICT_FSETCCS)
1856 TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1857 *DAG.getContext(), TmpEltVT);
1858
1859 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1860 SDValue Chain = Node->getOperand(0);
1861 SDLoc dl(Node);
1862
1863 SmallVector<SDValue, 32> OpValues;
1864 SmallVector<SDValue, 32> OpChains;
1865 for (unsigned i = 0; i < NumElems; ++i) {
1867 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1868
1869 // The Chain is the first operand.
1870 Opers.push_back(Chain);
1871
1872 // Now process the remaining operands.
1873 for (unsigned j = 1; j < NumOpers; ++j) {
1874 SDValue Oper = Node->getOperand(j);
1875 EVT OperVT = Oper.getValueType();
1876
1877 if (OperVT.isVector())
1878 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1879 OperVT.getVectorElementType(), Oper, Idx);
1880
1881 Opers.push_back(Oper);
1882 }
1883
1884 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1885 SDValue ScalarResult = ScalarOp.getValue(0);
1886 SDValue ScalarChain = ScalarOp.getValue(1);
1887
1888 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1889 Node->getOpcode() == ISD::STRICT_FSETCCS)
1890 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1891 DAG.getAllOnesConstant(dl, EltVT),
1892 DAG.getConstant(0, dl, EltVT));
1893
1894 OpValues.push_back(ScalarResult);
1895 OpChains.push_back(ScalarChain);
1896 }
1897
1898 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1899 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1900
1901 Results.push_back(Result);
1902 Results.push_back(NewChain);
1903}
1904
1905SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1906 EVT VT = Node->getValueType(0);
1907 unsigned NumElems = VT.getVectorNumElements();
1908 EVT EltVT = VT.getVectorElementType();
1909 SDValue LHS = Node->getOperand(0);
1910 SDValue RHS = Node->getOperand(1);
1911 SDValue CC = Node->getOperand(2);
1912 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1913 SDLoc dl(Node);
1914 SmallVector<SDValue, 8> Ops(NumElems);
1915 for (unsigned i = 0; i < NumElems; ++i) {
1916 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1917 DAG.getVectorIdxConstant(i, dl));
1918 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1919 DAG.getVectorIdxConstant(i, dl));
1920 Ops[i] = DAG.getNode(ISD::SETCC, dl,
1921 TLI.getSetCCResultType(DAG.getDataLayout(),
1922 *DAG.getContext(), TmpEltVT),
1923 LHSElem, RHSElem, CC);
1924 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1925 DAG.getConstant(0, dl, EltVT));
1926 }
1927 return DAG.getBuildVector(VT, dl, Ops);
1928}
1929
1931 return VectorLegalizer(*this).Run();
1932}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
BinaryOperator * Mul
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
size_t size() const
Definition: Function.h:800
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:534
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1337
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1368
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:977
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1029
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1353
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1357
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:820
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1367
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:937
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1350
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1354
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:928
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:646
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1369
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1362
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:976
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:969
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:809
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1370
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:982
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1351
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:942
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:831
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1358
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1338
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1512
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1492
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
#define N
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:333
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:363
bool isFixedLengthVector() const
Definition: ValueTypes.h:170
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:166
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291
This represents a list of ValueType's that has been intern'd by a SelectionDAG.