LLVM 19.0.0git
LegalizeVectorOps.cpp
Go to the documentation of this file.
1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
39#include "llvm/IR/DataLayout.h"
42#include "llvm/Support/Debug.h"
44#include <cassert>
45#include <cstdint>
46#include <iterator>
47#include <utility>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "legalizevectorops"
52
53namespace {
54
55class VectorLegalizer {
56 SelectionDAG& DAG;
57 const TargetLowering &TLI;
58 bool Changed = false; // Keep track of whether anything changed
59
60 /// For nodes that are of legal width, and that have more than one use, this
61 /// map indicates what regularized operand to use. This allows us to avoid
62 /// legalizing the same thing more than once.
64
65 /// Adds a node to the translation cache.
66 void AddLegalizedOperand(SDValue From, SDValue To) {
67 LegalizedNodes.insert(std::make_pair(From, To));
68 // If someone requests legalization of the new node, return itself.
69 if (From != To)
70 LegalizedNodes.insert(std::make_pair(To, To));
71 }
72
73 /// Legalizes the given node.
74 SDValue LegalizeOp(SDValue Op);
75
76 /// Assuming the node is legal, "legalize" the results.
77 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
78
79 /// Make sure Results are legal and update the translation cache.
80 SDValue RecursivelyLegalizeResults(SDValue Op,
82
83 /// Wrapper to interface LowerOperation with a vector of Results.
84 /// Returns false if the target wants to use default expansion. Otherwise
85 /// returns true. If return is true and the Results are empty, then the
86 /// target wants to keep the input node as is.
87 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
88
89 /// Implements unrolling a VSETCC.
90 SDValue UnrollVSETCC(SDNode *Node);
91
92 /// Implement expand-based legalization of vector operations.
93 ///
94 /// This is just a high-level routine to dispatch to specific code paths for
95 /// operations to legalize them.
97
98 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
99 /// FP_TO_SINT isn't legal.
100 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
101
102 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
103 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
104 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
105
106 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
107 SDValue ExpandSEXTINREG(SDNode *Node);
108
109 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
110 ///
111 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
112 /// type. The contents of the bits in the extended part of each element are
113 /// undef.
114 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
115
116 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
117 ///
118 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
119 /// type, then shifts left and arithmetic shifts right to introduce a sign
120 /// extension.
121 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
122
123 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
124 ///
125 /// Shuffles the low lanes of the operand into place and blends zeros into
126 /// the remaining lanes, finally bitcasting to the proper type.
127 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
128
129 /// Expand bswap of vectors into a shuffle if legal.
130 SDValue ExpandBSWAP(SDNode *Node);
131
132 /// Implement vselect in terms of XOR, AND, OR when blend is not
133 /// supported by the target.
134 SDValue ExpandVSELECT(SDNode *Node);
135 SDValue ExpandVP_SELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandSELECT(SDNode *Node);
139 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
140 SDValue ExpandStore(SDNode *N);
141 SDValue ExpandFNEG(SDNode *Node);
142 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151
152 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
154 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall Call_F32,
155 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
156 RTLIB::Libcall Call_F128,
157 RTLIB::Libcall Call_PPCF128,
159
160 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 /// Implements vector promotion.
163 ///
164 /// This is essentially just bitcasting the operands to a different type and
165 /// bitcasting the result back to the original type.
167
168 /// Implements [SU]INT_TO_FP vector promotion.
169 ///
170 /// This is a [zs]ext of the input operand to a larger integer type.
171 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
172
173 /// Implements FP_TO_[SU]INT vector promotion of the result type.
174 ///
175 /// It is promoted to a larger integer type. The result is then
176 /// truncated back to the original type.
177 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
178
179 /// Implements vector setcc operation promotion.
180 ///
181 /// All vector operands are promoted to a vector type with larger element
182 /// type.
183 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
184
185 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
186
187public:
188 VectorLegalizer(SelectionDAG& dag) :
189 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
190
191 /// Begin legalizer the vector operations in the DAG.
192 bool Run();
193};
194
195} // end anonymous namespace
196
197bool VectorLegalizer::Run() {
198 // Before we start legalizing vector nodes, check if there are any vectors.
199 bool HasVectors = false;
200 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
201 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
202 // Check if the values of the nodes contain vectors. We don't need to check
203 // the operands because we are going to check their values at some point.
204 HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
205
206 // If we found a vector node we can start the legalization.
207 if (HasVectors)
208 break;
209 }
210
211 // If this basic block has no vectors then no need to legalize vectors.
212 if (!HasVectors)
213 return false;
214
215 // The legalize process is inherently a bottom-up recursive process (users
216 // legalize their uses before themselves). Given infinite stack space, we
217 // could just start legalizing on the root and traverse the whole graph. In
218 // practice however, this causes us to run out of stack space on large basic
219 // blocks. To avoid this problem, compute an ordering of the nodes where each
220 // node is only legalized after all of its operands are legalized.
221 DAG.AssignTopologicalOrder();
222 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
223 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
224 LegalizeOp(SDValue(&*I, 0));
225
226 // Finally, it's possible the root changed. Get the new root.
227 SDValue OldRoot = DAG.getRoot();
228 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
229 DAG.setRoot(LegalizedNodes[OldRoot]);
230
231 LegalizedNodes.clear();
232
233 // Remove dead nodes now.
234 DAG.RemoveDeadNodes();
235
236 return Changed;
237}
238
239SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
240 assert(Op->getNumValues() == Result->getNumValues() &&
241 "Unexpected number of results");
242 // Generic legalization: just pass the operand through.
243 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
244 AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
245 return SDValue(Result, Op.getResNo());
246}
247
249VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
251 assert(Results.size() == Op->getNumValues() &&
252 "Unexpected number of results");
253 // Make sure that the generated code is itself legal.
254 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
255 Results[i] = LegalizeOp(Results[i]);
256 AddLegalizedOperand(Op.getValue(i), Results[i]);
257 }
258
259 return Results[Op.getResNo()];
260}
261
262SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
263 // Note that LegalizeOp may be reentered even from single-use nodes, which
264 // means that we always must cache transformed nodes.
265 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
266 if (I != LegalizedNodes.end()) return I->second;
267
268 // Legalize the operands
270 for (const SDValue &Oper : Op->op_values())
271 Ops.push_back(LegalizeOp(Oper));
272
273 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
274
275 bool HasVectorValueOrOp =
276 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
277 llvm::any_of(Node->op_values(),
278 [](SDValue O) { return O.getValueType().isVector(); });
279 if (!HasVectorValueOrOp)
280 return TranslateLegalizeResults(Op, Node);
281
282 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
283 EVT ValVT;
284 switch (Op.getOpcode()) {
285 default:
286 return TranslateLegalizeResults(Op, Node);
287 case ISD::LOAD: {
288 LoadSDNode *LD = cast<LoadSDNode>(Node);
289 ISD::LoadExtType ExtType = LD->getExtensionType();
290 EVT LoadedVT = LD->getMemoryVT();
291 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
292 Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
293 break;
294 }
295 case ISD::STORE: {
296 StoreSDNode *ST = cast<StoreSDNode>(Node);
297 EVT StVT = ST->getMemoryVT();
298 MVT ValVT = ST->getValue().getSimpleValueType();
299 if (StVT.isVector() && ST->isTruncatingStore())
300 Action = TLI.getTruncStoreAction(ValVT, StVT);
301 break;
302 }
304 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
305 // This operation lies about being legal: when it claims to be legal,
306 // it should actually be expanded.
307 if (Action == TargetLowering::Legal)
308 Action = TargetLowering::Expand;
309 break;
310#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
311 case ISD::STRICT_##DAGN:
312#include "llvm/IR/ConstrainedOps.def"
313 ValVT = Node->getValueType(0);
314 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
315 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
316 ValVT = Node->getOperand(1).getValueType();
317 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
318 Op.getOpcode() == ISD::STRICT_FSETCCS) {
319 MVT OpVT = Node->getOperand(1).getSimpleValueType();
320 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get();
321 Action = TLI.getCondCodeAction(CCCode, OpVT);
322 if (Action == TargetLowering::Legal)
323 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
324 } else {
325 Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
326 }
327 // If we're asked to expand a strict vector floating-point operation,
328 // by default we're going to simply unroll it. That is usually the
329 // best approach, except in the case where the resulting strict (scalar)
330 // operations would themselves use the fallback mutation to non-strict.
331 // In that specific case, just do the fallback on the vector op.
332 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
333 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
334 TargetLowering::Legal) {
335 EVT EltVT = ValVT.getVectorElementType();
336 if (TLI.getOperationAction(Node->getOpcode(), EltVT)
337 == TargetLowering::Expand &&
338 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
339 == TargetLowering::Legal)
340 Action = TargetLowering::Legal;
341 }
342 break;
343 case ISD::ADD:
344 case ISD::SUB:
345 case ISD::MUL:
346 case ISD::MULHS:
347 case ISD::MULHU:
348 case ISD::SDIV:
349 case ISD::UDIV:
350 case ISD::SREM:
351 case ISD::UREM:
352 case ISD::SDIVREM:
353 case ISD::UDIVREM:
354 case ISD::FADD:
355 case ISD::FSUB:
356 case ISD::FMUL:
357 case ISD::FDIV:
358 case ISD::FREM:
359 case ISD::AND:
360 case ISD::OR:
361 case ISD::XOR:
362 case ISD::SHL:
363 case ISD::SRA:
364 case ISD::SRL:
365 case ISD::FSHL:
366 case ISD::FSHR:
367 case ISD::ROTL:
368 case ISD::ROTR:
369 case ISD::ABS:
370 case ISD::ABDS:
371 case ISD::ABDU:
372 case ISD::AVGCEILS:
373 case ISD::AVGCEILU:
374 case ISD::AVGFLOORS:
375 case ISD::AVGFLOORU:
376 case ISD::BSWAP:
377 case ISD::BITREVERSE:
378 case ISD::CTLZ:
379 case ISD::CTTZ:
382 case ISD::CTPOP:
383 case ISD::SELECT:
384 case ISD::VSELECT:
385 case ISD::SELECT_CC:
386 case ISD::ZERO_EXTEND:
387 case ISD::ANY_EXTEND:
388 case ISD::TRUNCATE:
389 case ISD::SIGN_EXTEND:
390 case ISD::FP_TO_SINT:
391 case ISD::FP_TO_UINT:
392 case ISD::FNEG:
393 case ISD::FABS:
394 case ISD::FMINNUM:
395 case ISD::FMAXNUM:
398 case ISD::FMINIMUM:
399 case ISD::FMAXIMUM:
400 case ISD::FCOPYSIGN:
401 case ISD::FSQRT:
402 case ISD::FSIN:
403 case ISD::FCOS:
404 case ISD::FTAN:
405 case ISD::FLDEXP:
406 case ISD::FPOWI:
407 case ISD::FPOW:
408 case ISD::FLOG:
409 case ISD::FLOG2:
410 case ISD::FLOG10:
411 case ISD::FEXP:
412 case ISD::FEXP2:
413 case ISD::FEXP10:
414 case ISD::FCEIL:
415 case ISD::FTRUNC:
416 case ISD::FRINT:
417 case ISD::FNEARBYINT:
418 case ISD::FROUND:
419 case ISD::FROUNDEVEN:
420 case ISD::FFLOOR:
421 case ISD::FP_ROUND:
422 case ISD::FP_EXTEND:
424 case ISD::FMA:
429 case ISD::SMIN:
430 case ISD::SMAX:
431 case ISD::UMIN:
432 case ISD::UMAX:
433 case ISD::SMUL_LOHI:
434 case ISD::UMUL_LOHI:
435 case ISD::SADDO:
436 case ISD::UADDO:
437 case ISD::SSUBO:
438 case ISD::USUBO:
439 case ISD::SMULO:
440 case ISD::UMULO:
442 case ISD::FFREXP:
443 case ISD::SADDSAT:
444 case ISD::UADDSAT:
445 case ISD::SSUBSAT:
446 case ISD::USUBSAT:
447 case ISD::SSHLSAT:
448 case ISD::USHLSAT:
451 case ISD::MGATHER:
452 case ISD::SCMP:
453 case ISD::UCMP:
454 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
455 break;
456 case ISD::SMULFIX:
457 case ISD::SMULFIXSAT:
458 case ISD::UMULFIX:
459 case ISD::UMULFIXSAT:
460 case ISD::SDIVFIX:
461 case ISD::SDIVFIXSAT:
462 case ISD::UDIVFIX:
463 case ISD::UDIVFIXSAT: {
464 unsigned Scale = Node->getConstantOperandVal(2);
465 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
466 Node->getValueType(0), Scale);
467 break;
468 }
469 case ISD::LRINT:
470 case ISD::LLRINT:
471 case ISD::SINT_TO_FP:
472 case ISD::UINT_TO_FP:
488 Action = TLI.getOperationAction(Node->getOpcode(),
489 Node->getOperand(0).getValueType());
490 break;
493 Action = TLI.getOperationAction(Node->getOpcode(),
494 Node->getOperand(1).getValueType());
495 break;
496 case ISD::SETCC: {
497 MVT OpVT = Node->getOperand(0).getSimpleValueType();
498 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
499 Action = TLI.getCondCodeAction(CCCode, OpVT);
500 if (Action == TargetLowering::Legal)
501 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
502 break;
503 }
504
505#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
506 case ISD::VPID: { \
507 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
508 : Node->getOperand(LEGALPOS).getValueType(); \
509 if (ISD::VPID == ISD::VP_SETCC) { \
510 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
511 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
512 if (Action != TargetLowering::Legal) \
513 break; \
514 } \
515 /* Defer non-vector results to LegalizeDAG. */ \
516 if (!Node->getValueType(0).isVector() && \
517 Node->getValueType(0) != MVT::Other) { \
518 Action = TargetLowering::Legal; \
519 break; \
520 } \
521 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
522 } break;
523#include "llvm/IR/VPIntrinsics.def"
524 }
525
526 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
527
528 SmallVector<SDValue, 8> ResultVals;
529 switch (Action) {
530 default: llvm_unreachable("This action is not supported yet!");
531 case TargetLowering::Promote:
532 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
533 "This action is not supported yet!");
534 LLVM_DEBUG(dbgs() << "Promoting\n");
535 Promote(Node, ResultVals);
536 assert(!ResultVals.empty() && "No results for promotion?");
537 break;
538 case TargetLowering::Legal:
539 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
540 break;
541 case TargetLowering::Custom:
542 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
543 if (LowerOperationWrapper(Node, ResultVals))
544 break;
545 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
546 [[fallthrough]];
547 case TargetLowering::Expand:
548 LLVM_DEBUG(dbgs() << "Expanding\n");
549 Expand(Node, ResultVals);
550 break;
551 }
552
553 if (ResultVals.empty())
554 return TranslateLegalizeResults(Op, Node);
555
556 Changed = true;
557 return RecursivelyLegalizeResults(Op, ResultVals);
558}
559
560// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
561// merge them somehow?
562bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
564 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
565
566 if (!Res.getNode())
567 return false;
568
569 if (Res == SDValue(Node, 0))
570 return true;
571
572 // If the original node has one result, take the return value from
573 // LowerOperation as is. It might not be result number 0.
574 if (Node->getNumValues() == 1) {
575 Results.push_back(Res);
576 return true;
577 }
578
579 // If the original node has multiple results, then the return node should
580 // have the same number of results.
581 assert((Node->getNumValues() == Res->getNumValues()) &&
582 "Lowering returned the wrong number of results!");
583
584 // Places new result values base on N result number.
585 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
586 Results.push_back(Res.getValue(I));
587
588 return true;
589}
590
591void VectorLegalizer::PromoteSETCC(SDNode *Node,
593 MVT VecVT = Node->getOperand(0).getSimpleValueType();
594 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
595
596 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
597
598 SDLoc DL(Node);
599 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
600
601 Operands[0] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(0));
602 Operands[1] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(1));
603 Operands[2] = Node->getOperand(2);
604
605 if (Node->getOpcode() == ISD::VP_SETCC) {
606 Operands[3] = Node->getOperand(3); // mask
607 Operands[4] = Node->getOperand(4); // evl
608 }
609
610 SDValue Res = DAG.getNode(Node->getOpcode(), DL, Node->getSimpleValueType(0),
611 Operands, Node->getFlags());
612
613 Results.push_back(Res);
614}
615
616void VectorLegalizer::PromoteSTRICT(SDNode *Node,
618 MVT VecVT = Node->getOperand(1).getSimpleValueType();
619 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
620
621 assert(VecVT.isFloatingPoint());
622
623 SDLoc DL(Node);
624 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
626
627 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
628 if (Node->getOperand(j).getValueType().isVector() &&
629 !(ISD::isVPOpcode(Node->getOpcode()) &&
630 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand.
631 {
632 // promote the vector operand.
633 SDValue Ext =
634 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVecVT, MVT::Other},
635 {Node->getOperand(0), Node->getOperand(j)});
636 Operands[j] = Ext.getValue(0);
637 Chains.push_back(Ext.getValue(1));
638 } else
639 Operands[j] = Node->getOperand(j); // Skip no vector operand.
640
641 SDVTList VTs = DAG.getVTList(NewVecVT, Node->getValueType(1));
642
643 Operands[0] = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
644
645 SDValue Res =
646 DAG.getNode(Node->getOpcode(), DL, VTs, Operands, Node->getFlags());
647
648 SDValue Round =
649 DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VecVT, MVT::Other},
650 {Res.getValue(1), Res.getValue(0),
651 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
652
653 Results.push_back(Round.getValue(0));
654 Results.push_back(Round.getValue(1));
655}
656
657void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
658 // For a few operations there is a specific concept for promotion based on
659 // the operand's type.
660 switch (Node->getOpcode()) {
661 case ISD::SINT_TO_FP:
662 case ISD::UINT_TO_FP:
665 // "Promote" the operation by extending the operand.
666 PromoteINT_TO_FP(Node, Results);
667 return;
668 case ISD::FP_TO_UINT:
669 case ISD::FP_TO_SINT:
672 // Promote the operation by extending the operand.
673 PromoteFP_TO_INT(Node, Results);
674 return;
675 case ISD::VP_SETCC:
676 case ISD::SETCC:
677 // Promote the operation by extending the operand.
678 PromoteSETCC(Node, Results);
679 return;
680 case ISD::STRICT_FADD:
681 case ISD::STRICT_FSUB:
682 case ISD::STRICT_FMUL:
683 case ISD::STRICT_FDIV:
685 case ISD::STRICT_FMA:
686 PromoteSTRICT(Node, Results);
687 return;
688 case ISD::FP_ROUND:
689 case ISD::FP_EXTEND:
690 // These operations are used to do promotion so they can't be promoted
691 // themselves.
692 llvm_unreachable("Don't know how to promote this operation!");
693 }
694
695 // There are currently two cases of vector promotion:
696 // 1) Bitcasting a vector of integers to a different type to a vector of the
697 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
698 // 2) Extending a vector of floats to a vector of the same number of larger
699 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
700 assert(Node->getNumValues() == 1 &&
701 "Can't promote a vector with multiple results!");
702 MVT VT = Node->getSimpleValueType(0);
703 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
704 SDLoc dl(Node);
705 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
706
707 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
708 // Do not promote the mask operand of a VP OP.
709 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) &&
710 ISD::getVPMaskIdx(Node->getOpcode()) == j;
711 if (Node->getOperand(j).getValueType().isVector() && !SkipPromote)
712 if (Node->getOperand(j)
713 .getValueType()
714 .getVectorElementType()
715 .isFloatingPoint() &&
717 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
718 else
719 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
720 else
721 Operands[j] = Node->getOperand(j);
722 }
723
724 SDValue Res =
725 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
726
727 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
730 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res,
731 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
732 else
733 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
734
735 Results.push_back(Res);
736}
737
738void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
740 // INT_TO_FP operations may require the input operand be promoted even
741 // when the type is otherwise legal.
742 bool IsStrict = Node->isStrictFPOpcode();
743 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
744 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
746 "Vectors have different number of elements!");
747
748 SDLoc dl(Node);
749 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
750
751 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
752 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
755 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
756 if (Node->getOperand(j).getValueType().isVector())
757 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
758 else
759 Operands[j] = Node->getOperand(j);
760 }
761
762 if (IsStrict) {
763 SDValue Res = DAG.getNode(Node->getOpcode(), dl,
764 {Node->getValueType(0), MVT::Other}, Operands);
765 Results.push_back(Res);
766 Results.push_back(Res.getValue(1));
767 return;
768 }
769
770 SDValue Res =
771 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
772 Results.push_back(Res);
773}
774
775// For FP_TO_INT we promote the result type to a vector type with wider
776// elements and then truncate the result. This is different from the default
777// PromoteVector which uses bitcast to promote thus assumning that the
778// promoted vector type has the same overall size.
779void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
781 MVT VT = Node->getSimpleValueType(0);
782 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
783 bool IsStrict = Node->isStrictFPOpcode();
785 "Vectors have different number of elements!");
786
787 unsigned NewOpc = Node->getOpcode();
788 // Change FP_TO_UINT to FP_TO_SINT if possible.
789 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
790 if (NewOpc == ISD::FP_TO_UINT &&
791 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
792 NewOpc = ISD::FP_TO_SINT;
793
794 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
795 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
796 NewOpc = ISD::STRICT_FP_TO_SINT;
797
798 SDLoc dl(Node);
799 SDValue Promoted, Chain;
800 if (IsStrict) {
801 Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
802 {Node->getOperand(0), Node->getOperand(1)});
803 Chain = Promoted.getValue(1);
804 } else
805 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
806
807 // Assert that the converted value fits in the original type. If it doesn't
808 // (eg: because the value being converted is too big), then the result of the
809 // original operation was undefined anyway, so the assert is still correct.
810 if (Node->getOpcode() == ISD::FP_TO_UINT ||
811 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
812 NewOpc = ISD::AssertZext;
813 else
814 NewOpc = ISD::AssertSext;
815
816 Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
817 DAG.getValueType(VT.getScalarType()));
818 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
819 Results.push_back(Promoted);
820 if (IsStrict)
821 Results.push_back(Chain);
822}
823
824std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
825 LoadSDNode *LD = cast<LoadSDNode>(N);
826 return TLI.scalarizeVectorLoad(LD, DAG);
827}
828
829SDValue VectorLegalizer::ExpandStore(SDNode *N) {
830 StoreSDNode *ST = cast<StoreSDNode>(N);
831 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
832 return TF;
833}
834
835void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
836 switch (Node->getOpcode()) {
837 case ISD::LOAD: {
838 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
839 Results.push_back(Tmp.first);
840 Results.push_back(Tmp.second);
841 return;
842 }
843 case ISD::STORE:
844 Results.push_back(ExpandStore(Node));
845 return;
847 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
848 Results.push_back(Node->getOperand(i));
849 return;
851 Results.push_back(ExpandSEXTINREG(Node));
852 return;
854 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
855 return;
857 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
858 return;
860 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
861 return;
862 case ISD::BSWAP:
863 Results.push_back(ExpandBSWAP(Node));
864 return;
865 case ISD::VP_BSWAP:
866 Results.push_back(TLI.expandVPBSWAP(Node, DAG));
867 return;
868 case ISD::VSELECT:
869 Results.push_back(ExpandVSELECT(Node));
870 return;
871 case ISD::VP_SELECT:
872 Results.push_back(ExpandVP_SELECT(Node));
873 return;
874 case ISD::VP_SREM:
875 case ISD::VP_UREM:
876 if (SDValue Expanded = ExpandVP_REM(Node)) {
877 Results.push_back(Expanded);
878 return;
879 }
880 break;
881 case ISD::SELECT:
882 Results.push_back(ExpandSELECT(Node));
883 return;
884 case ISD::SELECT_CC: {
885 if (Node->getValueType(0).isScalableVector()) {
886 EVT CondVT = TLI.getSetCCResultType(
887 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
888 SDValue SetCC =
889 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),
890 Node->getOperand(1), Node->getOperand(4));
891 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,
892 Node->getOperand(2),
893 Node->getOperand(3)));
894 return;
895 }
896 break;
897 }
898 case ISD::FP_TO_UINT:
899 ExpandFP_TO_UINT(Node, Results);
900 return;
901 case ISD::UINT_TO_FP:
902 ExpandUINT_TO_FLOAT(Node, Results);
903 return;
904 case ISD::FNEG:
905 Results.push_back(ExpandFNEG(Node));
906 return;
907 case ISD::FSUB:
908 ExpandFSUB(Node, Results);
909 return;
910 case ISD::SETCC:
911 case ISD::VP_SETCC:
912 ExpandSETCC(Node, Results);
913 return;
914 case ISD::ABS:
915 if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
916 Results.push_back(Expanded);
917 return;
918 }
919 break;
920 case ISD::ABDS:
921 case ISD::ABDU:
922 if (SDValue Expanded = TLI.expandABD(Node, DAG)) {
923 Results.push_back(Expanded);
924 return;
925 }
926 break;
927 case ISD::AVGCEILS:
928 case ISD::AVGCEILU:
929 case ISD::AVGFLOORS:
930 case ISD::AVGFLOORU:
931 if (SDValue Expanded = TLI.expandAVG(Node, DAG)) {
932 Results.push_back(Expanded);
933 return;
934 }
935 break;
936 case ISD::BITREVERSE:
937 ExpandBITREVERSE(Node, Results);
938 return;
939 case ISD::VP_BITREVERSE:
940 if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) {
941 Results.push_back(Expanded);
942 return;
943 }
944 break;
945 case ISD::CTPOP:
946 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
947 Results.push_back(Expanded);
948 return;
949 }
950 break;
951 case ISD::VP_CTPOP:
952 if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) {
953 Results.push_back(Expanded);
954 return;
955 }
956 break;
957 case ISD::CTLZ:
959 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
960 Results.push_back(Expanded);
961 return;
962 }
963 break;
964 case ISD::VP_CTLZ:
965 case ISD::VP_CTLZ_ZERO_UNDEF:
966 if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) {
967 Results.push_back(Expanded);
968 return;
969 }
970 break;
971 case ISD::CTTZ:
973 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
974 Results.push_back(Expanded);
975 return;
976 }
977 break;
978 case ISD::VP_CTTZ:
979 case ISD::VP_CTTZ_ZERO_UNDEF:
980 if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) {
981 Results.push_back(Expanded);
982 return;
983 }
984 break;
985 case ISD::FSHL:
986 case ISD::VP_FSHL:
987 case ISD::FSHR:
988 case ISD::VP_FSHR:
989 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
990 Results.push_back(Expanded);
991 return;
992 }
993 break;
994 case ISD::ROTL:
995 case ISD::ROTR:
996 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
997 Results.push_back(Expanded);
998 return;
999 }
1000 break;
1001 case ISD::FMINNUM:
1002 case ISD::FMAXNUM:
1003 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
1004 Results.push_back(Expanded);
1005 return;
1006 }
1007 break;
1008 case ISD::FMINIMUM:
1009 case ISD::FMAXIMUM:
1010 Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(Node, DAG));
1011 return;
1012 case ISD::SMIN:
1013 case ISD::SMAX:
1014 case ISD::UMIN:
1015 case ISD::UMAX:
1016 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1017 Results.push_back(Expanded);
1018 return;
1019 }
1020 break;
1021 case ISD::UADDO:
1022 case ISD::USUBO:
1023 ExpandUADDSUBO(Node, Results);
1024 return;
1025 case ISD::SADDO:
1026 case ISD::SSUBO:
1027 ExpandSADDSUBO(Node, Results);
1028 return;
1029 case ISD::UMULO:
1030 case ISD::SMULO:
1031 ExpandMULO(Node, Results);
1032 return;
1033 case ISD::USUBSAT:
1034 case ISD::SSUBSAT:
1035 case ISD::UADDSAT:
1036 case ISD::SADDSAT:
1037 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1038 Results.push_back(Expanded);
1039 return;
1040 }
1041 break;
1042 case ISD::USHLSAT:
1043 case ISD::SSHLSAT:
1044 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1045 Results.push_back(Expanded);
1046 return;
1047 }
1048 break;
1051 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1052 if (Node->getValueType(0).isScalableVector()) {
1053 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {
1054 Results.push_back(Expanded);
1055 return;
1056 }
1057 }
1058 break;
1059 case ISD::SMULFIX:
1060 case ISD::UMULFIX:
1061 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1062 Results.push_back(Expanded);
1063 return;
1064 }
1065 break;
1066 case ISD::SMULFIXSAT:
1067 case ISD::UMULFIXSAT:
1068 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
1069 // why. Maybe it results in worse codegen compared to the unroll for some
1070 // targets? This should probably be investigated. And if we still prefer to
1071 // unroll an explanation could be helpful.
1072 break;
1073 case ISD::SDIVFIX:
1074 case ISD::UDIVFIX:
1075 ExpandFixedPointDiv(Node, Results);
1076 return;
1077 case ISD::SDIVFIXSAT:
1078 case ISD::UDIVFIXSAT:
1079 break;
1080#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1081 case ISD::STRICT_##DAGN:
1082#include "llvm/IR/ConstrainedOps.def"
1083 ExpandStrictFPOp(Node, Results);
1084 return;
1085 case ISD::VECREDUCE_ADD:
1086 case ISD::VECREDUCE_MUL:
1087 case ISD::VECREDUCE_AND:
1088 case ISD::VECREDUCE_OR:
1089 case ISD::VECREDUCE_XOR:
1100 Results.push_back(TLI.expandVecReduce(Node, DAG));
1101 return;
1104 Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
1105 return;
1106 case ISD::SREM:
1107 case ISD::UREM:
1108 ExpandREM(Node, Results);
1109 return;
1110 case ISD::VP_MERGE:
1111 Results.push_back(ExpandVP_MERGE(Node));
1112 return;
1113 case ISD::FREM:
1114 if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
1115 RTLIB::REM_F80, RTLIB::REM_F128,
1116 RTLIB::REM_PPCF128, Results))
1117 return;
1118
1119 break;
1120 }
1121
1122 SDValue Unrolled = DAG.UnrollVectorOp(Node);
1123 if (Node->getNumValues() == 1) {
1124 Results.push_back(Unrolled);
1125 } else {
1126 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1127 "VectorLegalizer Expand returned wrong number of results!");
1128 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1129 Results.push_back(Unrolled.getValue(I));
1130 }
1131}
1132
1133SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1134 // Lower a select instruction where the condition is a scalar and the
1135 // operands are vectors. Lower this select to VSELECT and implement it
1136 // using XOR AND OR. The selector bit is broadcasted.
1137 EVT VT = Node->getValueType(0);
1138 SDLoc DL(Node);
1139
1140 SDValue Mask = Node->getOperand(0);
1141 SDValue Op1 = Node->getOperand(1);
1142 SDValue Op2 = Node->getOperand(2);
1143
1144 assert(VT.isVector() && !Mask.getValueType().isVector()
1145 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1146
1147 // If we can't even use the basic vector operations of
1148 // AND,OR,XOR, we will have to scalarize the op.
1149 // Notice that the operation may be 'promoted' which means that it is
1150 // 'bitcasted' to another type which is handled.
1151 // Also, we need to be able to construct a splat vector using either
1152 // BUILD_VECTOR or SPLAT_VECTOR.
1153 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1154 // BUILD_VECTOR?
1155 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1156 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1157 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
1158 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1160 VT) == TargetLowering::Expand)
1161 return DAG.UnrollVectorOp(Node);
1162
1163 // Generate a mask operand.
1165
1166 // What is the size of each element in the vector mask.
1167 EVT BitTy = MaskTy.getScalarType();
1168
1169 Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
1170 DAG.getConstant(0, DL, BitTy));
1171
1172 // Broadcast the mask so that the entire vector is all one or all zero.
1173 Mask = DAG.getSplat(MaskTy, DL, Mask);
1174
1175 // Bitcast the operands to be the same type as the mask.
1176 // This is needed when we select between FP types because
1177 // the mask is a vector of integers.
1178 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
1179 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
1180
1181 SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
1182
1183 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
1184 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
1185 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
1186 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1187}
1188
1189SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1190 EVT VT = Node->getValueType(0);
1191
1192 // Make sure that the SRA and SHL instructions are available.
1193 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
1194 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
1195 return DAG.UnrollVectorOp(Node);
1196
1197 SDLoc DL(Node);
1198 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
1199
1200 unsigned BW = VT.getScalarSizeInBits();
1201 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1202 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
1203
1204 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
1205 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
1206}
1207
1208// Generically expand a vector anyext in register to a shuffle of the relevant
1209// lanes into the appropriate locations, with other lanes left undef.
1210SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1211 SDLoc DL(Node);
1212 EVT VT = Node->getValueType(0);
1213 int NumElements = VT.getVectorNumElements();
1214 SDValue Src = Node->getOperand(0);
1215 EVT SrcVT = Src.getValueType();
1216 int NumSrcElements = SrcVT.getVectorNumElements();
1217
1218 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1219 // into a larger vector type.
1220 if (SrcVT.bitsLE(VT)) {
1221 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1222 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1223 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1224 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1225 NumSrcElements);
1226 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1227 Src, DAG.getVectorIdxConstant(0, DL));
1228 }
1229
1230 // Build a base mask of undef shuffles.
1231 SmallVector<int, 16> ShuffleMask;
1232 ShuffleMask.resize(NumSrcElements, -1);
1233
1234 // Place the extended lanes into the correct locations.
1235 int ExtLaneScale = NumSrcElements / NumElements;
1236 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1237 for (int i = 0; i < NumElements; ++i)
1238 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1239
1240 return DAG.getNode(
1241 ISD::BITCAST, DL, VT,
1242 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1243}
1244
1245SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1246 SDLoc DL(Node);
1247 EVT VT = Node->getValueType(0);
1248 SDValue Src = Node->getOperand(0);
1249 EVT SrcVT = Src.getValueType();
1250
1251 // First build an any-extend node which can be legalized above when we
1252 // recurse through it.
1253 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1254
1255 // Now we need sign extend. Do this by shifting the elements. Even if these
1256 // aren't legal operations, they have a better chance of being legalized
1257 // without full scalarization than the sign extension does.
1258 unsigned EltWidth = VT.getScalarSizeInBits();
1259 unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1260 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1261 return DAG.getNode(ISD::SRA, DL, VT,
1262 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1263 ShiftAmount);
1264}
1265
1266// Generically expand a vector zext in register to a shuffle of the relevant
1267// lanes into the appropriate locations, a blend of zero into the high bits,
1268// and a bitcast to the wider element type.
1269SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1270 SDLoc DL(Node);
1271 EVT VT = Node->getValueType(0);
1272 int NumElements = VT.getVectorNumElements();
1273 SDValue Src = Node->getOperand(0);
1274 EVT SrcVT = Src.getValueType();
1275 int NumSrcElements = SrcVT.getVectorNumElements();
1276
1277 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1278 // into a larger vector type.
1279 if (SrcVT.bitsLE(VT)) {
1280 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1281 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1282 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1283 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1284 NumSrcElements);
1285 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1286 Src, DAG.getVectorIdxConstant(0, DL));
1287 }
1288
1289 // Build up a zero vector to blend into this one.
1290 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1291
1292 // Shuffle the incoming lanes into the correct position, and pull all other
1293 // lanes from the zero vector.
1294 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1295
1296 int ExtLaneScale = NumSrcElements / NumElements;
1297 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1298 for (int i = 0; i < NumElements; ++i)
1299 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1300
1301 return DAG.getNode(ISD::BITCAST, DL, VT,
1302 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1303}
1304
1305static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1306 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1307 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1308 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1309 ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1310}
1311
1312SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1313 EVT VT = Node->getValueType(0);
1314
1315 // Scalable vectors can't use shuffle expansion.
1316 if (VT.isScalableVector())
1317 return TLI.expandBSWAP(Node, DAG);
1318
1319 // Generate a byte wise shuffle mask for the BSWAP.
1320 SmallVector<int, 16> ShuffleMask;
1321 createBSWAPShuffleMask(VT, ShuffleMask);
1322 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1323
1324 // Only emit a shuffle if the mask is legal.
1325 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1326 SDLoc DL(Node);
1327 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1328 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1329 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1330 }
1331
1332 // If we have the appropriate vector bit operations, it is better to use them
1333 // than unrolling and expanding each component.
1334 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1335 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1336 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1337 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1338 return TLI.expandBSWAP(Node, DAG);
1339
1340 // Otherwise unroll.
1341 return DAG.UnrollVectorOp(Node);
1342}
1343
1344void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1346 EVT VT = Node->getValueType(0);
1347
1348 // We can't unroll or use shuffles for scalable vectors.
1349 if (VT.isScalableVector()) {
1350 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1351 return;
1352 }
1353
1354 // If we have the scalar operation, it's probably cheaper to unroll it.
1355 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1356 SDValue Tmp = DAG.UnrollVectorOp(Node);
1357 Results.push_back(Tmp);
1358 return;
1359 }
1360
1361 // If the vector element width is a whole number of bytes, test if its legal
1362 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1363 // vector. This greatly reduces the number of bit shifts necessary.
1364 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1365 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1366 SmallVector<int, 16> BSWAPMask;
1367 createBSWAPShuffleMask(VT, BSWAPMask);
1368
1369 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1370 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1371 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1372 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1373 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1374 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1375 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1376 SDLoc DL(Node);
1377 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1378 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1379 BSWAPMask);
1380 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1381 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1382 Results.push_back(Op);
1383 return;
1384 }
1385 }
1386
1387 // If we have the appropriate vector bit operations, it is better to use them
1388 // than unrolling and expanding each component.
1389 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1390 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1391 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1392 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1393 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1394 return;
1395 }
1396
1397 // Otherwise unroll.
1398 SDValue Tmp = DAG.UnrollVectorOp(Node);
1399 Results.push_back(Tmp);
1400}
1401
1402SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1403 // Implement VSELECT in terms of XOR, AND, OR
1404 // on platforms which do not support blend natively.
1405 SDLoc DL(Node);
1406
1407 SDValue Mask = Node->getOperand(0);
1408 SDValue Op1 = Node->getOperand(1);
1409 SDValue Op2 = Node->getOperand(2);
1410
1411 EVT VT = Mask.getValueType();
1412
1413 // If we can't even use the basic vector operations of
1414 // AND,OR,XOR, we will have to scalarize the op.
1415 // Notice that the operation may be 'promoted' which means that it is
1416 // 'bitcasted' to another type which is handled.
1417 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1418 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1419 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1420 return DAG.UnrollVectorOp(Node);
1421
1422 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1423 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1424 // vector constant to mask with.
1425 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1426 auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1427 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1428 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1429 Op1.getValueType().getVectorElementType() == MVT::i1))
1430 return DAG.UnrollVectorOp(Node);
1431
1432 // If the mask and the type are different sizes, unroll the vector op. This
1433 // can occur when getSetCCResultType returns something that is different in
1434 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1435 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1436 return DAG.UnrollVectorOp(Node);
1437
1438 // Bitcast the operands to be the same type as the mask.
1439 // This is needed when we select between FP types because
1440 // the mask is a vector of integers.
1441 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1442 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1443
1444 SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1445
1446 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1447 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1448 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1449 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1450}
1451
1452SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1453 // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1454 // do not support it natively.
1455 SDLoc DL(Node);
1456
1457 SDValue Mask = Node->getOperand(0);
1458 SDValue Op1 = Node->getOperand(1);
1459 SDValue Op2 = Node->getOperand(2);
1460 SDValue EVL = Node->getOperand(3);
1461
1462 EVT VT = Mask.getValueType();
1463
1464 // If we can't even use the basic vector operations of
1465 // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1466 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1467 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1468 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1469 return DAG.UnrollVectorOp(Node);
1470
1471 // This operation also isn't safe when the operands aren't also booleans.
1472 if (Op1.getValueType().getVectorElementType() != MVT::i1)
1473 return DAG.UnrollVectorOp(Node);
1474
1475 SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1476 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);
1477
1478 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL);
1479 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL);
1480 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL);
1481}
1482
1483SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1484 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1485 // indices less than the EVL/pivot are true. Combine that with the original
1486 // mask for a full-length mask. Use a full-length VSELECT to select between
1487 // the true and false values.
1488 SDLoc DL(Node);
1489
1490 SDValue Mask = Node->getOperand(0);
1491 SDValue Op1 = Node->getOperand(1);
1492 SDValue Op2 = Node->getOperand(2);
1493 SDValue EVL = Node->getOperand(3);
1494
1495 EVT MaskVT = Mask.getValueType();
1496 bool IsFixedLen = MaskVT.isFixedLengthVector();
1497
1498 EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1499 MaskVT.getVectorElementCount());
1500
1501 // If we can't construct the EVL mask efficiently, it's better to unroll.
1502 if ((IsFixedLen &&
1503 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1504 (!IsFixedLen &&
1505 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1506 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1507 return DAG.UnrollVectorOp(Node);
1508
1509 // If using a SETCC would result in a different type than the mask type,
1510 // unroll.
1511 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1512 EVLVecVT) != MaskVT)
1513 return DAG.UnrollVectorOp(Node);
1514
1515 SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1516 SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);
1517 SDValue EVLMask =
1518 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1519
1520 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1521 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1522}
1523
1524SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1525 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.
1526 EVT VT = Node->getValueType(0);
1527
1528 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1529
1530 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1531 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1532 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1533 return SDValue();
1534
1535 SDLoc DL(Node);
1536
1537 SDValue Dividend = Node->getOperand(0);
1538 SDValue Divisor = Node->getOperand(1);
1539 SDValue Mask = Node->getOperand(2);
1540 SDValue EVL = Node->getOperand(3);
1541
1542 // X % Y -> X-X/Y*Y
1543 SDValue Div = DAG.getNode(DivOpc, DL, VT, Dividend, Divisor, Mask, EVL);
1544 SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL);
1545 return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL);
1546}
1547
1548void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1550 // Attempt to expand using TargetLowering.
1551 SDValue Result, Chain;
1552 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1553 Results.push_back(Result);
1554 if (Node->isStrictFPOpcode())
1555 Results.push_back(Chain);
1556 return;
1557 }
1558
1559 // Otherwise go ahead and unroll.
1560 if (Node->isStrictFPOpcode()) {
1561 UnrollStrictFPOp(Node, Results);
1562 return;
1563 }
1564
1565 Results.push_back(DAG.UnrollVectorOp(Node));
1566}
1567
1568void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1570 bool IsStrict = Node->isStrictFPOpcode();
1571 unsigned OpNo = IsStrict ? 1 : 0;
1572 SDValue Src = Node->getOperand(OpNo);
1573 EVT VT = Src.getValueType();
1574 SDLoc DL(Node);
1575
1576 // Attempt to expand using TargetLowering.
1578 SDValue Chain;
1579 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1580 Results.push_back(Result);
1581 if (IsStrict)
1582 Results.push_back(Chain);
1583 return;
1584 }
1585
1586 // Make sure that the SINT_TO_FP and SRL instructions are available.
1587 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1588 TargetLowering::Expand) ||
1589 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1590 TargetLowering::Expand)) ||
1591 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1592 if (IsStrict) {
1593 UnrollStrictFPOp(Node, Results);
1594 return;
1595 }
1596
1597 Results.push_back(DAG.UnrollVectorOp(Node));
1598 return;
1599 }
1600
1601 unsigned BW = VT.getScalarSizeInBits();
1602 assert((BW == 64 || BW == 32) &&
1603 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1604
1605 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1606
1607 // Constants to clear the upper part of the word.
1608 // Notice that we can also use SHL+SHR, but using a constant is slightly
1609 // faster on x86.
1610 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1611 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1612
1613 // Two to the power of half-word-size.
1614 SDValue TWOHW =
1615 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1616
1617 // Clear upper part of LO, lower HI
1618 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1619 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1620
1621 if (IsStrict) {
1622 // Convert hi and lo to floats
1623 // Convert the hi part back to the upper values
1624 // TODO: Can any fast-math-flags be set on these nodes?
1626 {Node->getValueType(0), MVT::Other},
1627 {Node->getOperand(0), HI});
1628 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1629 {fHI.getValue(1), fHI, TWOHW});
1631 {Node->getValueType(0), MVT::Other},
1632 {Node->getOperand(0), LO});
1633
1634 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1635 fLO.getValue(1));
1636
1637 // Add the two halves
1638 SDValue Result =
1639 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1640 {TF, fHI, fLO});
1641
1642 Results.push_back(Result);
1643 Results.push_back(Result.getValue(1));
1644 return;
1645 }
1646
1647 // Convert hi and lo to floats
1648 // Convert the hi part back to the upper values
1649 // TODO: Can any fast-math-flags be set on these nodes?
1650 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1651 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1652 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1653
1654 // Add the two halves
1655 Results.push_back(
1656 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1657}
1658
1659SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1660 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1661 SDLoc DL(Node);
1662 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1663 // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1664 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1665 Node->getOperand(0));
1666 }
1667 return DAG.UnrollVectorOp(Node);
1668}
1669
1670void VectorLegalizer::ExpandFSUB(SDNode *Node,
1672 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1673 // we can defer this to operation legalization where it will be lowered as
1674 // a+(-b).
1675 EVT VT = Node->getValueType(0);
1676 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1677 TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1678 return; // Defer to LegalizeDAG
1679
1680 SDValue Tmp = DAG.UnrollVectorOp(Node);
1681 Results.push_back(Tmp);
1682}
1683
1684void VectorLegalizer::ExpandSETCC(SDNode *Node,
1686 bool NeedInvert = false;
1687 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
1688 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
1689 Node->getOpcode() == ISD::STRICT_FSETCCS;
1690 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
1691 unsigned Offset = IsStrict ? 1 : 0;
1692
1693 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
1694 SDValue LHS = Node->getOperand(0 + Offset);
1695 SDValue RHS = Node->getOperand(1 + Offset);
1696 SDValue CC = Node->getOperand(2 + Offset);
1697
1698 MVT OpVT = LHS.getSimpleValueType();
1699 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1700
1701 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1702 if (IsStrict) {
1703 UnrollStrictFPOp(Node, Results);
1704 return;
1705 }
1706 Results.push_back(UnrollVSETCC(Node));
1707 return;
1708 }
1709
1710 SDValue Mask, EVL;
1711 if (IsVP) {
1712 Mask = Node->getOperand(3 + Offset);
1713 EVL = Node->getOperand(4 + Offset);
1714 }
1715
1716 SDLoc dl(Node);
1717 bool Legalized =
1718 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
1719 EVL, NeedInvert, dl, Chain, IsSignaling);
1720
1721 if (Legalized) {
1722 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1723 // condition code, create a new SETCC node.
1724 if (CC.getNode()) {
1725 if (IsStrict) {
1726 LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
1727 {Chain, LHS, RHS, CC}, Node->getFlags());
1728 Chain = LHS.getValue(1);
1729 } else if (IsVP) {
1730 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
1731 {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
1732 } else {
1733 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1734 Node->getFlags());
1735 }
1736 }
1737
1738 // If we expanded the SETCC by inverting the condition code, then wrap
1739 // the existing SETCC in a NOT to restore the intended condition.
1740 if (NeedInvert) {
1741 if (!IsVP)
1742 LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1743 else
1744 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
1745 }
1746 } else {
1747 assert(!IsStrict && "Don't know how to expand for strict nodes.");
1748
1749 // Otherwise, SETCC for the given comparison type must be completely
1750 // illegal; expand it into a SELECT_CC.
1751 EVT VT = Node->getValueType(0);
1752 LHS =
1753 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1754 DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1755 DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1756 LHS->setFlags(Node->getFlags());
1757 }
1758
1759 Results.push_back(LHS);
1760 if (IsStrict)
1761 Results.push_back(Chain);
1762}
1763
1764void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1766 SDValue Result, Overflow;
1767 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1768 Results.push_back(Result);
1769 Results.push_back(Overflow);
1770}
1771
1772void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1774 SDValue Result, Overflow;
1775 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1776 Results.push_back(Result);
1777 Results.push_back(Overflow);
1778}
1779
1780void VectorLegalizer::ExpandMULO(SDNode *Node,
1782 SDValue Result, Overflow;
1783 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1784 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1785
1786 Results.push_back(Result);
1787 Results.push_back(Overflow);
1788}
1789
1790void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1792 SDNode *N = Node;
1793 if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1794 N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1795 Results.push_back(Expanded);
1796}
1797
1798void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1800 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1801 ExpandUINT_TO_FLOAT(Node, Results);
1802 return;
1803 }
1804 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1805 ExpandFP_TO_UINT(Node, Results);
1806 return;
1807 }
1808
1809 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1810 Node->getOpcode() == ISD::STRICT_FSETCCS) {
1811 ExpandSETCC(Node, Results);
1812 return;
1813 }
1814
1815 UnrollStrictFPOp(Node, Results);
1816}
1817
1818void VectorLegalizer::ExpandREM(SDNode *Node,
1820 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1821 "Expected REM node");
1822
1824 if (!TLI.expandREM(Node, Result, DAG))
1825 Result = DAG.UnrollVectorOp(Node);
1826 Results.push_back(Result);
1827}
1828
1829// Try to expand libm nodes into vector math routine calls. Callers provide the
1830// LibFunc equivalent of the passed in Node, which is used to lookup mappings
1831// within TargetLibraryInfo. The only mappings considered are those where the
1832// result and all operands are the same vector type. While predicated nodes are
1833// not supported, we will emit calls to masked routines by passing in an all
1834// true mask.
1835bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
1837 // Chain must be propagated but currently strict fp operations are down
1838 // converted to their none strict counterpart.
1839 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
1840
1841 const char *LCName = TLI.getLibcallName(LC);
1842 if (!LCName)
1843 return false;
1844 LLVM_DEBUG(dbgs() << "Looking for vector variant of " << LCName << "\n");
1845
1846 EVT VT = Node->getValueType(0);
1848
1849 // Lookup a vector function equivalent to the specified libcall. Prefer
1850 // unmasked variants but we will generate a mask if need be.
1851 const TargetLibraryInfo &TLibInfo = DAG.getLibInfo();
1852 const VecDesc *VD = TLibInfo.getVectorMappingInfo(LCName, VL, false);
1853 if (!VD)
1854 VD = TLibInfo.getVectorMappingInfo(LCName, VL, /*Masked=*/true);
1855 if (!VD)
1856 return false;
1857
1858 LLVMContext *Ctx = DAG.getContext();
1859 Type *Ty = VT.getTypeForEVT(*Ctx);
1860 Type *ScalarTy = Ty->getScalarType();
1861
1862 // Construct a scalar function type based on Node's operands.
1864 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
1865 assert(Node->getOperand(i).getValueType() == VT &&
1866 "Expected matching vector types!");
1867 ArgTys.push_back(ScalarTy);
1868 }
1869 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys, false);
1870
1871 // Generate call information for the vector function.
1872 const std::string MangledName = VD->getVectorFunctionABIVariantString();
1873 auto OptVFInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy);
1874 if (!OptVFInfo)
1875 return false;
1876
1877 LLVM_DEBUG(dbgs() << "Found vector variant " << VD->getVectorFnName()
1878 << "\n");
1879
1880 // Sanity check just in case OptVFInfo has unexpected parameters.
1881 if (OptVFInfo->Shape.Parameters.size() !=
1882 Node->getNumOperands() + VD->isMasked())
1883 return false;
1884
1885 // Collect vector call operands.
1886
1887 SDLoc DL(Node);
1890 Entry.IsSExt = false;
1891 Entry.IsZExt = false;
1892
1893 unsigned OpNum = 0;
1894 for (auto &VFParam : OptVFInfo->Shape.Parameters) {
1895 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1896 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1897 Entry.Node = DAG.getBoolConstant(true, DL, MaskVT, VT);
1898 Entry.Ty = MaskVT.getTypeForEVT(*Ctx);
1899 Args.push_back(Entry);
1900 continue;
1901 }
1902
1903 // Only vector operands are supported.
1904 if (VFParam.ParamKind != VFParamKind::Vector)
1905 return false;
1906
1907 Entry.Node = Node->getOperand(OpNum++);
1908 Entry.Ty = Ty;
1909 Args.push_back(Entry);
1910 }
1911
1912 // Emit a call to the vector function.
1913 SDValue Callee = DAG.getExternalSymbol(VD->getVectorFnName().data(),
1914 TLI.getPointerTy(DAG.getDataLayout()));
1916 CLI.setDebugLoc(DL)
1917 .setChain(DAG.getEntryNode())
1918 .setLibCallee(CallingConv::C, Ty, Callee, std::move(Args));
1919
1920 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1921 Results.push_back(CallResult.first);
1922 return true;
1923}
1924
1925/// Try to expand the node to a vector libcall based on the result type.
1926bool VectorLegalizer::tryExpandVecMathCall(
1927 SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
1928 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
1931 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1932 Call_F80, Call_F128, Call_PPCF128);
1933
1934 if (LC == RTLIB::UNKNOWN_LIBCALL)
1935 return false;
1936
1937 return tryExpandVecMathCall(Node, LC, Results);
1938}
1939
1940void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1942 EVT VT = Node->getValueType(0);
1943 EVT EltVT = VT.getVectorElementType();
1944 unsigned NumElems = VT.getVectorNumElements();
1945 unsigned NumOpers = Node->getNumOperands();
1946 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1947
1948 EVT TmpEltVT = EltVT;
1949 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1950 Node->getOpcode() == ISD::STRICT_FSETCCS)
1951 TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1952 *DAG.getContext(), TmpEltVT);
1953
1954 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1955 SDValue Chain = Node->getOperand(0);
1956 SDLoc dl(Node);
1957
1958 SmallVector<SDValue, 32> OpValues;
1959 SmallVector<SDValue, 32> OpChains;
1960 for (unsigned i = 0; i < NumElems; ++i) {
1962 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1963
1964 // The Chain is the first operand.
1965 Opers.push_back(Chain);
1966
1967 // Now process the remaining operands.
1968 for (unsigned j = 1; j < NumOpers; ++j) {
1969 SDValue Oper = Node->getOperand(j);
1970 EVT OperVT = Oper.getValueType();
1971
1972 if (OperVT.isVector())
1973 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1974 OperVT.getVectorElementType(), Oper, Idx);
1975
1976 Opers.push_back(Oper);
1977 }
1978
1979 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1980 SDValue ScalarResult = ScalarOp.getValue(0);
1981 SDValue ScalarChain = ScalarOp.getValue(1);
1982
1983 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1984 Node->getOpcode() == ISD::STRICT_FSETCCS)
1985 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1986 DAG.getAllOnesConstant(dl, EltVT),
1987 DAG.getConstant(0, dl, EltVT));
1988
1989 OpValues.push_back(ScalarResult);
1990 OpChains.push_back(ScalarChain);
1991 }
1992
1993 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1994 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1995
1996 Results.push_back(Result);
1997 Results.push_back(NewChain);
1998}
1999
2000SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2001 EVT VT = Node->getValueType(0);
2002 unsigned NumElems = VT.getVectorNumElements();
2003 EVT EltVT = VT.getVectorElementType();
2004 SDValue LHS = Node->getOperand(0);
2005 SDValue RHS = Node->getOperand(1);
2006 SDValue CC = Node->getOperand(2);
2007 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2008 SDLoc dl(Node);
2009 SmallVector<SDValue, 8> Ops(NumElems);
2010 for (unsigned i = 0; i < NumElems; ++i) {
2011 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
2012 DAG.getVectorIdxConstant(i, dl));
2013 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
2014 DAG.getVectorIdxConstant(i, dl));
2015 Ops[i] = DAG.getNode(ISD::SETCC, dl,
2016 TLI.getSetCCResultType(DAG.getDataLayout(),
2017 *DAG.getContext(), TmpEltVT),
2018 LHSElem, RHSElem, CC);
2019 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2020 DAG.getConstant(0, dl, EltVT));
2021 }
2022 return DAG.getBuildVector(VT, dl, Ops);
2023}
2024
2026 return VectorLegalizer(*this).Run();
2027}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
BinaryOperator * Mul
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
size_t size() const
Definition: Function.h:811
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:227
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:492
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:548
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
Provides info so a possible vectorization of a function can be computed.
bool isMasked() const
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ Entry
Definition: COFF.h:811
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:758
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:237
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:731
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:478
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1354
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1385
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:251
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:561
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:722
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:368
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:994
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:240
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1046
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:374
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:791
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:485
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:798
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1370
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:391
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1374
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:696
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:828
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:256
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1384
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:479
@ FPTRUNC_ROUND
Definition: ISDOpcodes.h:482
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:911
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:948
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:381
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:412
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:782
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:664
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:452
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1367
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:730
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1371
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:938
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:328
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:648
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:502
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:350
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:735
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1386
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:629
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:324
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1379
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:653
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:713
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:993
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:537
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:788
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:864
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:750
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:980
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:360
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:332
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:817
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:806
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:676
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:387
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:896
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:744
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:451
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1387
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:445
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:467
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:444
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:999
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:844
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:472
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:688
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:684
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:659
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1368
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:401
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:953
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:877
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:839
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:863
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1375
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:794
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1355
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:495
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:341
@ AssertZext
Definition: ISDOpcodes.h:62
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:517
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1548
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1528
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
#define N
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:203
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.