Line data Source code
1 : //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file performs vector type splitting and scalarization for LegalizeTypes.
11 : // Scalarization is the act of changing a computation in an illegal one-element
12 : // vector type to be a computation in its scalar element type. For example,
13 : // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 : // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 : // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16 : // types.
17 : // Splitting is the act of changing a computation in an invalid vector type to
18 : // be a computation in two vectors of half the size. For example, implementing
19 : // <128 x f32> operations in terms of two <64 x f32> operations.
20 : //
21 : //===----------------------------------------------------------------------===//
22 :
23 : #include "LegalizeTypes.h"
24 : #include "llvm/IR/DataLayout.h"
25 : #include "llvm/Support/ErrorHandling.h"
26 : #include "llvm/Support/raw_ostream.h"
27 : using namespace llvm;
28 :
29 : #define DEBUG_TYPE "legalize-types"
30 :
31 : //===----------------------------------------------------------------------===//
32 : // Result Vector Scalarization: <1 x ty> -> ty.
33 : //===----------------------------------------------------------------------===//
34 :
35 63894 : void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36 : LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
37 : dbgs() << "\n");
38 63894 : SDValue R = SDValue();
39 :
40 127788 : switch (N->getOpcode()) {
41 0 : default:
42 : #ifndef NDEBUG
43 : dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
44 : N->dump(&DAG);
45 : dbgs() << "\n";
46 : #endif
47 0 : report_fatal_error("Do not know how to scalarize the result of this "
48 : "operator!\n");
49 :
50 0 : case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
51 17312 : case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
52 9847 : case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
53 5286 : case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
54 754 : case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55 0 : case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
56 0 : case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57 148 : case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58 5376 : case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59 151 : case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60 21 : case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61 353 : case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62 209 : case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63 0 : case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64 1138 : case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65 133 : case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66 0 : case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
67 20 : case ISD::ANY_EXTEND_VECTOR_INREG:
68 : case ISD::SIGN_EXTEND_VECTOR_INREG:
69 : case ISD::ZERO_EXTEND_VECTOR_INREG:
70 20 : R = ScalarizeVecRes_VecInregOp(N);
71 20 : break;
72 17522 : case ISD::ANY_EXTEND:
73 : case ISD::BITREVERSE:
74 : case ISD::BSWAP:
75 : case ISD::CTLZ:
76 : case ISD::CTLZ_ZERO_UNDEF:
77 : case ISD::CTPOP:
78 : case ISD::CTTZ:
79 : case ISD::CTTZ_ZERO_UNDEF:
80 : case ISD::FABS:
81 : case ISD::FCEIL:
82 : case ISD::FCOS:
83 : case ISD::FEXP:
84 : case ISD::FEXP2:
85 : case ISD::FFLOOR:
86 : case ISD::FLOG:
87 : case ISD::FLOG10:
88 : case ISD::FLOG2:
89 : case ISD::FNEARBYINT:
90 : case ISD::FNEG:
91 : case ISD::FP_EXTEND:
92 : case ISD::FP_TO_SINT:
93 : case ISD::FP_TO_UINT:
94 : case ISD::FRINT:
95 : case ISD::FROUND:
96 : case ISD::FSIN:
97 : case ISD::FSQRT:
98 : case ISD::FTRUNC:
99 : case ISD::SIGN_EXTEND:
100 : case ISD::SINT_TO_FP:
101 : case ISD::TRUNCATE:
102 : case ISD::UINT_TO_FP:
103 : case ISD::ZERO_EXTEND:
104 : case ISD::FCANONICALIZE:
105 17522 : R = ScalarizeVecRes_UnaryOp(N);
106 17522 : break;
107 :
108 5544 : case ISD::ADD:
109 : case ISD::AND:
110 : case ISD::FADD:
111 : case ISD::FCOPYSIGN:
112 : case ISD::FDIV:
113 : case ISD::FMUL:
114 : case ISD::FMINNUM:
115 : case ISD::FMAXNUM:
116 : case ISD::FMINNAN:
117 : case ISD::FMAXNAN:
118 : case ISD::SMIN:
119 : case ISD::SMAX:
120 : case ISD::UMIN:
121 : case ISD::UMAX:
122 :
123 : case ISD::SADDSAT:
124 :
125 : case ISD::FPOW:
126 : case ISD::FREM:
127 : case ISD::FSUB:
128 : case ISD::MUL:
129 : case ISD::OR:
130 : case ISD::SDIV:
131 : case ISD::SREM:
132 : case ISD::SUB:
133 : case ISD::UDIV:
134 : case ISD::UREM:
135 : case ISD::XOR:
136 : case ISD::SHL:
137 : case ISD::SRA:
138 : case ISD::SRL:
139 5544 : R = ScalarizeVecRes_BinOp(N);
140 5544 : break;
141 44 : case ISD::FMA:
142 44 : R = ScalarizeVecRes_TernaryOp(N);
143 44 : break;
144 36 : case ISD::STRICT_FADD:
145 : case ISD::STRICT_FSUB:
146 : case ISD::STRICT_FMUL:
147 : case ISD::STRICT_FDIV:
148 : case ISD::STRICT_FREM:
149 : case ISD::STRICT_FSQRT:
150 : case ISD::STRICT_FMA:
151 : case ISD::STRICT_FPOW:
152 : case ISD::STRICT_FPOWI:
153 : case ISD::STRICT_FSIN:
154 : case ISD::STRICT_FCOS:
155 : case ISD::STRICT_FEXP:
156 : case ISD::STRICT_FEXP2:
157 : case ISD::STRICT_FLOG:
158 : case ISD::STRICT_FLOG10:
159 : case ISD::STRICT_FLOG2:
160 : case ISD::STRICT_FRINT:
161 : case ISD::STRICT_FNEARBYINT:
162 36 : R = ScalarizeVecRes_StrictFPOp(N);
163 36 : break;
164 : }
165 :
166 : // If R is null, the sub-method took care of registering the result.
167 63894 : if (R.getNode())
168 63894 : SetScalarizedVector(SDValue(N, ResNo), R);
169 63894 : }
170 :
171 5544 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
172 11088 : SDValue LHS = GetScalarizedVector(N->getOperand(0));
173 11088 : SDValue RHS = GetScalarizedVector(N->getOperand(1));
174 5544 : return DAG.getNode(N->getOpcode(), SDLoc(N),
175 5544 : LHS.getValueType(), LHS, RHS, N->getFlags());
176 : }
177 :
178 44 : SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
179 88 : SDValue Op0 = GetScalarizedVector(N->getOperand(0));
180 88 : SDValue Op1 = GetScalarizedVector(N->getOperand(1));
181 88 : SDValue Op2 = GetScalarizedVector(N->getOperand(2));
182 44 : return DAG.getNode(N->getOpcode(), SDLoc(N),
183 44 : Op0.getValueType(), Op0, Op1, Op2);
184 : }
185 :
186 36 : SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
187 72 : EVT VT = N->getValueType(0).getVectorElementType();
188 36 : unsigned NumOpers = N->getNumOperands();
189 36 : SDValue Chain = N->getOperand(0);
190 36 : EVT ValueVTs[] = {VT, MVT::Other};
191 : SDLoc dl(N);
192 :
193 : SmallVector<SDValue, 4> Opers;
194 :
195 : // The Chain is the first operand.
196 36 : Opers.push_back(Chain);
197 :
198 : // Now process the remaining operands.
199 90 : for (unsigned i = 1; i < NumOpers; ++i) {
200 54 : SDValue Oper = N->getOperand(i);
201 :
202 162 : if (Oper.getValueType().isVector())
203 52 : Oper = GetScalarizedVector(Oper);
204 :
205 54 : Opers.push_back(Oper);
206 : }
207 :
208 108 : SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
209 :
210 : // Legalize the chain result - switch anything that used the old chain to
211 : // use the new one.
212 36 : ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
213 36 : return Result;
214 : }
215 :
216 0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
217 : unsigned ResNo) {
218 0 : SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
219 0 : return GetScalarizedVector(Op);
220 : }
221 :
222 17312 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
223 17312 : SDValue Op = N->getOperand(0);
224 17312 : if (Op.getValueType().isVector()
225 17699 : && Op.getValueType().getVectorNumElements() == 1
226 17312 : && !isSimpleLegalType(Op.getValueType()))
227 130 : Op = GetScalarizedVector(Op);
228 34624 : EVT NewVT = N->getValueType(0).getVectorElementType();
229 17312 : return DAG.getNode(ISD::BITCAST, SDLoc(N),
230 17312 : NewVT, Op);
231 : }
232 :
233 9847 : SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
234 19694 : EVT EltVT = N->getValueType(0).getVectorElementType();
235 9847 : SDValue InOp = N->getOperand(0);
236 : // The BUILD_VECTOR operands may be of wider element types and
237 : // we may need to truncate them back to the requested return type.
238 9847 : if (EltVT.isInteger())
239 14950 : return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
240 2372 : return InOp;
241 : }
242 :
243 5286 : SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
244 5286 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
245 5286 : N->getValueType(0).getVectorElementType(),
246 15858 : N->getOperand(0), N->getOperand(1));
247 : }
248 :
249 754 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
250 1508 : EVT NewVT = N->getValueType(0).getVectorElementType();
251 1508 : SDValue Op = GetScalarizedVector(N->getOperand(0));
252 754 : return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
253 754 : NewVT, Op, N->getOperand(1));
254 : }
255 :
256 0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
257 0 : SDValue Op = GetScalarizedVector(N->getOperand(0));
258 0 : return DAG.getNode(ISD::FPOWI, SDLoc(N),
259 0 : Op.getValueType(), Op, N->getOperand(1));
260 : }
261 :
262 148 : SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
263 : // The value to insert may have a wider type than the vector element type,
264 : // so be sure to truncate it to the element type if necessary.
265 148 : SDValue Op = N->getOperand(1);
266 296 : EVT EltVT = N->getValueType(0).getVectorElementType();
267 172 : if (Op.getValueType() != EltVT)
268 : // FIXME: Can this happen for floating point types?
269 0 : Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
270 148 : return Op;
271 : }
272 :
273 5376 : SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
274 : assert(N->isUnindexed() && "Indexed vector load?");
275 :
276 5376 : SDValue Result = DAG.getLoad(
277 : ISD::UNINDEXED, N->getExtensionType(),
278 5376 : N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
279 5376 : N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
280 5376 : N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
281 5376 : N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
282 21504 : N->getAAInfo());
283 :
284 : // Legalize the chain result - switch anything that used the old chain to
285 : // use the new one.
286 5376 : ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
287 5376 : return Result;
288 : }
289 :
290 17522 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
291 : // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
292 35044 : EVT DestVT = N->getValueType(0).getVectorElementType();
293 17522 : SDValue Op = N->getOperand(0);
294 17522 : EVT OpVT = Op.getValueType();
295 : SDLoc DL(N);
296 : // The result needs scalarizing, but it's not a given that the source does.
297 : // This is a workaround for targets where it's impossible to scalarize the
298 : // result of a conversion, because the source type is legal.
299 : // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
300 : // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
301 : // legal and was not scalarized.
302 : // See the similar logic in ScalarizeVecRes_SETCC
303 17522 : if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
304 17516 : Op = GetScalarizedVector(Op);
305 : } else {
306 6 : EVT VT = OpVT.getVectorElementType();
307 6 : Op = DAG.getNode(
308 : ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
309 6 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
310 : }
311 35044 : return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
312 : }
313 :
314 21 : SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
315 42 : EVT EltVT = N->getValueType(0).getVectorElementType();
316 42 : EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
317 42 : SDValue LHS = GetScalarizedVector(N->getOperand(0));
318 21 : return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
319 63 : LHS, DAG.getValueType(ExtVT));
320 : }
321 :
322 20 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
323 : SDLoc DL(N);
324 20 : SDValue Op = N->getOperand(0);
325 :
326 20 : EVT OpVT = Op.getValueType();
327 20 : EVT OpEltVT = OpVT.getVectorElementType();
328 40 : EVT EltVT = N->getValueType(0).getVectorElementType();
329 :
330 20 : if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
331 0 : Op = GetScalarizedVector(Op);
332 : } else {
333 20 : Op = DAG.getNode(
334 : ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
335 20 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
336 : }
337 :
338 40 : switch (N->getOpcode()) {
339 20 : case ISD::ANY_EXTEND_VECTOR_INREG:
340 40 : return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
341 0 : case ISD::SIGN_EXTEND_VECTOR_INREG:
342 0 : return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
343 0 : case ISD::ZERO_EXTEND_VECTOR_INREG:
344 0 : return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
345 : }
346 :
347 0 : llvm_unreachable("Illegal extend_vector_inreg opcode");
348 : }
349 :
350 151 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
351 : // If the operand is wider than the vector element type then it is implicitly
352 : // truncated. Make that explicit here.
353 302 : EVT EltVT = N->getValueType(0).getVectorElementType();
354 151 : SDValue InOp = N->getOperand(0);
355 159 : if (InOp.getValueType() != EltVT)
356 0 : return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
357 151 : return InOp;
358 : }
359 :
360 353 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
361 353 : SDValue Cond = N->getOperand(0);
362 353 : EVT OpVT = Cond.getValueType();
363 : SDLoc DL(N);
364 : // The vselect result and true/value operands needs scalarizing, but it's
365 : // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
366 : // See the similar logic in ScalarizeVecRes_SETCC
367 353 : if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
368 337 : Cond = GetScalarizedVector(Cond);
369 : } else {
370 16 : EVT VT = OpVT.getVectorElementType();
371 16 : Cond = DAG.getNode(
372 : ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
373 16 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
374 : }
375 :
376 706 : SDValue LHS = GetScalarizedVector(N->getOperand(1));
377 : TargetLowering::BooleanContent ScalarBool =
378 353 : TLI.getBooleanContents(false, false);
379 : TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
380 :
381 : // If integer and float booleans have different contents then we can't
382 : // reliably optimize in all cases. There is a full explanation for this in
383 : // DAGCombiner::visitSELECT() where the same issue affects folding
384 : // (select C, 0, 1) to (xor C, 1).
385 353 : if (TLI.getBooleanContents(false, false) !=
386 : TLI.getBooleanContents(false, true)) {
387 : // At least try the common case where the boolean is generated by a
388 : // comparison.
389 0 : if (Cond->getOpcode() == ISD::SETCC) {
390 0 : EVT OpVT = Cond->getOperand(0).getValueType();
391 0 : ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
392 0 : VecBool = TLI.getBooleanContents(OpVT);
393 : } else
394 : ScalarBool = TargetLowering::UndefinedBooleanContent;
395 : }
396 :
397 353 : EVT CondVT = Cond.getValueType();
398 353 : if (ScalarBool != VecBool) {
399 49 : switch (ScalarBool) {
400 : case TargetLowering::UndefinedBooleanContent:
401 : break;
402 49 : case TargetLowering::ZeroOrOneBooleanContent:
403 : assert(VecBool == TargetLowering::UndefinedBooleanContent ||
404 : VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
405 : // Vector read from all ones, scalar expects a single 1 so mask.
406 98 : Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
407 98 : Cond, DAG.getConstant(1, SDLoc(N), CondVT));
408 49 : break;
409 0 : case TargetLowering::ZeroOrNegativeOneBooleanContent:
410 : assert(VecBool == TargetLowering::UndefinedBooleanContent ||
411 : VecBool == TargetLowering::ZeroOrOneBooleanContent);
412 : // Vector reads from a one, scalar from all ones so sign extend.
413 0 : Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
414 0 : Cond, DAG.getValueType(MVT::i1));
415 0 : break;
416 : }
417 : }
418 :
419 : // Truncate the condition if needed
420 353 : auto BoolVT = getSetCCResultType(CondVT);
421 353 : if (BoolVT.bitsLT(CondVT))
422 4 : Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
423 :
424 353 : return DAG.getSelect(SDLoc(N),
425 : LHS.getValueType(), Cond, LHS,
426 706 : GetScalarizedVector(N->getOperand(2)));
427 : }
428 :
429 209 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
430 418 : SDValue LHS = GetScalarizedVector(N->getOperand(1));
431 418 : return DAG.getSelect(SDLoc(N),
432 209 : LHS.getValueType(), N->getOperand(0), LHS,
433 418 : GetScalarizedVector(N->getOperand(2)));
434 : }
435 :
436 0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
437 0 : SDValue LHS = GetScalarizedVector(N->getOperand(2));
438 0 : return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
439 0 : N->getOperand(0), N->getOperand(1),
440 : LHS, GetScalarizedVector(N->getOperand(3)),
441 0 : N->getOperand(4));
442 : }
443 :
444 133 : SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
445 266 : return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
446 : }
447 :
448 0 : SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
449 : // Figure out if the scalar is the LHS or RHS and return it.
450 0 : SDValue Arg = N->getOperand(2).getOperand(0);
451 0 : if (Arg.isUndef())
452 0 : return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
453 0 : unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
454 0 : return GetScalarizedVector(N->getOperand(Op));
455 : }
456 :
457 1138 : SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
458 : assert(N->getValueType(0).isVector() &&
459 : N->getOperand(0).getValueType().isVector() &&
460 : "Operand types must be vectors");
461 1138 : SDValue LHS = N->getOperand(0);
462 1138 : SDValue RHS = N->getOperand(1);
463 1138 : EVT OpVT = LHS.getValueType();
464 2276 : EVT NVT = N->getValueType(0).getVectorElementType();
465 : SDLoc DL(N);
466 :
467 : // The result needs scalarizing, but it's not a given that the source does.
468 1138 : if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
469 1132 : LHS = GetScalarizedVector(LHS);
470 1132 : RHS = GetScalarizedVector(RHS);
471 : } else {
472 6 : EVT VT = OpVT.getVectorElementType();
473 6 : LHS = DAG.getNode(
474 : ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
475 6 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
476 6 : RHS = DAG.getNode(
477 : ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
478 6 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
479 : }
480 :
481 : // Turn it into a scalar SETCC.
482 1138 : SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
483 2276 : N->getOperand(2));
484 : // Vectors may have a different boolean contents to scalars. Promote the
485 : // value appropriately.
486 : ISD::NodeType ExtendCode =
487 1138 : TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
488 2276 : return DAG.getNode(ExtendCode, DL, NVT, Res);
489 : }
490 :
491 :
492 : //===----------------------------------------------------------------------===//
493 : // Operand Vector Scalarization <1 x ty> -> ty.
494 : //===----------------------------------------------------------------------===//
495 :
496 18912 : bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
497 : LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
498 : dbgs() << "\n");
499 18912 : SDValue Res = SDValue();
500 :
501 : if (!Res.getNode()) {
502 37824 : switch (N->getOpcode()) {
503 0 : default:
504 : #ifndef NDEBUG
505 : dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
506 : N->dump(&DAG);
507 : dbgs() << "\n";
508 : #endif
509 0 : report_fatal_error("Do not know how to scalarize this operator's "
510 : "operand!\n");
511 1817 : case ISD::BITCAST:
512 1817 : Res = ScalarizeVecOp_BITCAST(N);
513 1817 : break;
514 22 : case ISD::ANY_EXTEND:
515 : case ISD::ZERO_EXTEND:
516 : case ISD::SIGN_EXTEND:
517 : case ISD::TRUNCATE:
518 : case ISD::FP_TO_SINT:
519 : case ISD::FP_TO_UINT:
520 : case ISD::SINT_TO_FP:
521 : case ISD::UINT_TO_FP:
522 22 : Res = ScalarizeVecOp_UnaryOp(N);
523 22 : break;
524 6150 : case ISD::CONCAT_VECTORS:
525 6150 : Res = ScalarizeVecOp_CONCAT_VECTORS(N);
526 6150 : break;
527 5706 : case ISD::EXTRACT_VECTOR_ELT:
528 5706 : Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
529 5706 : break;
530 1 : case ISD::VSELECT:
531 1 : Res = ScalarizeVecOp_VSELECT(N);
532 1 : break;
533 12 : case ISD::SETCC:
534 12 : Res = ScalarizeVecOp_VSETCC(N);
535 12 : break;
536 : case ISD::STORE:
537 5201 : Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
538 5201 : break;
539 3 : case ISD::FP_ROUND:
540 3 : Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
541 3 : break;
542 : }
543 : }
544 :
545 : // If the result is null, the sub-method took care of registering results etc.
546 18912 : if (!Res.getNode()) return false;
547 :
548 : // If the result is N, the sub-method updated N in place. Tell the legalizer
549 : // core about this.
550 18912 : if (Res.getNode() == N)
551 : return true;
552 :
553 : assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
554 : "Invalid operand expansion");
555 :
556 18912 : ReplaceValueWith(SDValue(N, 0), Res);
557 18912 : return false;
558 : }
559 :
560 : /// If the value to convert is a vector that needs to be scalarized, it must be
561 : /// <1 x ty>. Convert the element instead.
562 1817 : SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
563 3634 : SDValue Elt = GetScalarizedVector(N->getOperand(0));
564 1817 : return DAG.getNode(ISD::BITCAST, SDLoc(N),
565 1817 : N->getValueType(0), Elt);
566 : }
567 :
568 : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
569 : /// Do the operation on the element instead.
570 22 : SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
571 : assert(N->getValueType(0).getVectorNumElements() == 1 &&
572 : "Unexpected vector type!");
573 44 : SDValue Elt = GetScalarizedVector(N->getOperand(0));
574 22 : SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
575 88 : N->getValueType(0).getScalarType(), Elt);
576 : // Revectorize the result so the types line up with what the uses of this
577 : // expression expect.
578 44 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
579 : }
580 :
581 : /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
582 6150 : SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
583 6150 : SmallVector<SDValue, 8> Ops(N->getNumOperands());
584 18462 : for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
585 24624 : Ops[i] = GetScalarizedVector(N->getOperand(i));
586 12300 : return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
587 : }
588 :
589 : /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
590 : /// so just return the element, ignoring the index.
591 5706 : SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
592 5706 : EVT VT = N->getValueType(0);
593 11412 : SDValue Res = GetScalarizedVector(N->getOperand(0));
594 0 : if (Res.getValueType() != VT)
595 1544 : Res = VT.isFloatingPoint()
596 1544 : ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
597 1541 : : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
598 5706 : return Res;
599 : }
600 :
601 : /// If the input condition is a vector that needs to be scalarized, it must be
602 : /// <1 x i1>, so just convert to a normal ISD::SELECT
603 : /// (still with vector output type since that was acceptable if we got here).
604 1 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
605 2 : SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
606 1 : EVT VT = N->getValueType(0);
607 :
608 1 : return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
609 1 : N->getOperand(2));
610 : }
611 :
612 : /// If the operand is a vector that needs to be scalarized then the
613 : /// result must be v1i1, so just convert to a scalar SETCC and wrap
614 : /// with a scalar_to_vector since the res type is legal if we got here
615 12 : SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
616 : assert(N->getValueType(0).isVector() &&
617 : N->getOperand(0).getValueType().isVector() &&
618 : "Operand types must be vectors");
619 : assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
620 :
621 12 : EVT VT = N->getValueType(0);
622 24 : SDValue LHS = GetScalarizedVector(N->getOperand(0));
623 24 : SDValue RHS = GetScalarizedVector(N->getOperand(1));
624 :
625 12 : EVT OpVT = N->getOperand(0).getValueType();
626 12 : EVT NVT = VT.getVectorElementType();
627 : SDLoc DL(N);
628 : // Turn it into a scalar SETCC.
629 12 : SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
630 24 : N->getOperand(2));
631 :
632 : // Vectors may have a different boolean contents to scalars. Promote the
633 : // value appropriately.
634 : ISD::NodeType ExtendCode =
635 12 : TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
636 :
637 24 : Res = DAG.getNode(ExtendCode, DL, NVT, Res);
638 :
639 24 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
640 : }
641 :
642 : /// If the value to store is a vector that needs to be scalarized, it must be
643 : /// <1 x ty>. Just store the element.
644 5201 : SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
645 : assert(N->isUnindexed() && "Indexed store of one-element vector?");
646 : assert(OpNo == 1 && "Do not know how to scalarize this operand!");
647 : SDLoc dl(N);
648 :
649 5201 : if (N->isTruncatingStore())
650 0 : return DAG.getTruncStore(
651 : N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
652 0 : N->getBasePtr(), N->getPointerInfo(),
653 0 : N->getMemoryVT().getVectorElementType(), N->getAlignment(),
654 0 : N->getMemOperand()->getFlags(), N->getAAInfo());
655 :
656 5201 : return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
657 : N->getBasePtr(), N->getPointerInfo(),
658 5201 : N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
659 15603 : N->getAAInfo());
660 : }
661 :
662 : /// If the value to round is a vector that needs to be scalarized, it must be
663 : /// <1 x ty>. Convert the element instead.
664 3 : SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
665 6 : SDValue Elt = GetScalarizedVector(N->getOperand(0));
666 3 : SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
667 3 : N->getValueType(0).getVectorElementType(), Elt,
668 9 : N->getOperand(1));
669 6 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
670 : }
671 :
672 : //===----------------------------------------------------------------------===//
673 : // Result Vector Splitting
674 : //===----------------------------------------------------------------------===//
675 :
676 : /// This method is called when the specified result of the specified node is
677 : /// found to need vector splitting. At this point, the node may also have
678 : /// invalid operands or may have other results that need legalization, we just
679 : /// know that (at least) one result needs vector splitting.
680 92822 : void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
681 : LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
682 92822 : SDValue Lo, Hi;
683 :
684 : // See if the target wants to custom expand this node.
685 185644 : if (CustomLowerNode(N, N->getValueType(ResNo), true))
686 357 : return;
687 :
688 184930 : switch (N->getOpcode()) {
689 0 : default:
690 : #ifndef NDEBUG
691 : dbgs() << "SplitVectorResult #" << ResNo << ": ";
692 : N->dump(&DAG);
693 : dbgs() << "\n";
694 : #endif
695 0 : report_fatal_error("Do not know how to split the result of this "
696 : "operator!\n");
697 :
698 0 : case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
699 1275 : case ISD::VSELECT:
700 1275 : case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
701 0 : case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
702 1814 : case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
703 21409 : case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
704 10999 : case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
705 15410 : case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
706 1702 : case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
707 29 : case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
708 0 : case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
709 8 : case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
710 11 : case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
711 764 : case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
712 210 : case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
713 83 : case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
714 : case ISD::LOAD:
715 4221 : SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
716 4221 : break;
717 : case ISD::MLOAD:
718 20 : SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
719 20 : break;
720 : case ISD::MGATHER:
721 16 : SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
722 16 : break;
723 2614 : case ISD::SETCC:
724 2614 : SplitVecRes_SETCC(N, Lo, Hi);
725 2614 : break;
726 : case ISD::VECTOR_SHUFFLE:
727 5126 : SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
728 5126 : break;
729 :
730 413 : case ISD::ANY_EXTEND_VECTOR_INREG:
731 : case ISD::SIGN_EXTEND_VECTOR_INREG:
732 : case ISD::ZERO_EXTEND_VECTOR_INREG:
733 413 : SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
734 413 : break;
735 :
736 3333 : case ISD::BITREVERSE:
737 : case ISD::BSWAP:
738 : case ISD::CTLZ:
739 : case ISD::CTTZ:
740 : case ISD::CTLZ_ZERO_UNDEF:
741 : case ISD::CTTZ_ZERO_UNDEF:
742 : case ISD::CTPOP:
743 : case ISD::FABS:
744 : case ISD::FCEIL:
745 : case ISD::FCOS:
746 : case ISD::FEXP:
747 : case ISD::FEXP2:
748 : case ISD::FFLOOR:
749 : case ISD::FLOG:
750 : case ISD::FLOG10:
751 : case ISD::FLOG2:
752 : case ISD::FNEARBYINT:
753 : case ISD::FNEG:
754 : case ISD::FP_EXTEND:
755 : case ISD::FP_ROUND:
756 : case ISD::FP_TO_SINT:
757 : case ISD::FP_TO_UINT:
758 : case ISD::FRINT:
759 : case ISD::FROUND:
760 : case ISD::FSIN:
761 : case ISD::FSQRT:
762 : case ISD::FTRUNC:
763 : case ISD::SINT_TO_FP:
764 : case ISD::TRUNCATE:
765 : case ISD::UINT_TO_FP:
766 : case ISD::FCANONICALIZE:
767 3333 : SplitVecRes_UnaryOp(N, Lo, Hi);
768 3333 : break;
769 :
770 7785 : case ISD::ANY_EXTEND:
771 : case ISD::SIGN_EXTEND:
772 : case ISD::ZERO_EXTEND:
773 7785 : SplitVecRes_ExtendOp(N, Lo, Hi);
774 7785 : break;
775 :
776 15015 : case ISD::ADD:
777 : case ISD::SUB:
778 : case ISD::MUL:
779 : case ISD::MULHS:
780 : case ISD::MULHU:
781 : case ISD::FADD:
782 : case ISD::FSUB:
783 : case ISD::FMUL:
784 : case ISD::FMINNUM:
785 : case ISD::FMAXNUM:
786 : case ISD::FMINNAN:
787 : case ISD::FMAXNAN:
788 : case ISD::SDIV:
789 : case ISD::UDIV:
790 : case ISD::FDIV:
791 : case ISD::FPOW:
792 : case ISD::AND:
793 : case ISD::OR:
794 : case ISD::XOR:
795 : case ISD::SHL:
796 : case ISD::SRA:
797 : case ISD::SRL:
798 : case ISD::UREM:
799 : case ISD::SREM:
800 : case ISD::FREM:
801 : case ISD::SMIN:
802 : case ISD::SMAX:
803 : case ISD::UMIN:
804 : case ISD::UMAX:
805 : case ISD::SADDSAT:
806 15015 : SplitVecRes_BinOp(N, Lo, Hi);
807 15015 : break;
808 189 : case ISD::FMA:
809 189 : SplitVecRes_TernaryOp(N, Lo, Hi);
810 189 : break;
811 19 : case ISD::STRICT_FADD:
812 : case ISD::STRICT_FSUB:
813 : case ISD::STRICT_FMUL:
814 : case ISD::STRICT_FDIV:
815 : case ISD::STRICT_FREM:
816 : case ISD::STRICT_FSQRT:
817 : case ISD::STRICT_FMA:
818 : case ISD::STRICT_FPOW:
819 : case ISD::STRICT_FPOWI:
820 : case ISD::STRICT_FSIN:
821 : case ISD::STRICT_FCOS:
822 : case ISD::STRICT_FEXP:
823 : case ISD::STRICT_FEXP2:
824 : case ISD::STRICT_FLOG:
825 : case ISD::STRICT_FLOG10:
826 : case ISD::STRICT_FLOG2:
827 : case ISD::STRICT_FRINT:
828 : case ISD::STRICT_FNEARBYINT:
829 19 : SplitVecRes_StrictFPOp(N, Lo, Hi);
830 19 : break;
831 : }
832 :
833 : // If Lo/Hi is null, the sub-method took care of registering results etc.
834 92465 : if (Lo.getNode())
835 92465 : SetSplitVector(SDValue(N, ResNo), Lo, Hi);
836 : }
837 :
838 15015 : void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
839 : SDValue &Hi) {
840 15015 : SDValue LHSLo, LHSHi;
841 30030 : GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
842 15015 : SDValue RHSLo, RHSHi;
843 30030 : GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
844 : SDLoc dl(N);
845 :
846 15015 : const SDNodeFlags Flags = N->getFlags();
847 15015 : unsigned Opcode = N->getOpcode();
848 30030 : Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
849 30030 : Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
850 15015 : }
851 :
852 189 : void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
853 : SDValue &Hi) {
854 189 : SDValue Op0Lo, Op0Hi;
855 378 : GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
856 189 : SDValue Op1Lo, Op1Hi;
857 378 : GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
858 189 : SDValue Op2Lo, Op2Hi;
859 378 : GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
860 : SDLoc dl(N);
861 :
862 189 : Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
863 567 : Op0Lo, Op1Lo, Op2Lo);
864 189 : Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
865 567 : Op0Hi, Op1Hi, Op2Hi);
866 189 : }
867 :
868 21409 : void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
869 : SDValue &Hi) {
870 : // We know the result is a vector. The input may be either a vector or a
871 : // scalar value.
872 21409 : EVT LoVT, HiVT;
873 42818 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
874 : SDLoc dl(N);
875 :
876 21409 : SDValue InOp = N->getOperand(0);
877 : EVT InVT = InOp.getValueType();
878 :
879 : // Handle some special cases efficiently.
880 21409 : switch (getTypeAction(InVT)) {
881 : case TargetLowering::TypeLegal:
882 : case TargetLowering::TypePromoteInteger:
883 : case TargetLowering::TypePromoteFloat:
884 : case TargetLowering::TypeSoftenFloat:
885 : case TargetLowering::TypeScalarizeVector:
886 : case TargetLowering::TypeWidenVector:
887 : break;
888 2103 : case TargetLowering::TypeExpandInteger:
889 : case TargetLowering::TypeExpandFloat:
890 : // A scalar to vector conversion, where the scalar needs expansion.
891 : // If the vector is being split in two then we can just convert the
892 : // expanded pieces.
893 2103 : if (LoVT == HiVT) {
894 2103 : GetExpandedOp(InOp, Lo, Hi);
895 2103 : if (DAG.getDataLayout().isBigEndian())
896 : std::swap(Lo, Hi);
897 4206 : Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
898 4206 : Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
899 2103 : return;
900 : }
901 : break;
902 2629 : case TargetLowering::TypeSplitVector:
903 : // If the input is a vector that needs to be split, convert each split
904 : // piece of the input now.
905 2629 : GetSplitVector(InOp, Lo, Hi);
906 5258 : Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
907 5258 : Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
908 2629 : return;
909 : }
910 :
911 : // In the general case, convert the input to an integer and split it by hand.
912 16677 : EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
913 16677 : EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
914 16677 : if (DAG.getDataLayout().isBigEndian())
915 : std::swap(LoIntVT, HiIntVT);
916 :
917 16677 : SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
918 :
919 16677 : if (DAG.getDataLayout().isBigEndian())
920 : std::swap(Lo, Hi);
921 33354 : Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
922 33354 : Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
923 : }
924 :
925 10999 : void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
926 : SDValue &Hi) {
927 10999 : EVT LoVT, HiVT;
928 : SDLoc dl(N);
929 21998 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
930 : unsigned LoNumElts = LoVT.getVectorNumElements();
931 10999 : SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
932 21998 : Lo = DAG.getBuildVector(LoVT, dl, LoOps);
933 :
934 21998 : SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
935 21998 : Hi = DAG.getBuildVector(HiVT, dl, HiOps);
936 10999 : }
937 :
938 15410 : void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
939 : SDValue &Hi) {
940 : assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
941 : SDLoc dl(N);
942 15410 : unsigned NumSubvectors = N->getNumOperands() / 2;
943 15410 : if (NumSubvectors == 1) {
944 11046 : Lo = N->getOperand(0);
945 11046 : Hi = N->getOperand(1);
946 : return;
947 : }
948 :
949 : EVT LoVT, HiVT;
950 8728 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
951 :
952 4364 : SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
953 8728 : Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
954 :
955 8728 : SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
956 8728 : Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
957 : }
958 :
959 1702 : void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
960 : SDValue &Hi) {
961 1702 : SDValue Vec = N->getOperand(0);
962 1702 : SDValue Idx = N->getOperand(1);
963 : SDLoc dl(N);
964 :
965 1702 : EVT LoVT, HiVT;
966 3404 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
967 :
968 3404 : Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
969 1702 : uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
970 1702 : Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
971 1702 : DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
972 3404 : TLI.getVectorIdxTy(DAG.getDataLayout())));
973 1702 : }
974 :
975 29 : void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
976 : SDValue &Hi) {
977 29 : SDValue Vec = N->getOperand(0);
978 29 : SDValue SubVec = N->getOperand(1);
979 29 : SDValue Idx = N->getOperand(2);
980 : SDLoc dl(N);
981 29 : GetSplitVector(Vec, Lo, Hi);
982 :
983 29 : EVT VecVT = Vec.getValueType();
984 : unsigned VecElems = VecVT.getVectorNumElements();
985 58 : unsigned SubElems = SubVec.getValueType().getVectorNumElements();
986 :
987 : // If we know the index is 0, and we know the subvector doesn't cross the
988 : // boundary between the halves, we can avoid spilling the vector, and insert
989 : // into the lower half of the split vector directly.
990 : // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
991 : // the index is constant and there is no boundary crossing. But those cases
992 : // don't seem to get hit in practice.
993 : if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
994 29 : unsigned IdxVal = ConstIdx->getZExtValue();
995 29 : if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
996 : EVT LoVT, HiVT;
997 58 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
998 58 : Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
999 : return;
1000 : }
1001 : }
1002 :
1003 : // Spill the vector to the stack.
1004 0 : SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1005 : SDValue Store =
1006 0 : DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1007 :
1008 : // Store the new subvector into the specified index.
1009 0 : SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1010 0 : Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1011 0 : unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1012 0 : Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1013 :
1014 : // Load the Lo part from the stack slot.
1015 0 : Lo =
1016 0 : DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1017 :
1018 : // Increment the pointer to the other part.
1019 0 : unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1020 0 : StackPtr =
1021 0 : DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1022 0 : DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1023 :
1024 : // Load the Hi part from the stack slot.
1025 0 : Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1026 0 : MinAlign(Alignment, IncrementSize));
1027 : }
1028 :
1029 8 : void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1030 : SDValue &Hi) {
1031 : SDLoc dl(N);
1032 16 : GetSplitVector(N->getOperand(0), Lo, Hi);
1033 32 : Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1034 32 : Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1035 8 : }
1036 :
1037 11 : void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1038 : SDValue &Hi) {
1039 11 : SDValue LHSLo, LHSHi;
1040 22 : GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1041 : SDLoc DL(N);
1042 :
1043 11 : SDValue RHSLo, RHSHi;
1044 11 : SDValue RHS = N->getOperand(1);
1045 11 : EVT RHSVT = RHS.getValueType();
1046 11 : if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1047 9 : GetSplitVector(RHS, RHSLo, RHSHi);
1048 : else
1049 2 : std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1050 :
1051 :
1052 33 : Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1053 33 : Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1054 11 : }
1055 :
1056 83 : void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1057 : SDValue &Hi) {
1058 83 : SDValue LHSLo, LHSHi;
1059 166 : GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1060 : SDLoc dl(N);
1061 :
1062 : EVT LoVT, HiVT;
1063 : std::tie(LoVT, HiVT) =
1064 166 : DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1065 :
1066 83 : Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1067 83 : DAG.getValueType(LoVT));
1068 83 : Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1069 83 : DAG.getValueType(HiVT));
1070 83 : }
1071 :
1072 419 : void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1073 : SDValue &Hi) {
1074 419 : unsigned Opcode = N->getOpcode();
1075 419 : SDValue N0 = N->getOperand(0);
1076 :
1077 : SDLoc dl(N);
1078 419 : SDValue InLo, InHi;
1079 :
1080 419 : if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1081 417 : GetSplitVector(N0, InLo, InHi);
1082 : else
1083 2 : std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1084 :
1085 838 : EVT InLoVT = InLo.getValueType();
1086 : unsigned InNumElements = InLoVT.getVectorNumElements();
1087 :
1088 419 : EVT OutLoVT, OutHiVT;
1089 838 : std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1090 : unsigned OutNumElements = OutLoVT.getVectorNumElements();
1091 : assert((2 * OutNumElements) <= InNumElements &&
1092 : "Illegal extend vector in reg split");
1093 :
1094 : // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1095 : // input vector (i.e. we only use InLo):
1096 : // OutLo will extend the first OutNumElements from InLo.
1097 : // OutHi will extend the next OutNumElements from InLo.
1098 :
1099 : // Shuffle the elements from InLo for OutHi into the bottom elements to
1100 : // create a 'fake' InHi.
1101 419 : SmallVector<int, 8> SplitHi(InNumElements, -1);
1102 2661 : for (unsigned i = 0; i != OutNumElements; ++i)
1103 4484 : SplitHi[i] = i + OutNumElements;
1104 838 : InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1105 :
1106 838 : Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1107 838 : Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1108 419 : }
1109 :
1110 19 : void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1111 : SDValue &Hi) {
1112 19 : unsigned NumOps = N->getNumOperands();
1113 19 : SDValue Chain = N->getOperand(0);
1114 19 : EVT LoVT, HiVT;
1115 : SDLoc dl(N);
1116 38 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1117 :
1118 : SmallVector<SDValue, 4> OpsLo;
1119 : SmallVector<SDValue, 4> OpsHi;
1120 :
1121 : // The Chain is the first operand.
1122 19 : OpsLo.push_back(Chain);
1123 19 : OpsHi.push_back(Chain);
1124 :
1125 : // Now process the remaining operands.
1126 49 : for (unsigned i = 1; i < NumOps; ++i) {
1127 30 : SDValue Op = N->getOperand(i);
1128 30 : SDValue OpLo = Op;
1129 30 : SDValue OpHi = Op;
1130 :
1131 30 : EVT InVT = Op.getValueType();
1132 30 : if (InVT.isVector()) {
1133 : // If the input also splits, handle it directly for a
1134 : // compile time speedup. Otherwise split it by hand.
1135 29 : if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1136 29 : GetSplitVector(Op, OpLo, OpHi);
1137 : else
1138 0 : std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1139 : }
1140 :
1141 30 : OpsLo.push_back(OpLo);
1142 30 : OpsHi.push_back(OpHi);
1143 : }
1144 :
1145 19 : EVT LoValueVTs[] = {LoVT, MVT::Other};
1146 19 : EVT HiValueVTs[] = {HiVT, MVT::Other};
1147 57 : Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1148 57 : Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1149 :
1150 : // Build a factor node to remember that this Op is independent of the
1151 : // other one.
1152 19 : Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1153 38 : Lo.getValue(1), Hi.getValue(1));
1154 :
1155 : // Legalize the chain result - switch anything that used the old chain to
1156 : // use the new one.
1157 19 : ReplaceValueWith(SDValue(N, 1), Chain);
1158 19 : }
1159 :
1160 764 : void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1161 : SDValue &Hi) {
1162 764 : SDValue Vec = N->getOperand(0);
1163 764 : SDValue Elt = N->getOperand(1);
1164 764 : SDValue Idx = N->getOperand(2);
1165 : SDLoc dl(N);
1166 764 : GetSplitVector(Vec, Lo, Hi);
1167 :
1168 : if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1169 715 : unsigned IdxVal = CIdx->getZExtValue();
1170 1430 : unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1171 715 : if (IdxVal < LoNumElts)
1172 405 : Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1173 405 : Lo.getValueType(), Lo, Elt, Idx);
1174 : else
1175 310 : Hi =
1176 310 : DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1177 310 : DAG.getConstant(IdxVal - LoNumElts, dl,
1178 310 : TLI.getVectorIdxTy(DAG.getDataLayout())));
1179 715 : return;
1180 : }
1181 :
1182 : // See if the target wants to custom expand this node.
1183 98 : if (CustomLowerNode(N, N->getValueType(0), true))
1184 : return;
1185 :
1186 : // Make the vector elements byte-addressable if they aren't already.
1187 49 : EVT VecVT = Vec.getValueType();
1188 49 : EVT EltVT = VecVT.getVectorElementType();
1189 49 : if (VecVT.getScalarSizeInBits() < 8) {
1190 8 : EltVT = MVT::i8;
1191 8 : VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1192 8 : VecVT.getVectorNumElements());
1193 16 : Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1194 : // Extend the element type to match if needed.
1195 8 : if (EltVT.bitsGT(Elt.getValueType()))
1196 16 : Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1197 : }
1198 :
1199 : // Spill the vector to the stack.
1200 49 : SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1201 49 : auto &MF = DAG.getMachineFunction();
1202 49 : auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1203 49 : auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1204 98 : SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1205 :
1206 : // Store the new element. This may be larger than the vector element type,
1207 : // so use a truncating store.
1208 49 : SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1209 49 : Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1210 49 : unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1211 49 : Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1212 49 : MachinePointerInfo::getUnknownStack(MF), EltVT);
1213 :
1214 49 : EVT LoVT, HiVT;
1215 49 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1216 :
1217 : // Load the Lo part from the stack slot.
1218 98 : Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1219 :
1220 : // Increment the pointer to the other part.
1221 49 : unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1222 49 : StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1223 : DAG.getConstant(IncrementSize, dl,
1224 49 : StackPtr.getValueType()));
1225 :
1226 : // Load the Hi part from the stack slot.
1227 49 : Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1228 : PtrInfo.getWithOffset(IncrementSize),
1229 49 : MinAlign(Alignment, IncrementSize));
1230 :
1231 : // If we adjusted the original type, we need to truncate the results.
1232 98 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1233 106 : if (LoVT != Lo.getValueType())
1234 16 : Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1235 106 : if (HiVT != Hi.getValueType())
1236 16 : Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1237 : }
1238 :
1239 210 : void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1240 : SDValue &Hi) {
1241 210 : EVT LoVT, HiVT;
1242 : SDLoc dl(N);
1243 420 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1244 630 : Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1245 210 : Hi = DAG.getUNDEF(HiVT);
1246 210 : }
1247 :
1248 4221 : void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1249 : SDValue &Hi) {
1250 : assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1251 4221 : EVT LoVT, HiVT;
1252 : SDLoc dl(LD);
1253 8442 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1254 :
1255 : ISD::LoadExtType ExtType = LD->getExtensionType();
1256 4221 : SDValue Ch = LD->getChain();
1257 4221 : SDValue Ptr = LD->getBasePtr();
1258 8442 : SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1259 4221 : EVT MemoryVT = LD->getMemoryVT();
1260 4221 : unsigned Alignment = LD->getOriginalAlignment();
1261 4221 : MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1262 : AAMDNodes AAInfo = LD->getAAInfo();
1263 :
1264 4221 : EVT LoMemVT, HiMemVT;
1265 4221 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1266 :
1267 8442 : Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1268 8442 : LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1269 :
1270 4221 : unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1271 4221 : Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1272 8442 : Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1273 4221 : LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1274 4221 : Alignment, MMOFlags, AAInfo);
1275 :
1276 : // Build a factor node to remember that this load is independent of the
1277 : // other one.
1278 4221 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1279 8442 : Hi.getValue(1));
1280 :
1281 : // Legalize the chain result - switch anything that used the old chain to
1282 : // use the new one.
1283 4221 : ReplaceValueWith(SDValue(LD, 1), Ch);
1284 4221 : }
1285 :
1286 20 : void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1287 : SDValue &Lo, SDValue &Hi) {
1288 20 : EVT LoVT, HiVT;
1289 : SDLoc dl(MLD);
1290 40 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1291 :
1292 20 : SDValue Ch = MLD->getChain();
1293 20 : SDValue Ptr = MLD->getBasePtr();
1294 20 : SDValue Mask = MLD->getMask();
1295 20 : SDValue PassThru = MLD->getPassThru();
1296 20 : unsigned Alignment = MLD->getOriginalAlignment();
1297 : ISD::LoadExtType ExtType = MLD->getExtensionType();
1298 :
1299 : // if Alignment is equal to the vector size,
1300 : // take the half of it for the second part
1301 : unsigned SecondHalfAlignment =
1302 40 : (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
1303 : Alignment/2 : Alignment;
1304 :
1305 : // Split Mask operand
1306 20 : SDValue MaskLo, MaskHi;
1307 40 : if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1308 1 : GetSplitVector(Mask, MaskLo, MaskHi);
1309 : else
1310 19 : std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1311 :
1312 20 : EVT MemoryVT = MLD->getMemoryVT();
1313 20 : EVT LoMemVT, HiMemVT;
1314 20 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1315 :
1316 20 : SDValue PassThruLo, PassThruHi;
1317 40 : if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1318 20 : GetSplitVector(PassThru, PassThruLo, PassThruHi);
1319 : else
1320 0 : std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1321 :
1322 20 : MachineMemOperand *MMO = DAG.getMachineFunction().
1323 60 : getMachineMemOperand(MLD->getPointerInfo(),
1324 : MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1325 20 : Alignment, MLD->getAAInfo(), MLD->getRanges());
1326 :
1327 20 : Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1328 20 : ExtType, MLD->isExpandingLoad());
1329 :
1330 20 : Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1331 20 : MLD->isExpandingLoad());
1332 : unsigned HiOffset = LoMemVT.getStoreSize();
1333 :
1334 60 : MMO = DAG.getMachineFunction().getMachineMemOperand(
1335 : MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
1336 20 : HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
1337 : MLD->getRanges());
1338 :
1339 20 : Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1340 20 : ExtType, MLD->isExpandingLoad());
1341 :
1342 : // Build a factor node to remember that this load is independent of the
1343 : // other one.
1344 20 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1345 40 : Hi.getValue(1));
1346 :
1347 : // Legalize the chain result - switch anything that used the old chain to
1348 : // use the new one.
1349 20 : ReplaceValueWith(SDValue(MLD, 1), Ch);
1350 :
1351 20 : }
1352 :
1353 16 : void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1354 : SDValue &Lo, SDValue &Hi) {
1355 16 : EVT LoVT, HiVT;
1356 : SDLoc dl(MGT);
1357 32 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1358 :
1359 16 : SDValue Ch = MGT->getChain();
1360 16 : SDValue Ptr = MGT->getBasePtr();
1361 16 : SDValue Mask = MGT->getMask();
1362 16 : SDValue PassThru = MGT->getPassThru();
1363 16 : SDValue Index = MGT->getIndex();
1364 16 : SDValue Scale = MGT->getScale();
1365 16 : unsigned Alignment = MGT->getOriginalAlignment();
1366 :
1367 : // Split Mask operand
1368 16 : SDValue MaskLo, MaskHi;
1369 32 : if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1370 0 : GetSplitVector(Mask, MaskLo, MaskHi);
1371 : else
1372 16 : std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1373 :
1374 16 : EVT MemoryVT = MGT->getMemoryVT();
1375 16 : EVT LoMemVT, HiMemVT;
1376 : // Split MemoryVT
1377 16 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1378 :
1379 16 : SDValue PassThruLo, PassThruHi;
1380 32 : if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1381 16 : GetSplitVector(PassThru, PassThruLo, PassThruHi);
1382 : else
1383 0 : std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1384 :
1385 16 : SDValue IndexHi, IndexLo;
1386 32 : if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1387 12 : GetSplitVector(Index, IndexLo, IndexHi);
1388 : else
1389 4 : std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1390 :
1391 16 : MachineMemOperand *MMO = DAG.getMachineFunction().
1392 48 : getMachineMemOperand(MGT->getPointerInfo(),
1393 : MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1394 16 : Alignment, MGT->getAAInfo(), MGT->getRanges());
1395 :
1396 16 : SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1397 16 : Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1398 16 : MMO);
1399 :
1400 16 : SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1401 16 : Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1402 16 : MMO);
1403 :
1404 : // Build a factor node to remember that this load is independent of the
1405 : // other one.
1406 16 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1407 32 : Hi.getValue(1));
1408 :
1409 : // Legalize the chain result - switch anything that used the old chain to
1410 : // use the new one.
1411 16 : ReplaceValueWith(SDValue(MGT, 1), Ch);
1412 16 : }
1413 :
1414 :
1415 2614 : void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1416 : assert(N->getValueType(0).isVector() &&
1417 : N->getOperand(0).getValueType().isVector() &&
1418 : "Operand types must be vectors");
1419 :
1420 2614 : EVT LoVT, HiVT;
1421 : SDLoc DL(N);
1422 5228 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1423 :
1424 : // If the input also splits, handle it directly. Otherwise split it by hand.
1425 2614 : SDValue LL, LH, RL, RH;
1426 5228 : if (getTypeAction(N->getOperand(0).getValueType()) ==
1427 : TargetLowering::TypeSplitVector)
1428 4834 : GetSplitVector(N->getOperand(0), LL, LH);
1429 : else
1430 197 : std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1431 :
1432 5228 : if (getTypeAction(N->getOperand(1).getValueType()) ==
1433 : TargetLowering::TypeSplitVector)
1434 4834 : GetSplitVector(N->getOperand(1), RL, RH);
1435 : else
1436 197 : std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1437 :
1438 10456 : Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1439 10456 : Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1440 2614 : }
1441 :
1442 11050 : void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1443 : SDValue &Hi) {
1444 : // Get the dest types - they may not match the input types, e.g. int_to_fp.
1445 11050 : EVT LoVT, HiVT;
1446 : SDLoc dl(N);
1447 22100 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1448 :
1449 : // If the input also splits, handle it directly for a compile time speedup.
1450 : // Otherwise split it by hand.
1451 11050 : EVT InVT = N->getOperand(0).getValueType();
1452 11050 : if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1453 13250 : GetSplitVector(N->getOperand(0), Lo, Hi);
1454 : else
1455 4425 : std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
1456 :
1457 22100 : if (N->getOpcode() == ISD::FP_ROUND) {
1458 1809 : Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1459 2412 : Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1460 : } else {
1461 20894 : Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1462 31341 : Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1463 : }
1464 11050 : }
1465 :
1466 7785 : void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1467 : SDValue &Hi) {
1468 : SDLoc dl(N);
1469 7785 : EVT SrcVT = N->getOperand(0).getValueType();
1470 7785 : EVT DestVT = N->getValueType(0);
1471 : EVT LoVT, HiVT;
1472 7785 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1473 :
1474 : // We can do better than a generic split operation if the extend is doing
1475 : // more than just doubling the width of the elements and the following are
1476 : // true:
1477 : // - The number of vector elements is even,
1478 : // - the source type is legal,
1479 : // - the type of a split source is illegal,
1480 : // - the type of an extended (by doubling element size) source is legal, and
1481 : // - the type of that extended source when split is legal.
1482 : //
1483 : // This won't necessarily completely legalize the operation, but it will
1484 : // more effectively move in the right direction and prevent falling down
1485 : // to scalarization in many cases due to the input vector being split too
1486 : // far.
1487 : unsigned NumElements = SrcVT.getVectorNumElements();
1488 7785 : if ((NumElements & 1) == 0 &&
1489 7785 : SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1490 5069 : LLVMContext &Ctx = *DAG.getContext();
1491 5069 : EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1492 5069 : EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1493 :
1494 : EVT SplitLoVT, SplitHiVT;
1495 5069 : std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1496 5421 : if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1497 159 : TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1498 : LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1499 : N->dump(&DAG); dbgs() << "\n");
1500 : // Extend the source vector by one step.
1501 : SDValue NewSrc =
1502 272 : DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1503 : // Get the low and high halves of the new, extended one step, vector.
1504 68 : std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1505 : // Extend those vector halves the rest of the way.
1506 204 : Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1507 204 : Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1508 : return;
1509 : }
1510 : }
1511 : // Fall back to the generic unary operator splitting otherwise.
1512 7717 : SplitVecRes_UnaryOp(N, Lo, Hi);
1513 : }
1514 :
1515 5126 : void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1516 : SDValue &Lo, SDValue &Hi) {
1517 : // The low and high parts of the original input give four input vectors.
1518 5126 : SDValue Inputs[4];
1519 : SDLoc dl(N);
1520 10252 : GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1521 10252 : GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1522 10252 : EVT NewVT = Inputs[0].getValueType();
1523 : unsigned NewElts = NewVT.getVectorNumElements();
1524 :
1525 : // If Lo or Hi uses elements from at most two of the four input vectors, then
1526 : // express it as a vector shuffle of those two inputs. Otherwise extract the
1527 : // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1528 : SmallVector<int, 16> Ops;
1529 15378 : for (unsigned High = 0; High < 2; ++High) {
1530 10252 : SDValue &Output = High ? Hi : Lo;
1531 :
1532 : // Build a shuffle mask for the output, discovering on the fly which
1533 : // input vectors to use as shuffle operands (recorded in InputUsed).
1534 : // If building a suitable shuffle vector proves too hard, then bail
1535 : // out with useBuildVector set.
1536 10252 : unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1537 10252 : unsigned FirstMaskIdx = High * NewElts;
1538 : bool useBuildVector = false;
1539 152225 : for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1540 : // The mask element. This indexes into the input.
1541 142052 : int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1542 :
1543 : // The input vector this mask element indexes into.
1544 142052 : unsigned Input = (unsigned)Idx / NewElts;
1545 :
1546 142052 : if (Input >= array_lengthof(Inputs)) {
1547 : // The mask element does not index into any input vector.
1548 105832 : Ops.push_back(-1);
1549 105832 : continue;
1550 : }
1551 :
1552 : // Turn the index into an offset from the start of the input vector.
1553 36220 : Idx -= Input * NewElts;
1554 :
1555 : // Find or create a shuffle vector operand to hold this input.
1556 : unsigned OpNo;
1557 42109 : for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1558 42030 : if (InputUsed[OpNo] == Input) {
1559 : // This input vector is already an operand.
1560 : break;
1561 13182 : } else if (InputUsed[OpNo] == -1U) {
1562 : // Create a new operand for this input vector.
1563 7293 : InputUsed[OpNo] = Input;
1564 7293 : break;
1565 : }
1566 : }
1567 :
1568 36220 : if (OpNo >= array_lengthof(InputUsed)) {
1569 : // More than two input vectors used! Give up on trying to create a
1570 : // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1571 : useBuildVector = true;
1572 : break;
1573 : }
1574 :
1575 : // Add the mask index for the new shuffle vector.
1576 36141 : Ops.push_back(Idx + OpNo * NewElts);
1577 : }
1578 :
1579 10252 : if (useBuildVector) {
1580 79 : EVT EltVT = NewVT.getVectorElementType();
1581 : SmallVector<SDValue, 16> SVOps;
1582 :
1583 : // Extract the input elements by hand.
1584 691 : for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1585 : // The mask element. This indexes into the input.
1586 612 : int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1587 :
1588 : // The input vector this mask element indexes into.
1589 612 : unsigned Input = (unsigned)Idx / NewElts;
1590 :
1591 612 : if (Input >= array_lengthof(Inputs)) {
1592 : // The mask element is "undef" or indexes off the end of the input.
1593 123 : SVOps.push_back(DAG.getUNDEF(EltVT));
1594 123 : continue;
1595 : }
1596 :
1597 : // Turn the index into an offset from the start of the input vector.
1598 489 : Idx -= Input * NewElts;
1599 :
1600 : // Extract the vector element by hand.
1601 978 : SVOps.push_back(DAG.getNode(
1602 : ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1603 978 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1604 : }
1605 :
1606 : // Construct the Lo/Hi output using a BUILD_VECTOR.
1607 158 : Output = DAG.getBuildVector(NewVT, dl, SVOps);
1608 10173 : } else if (InputUsed[0] == -1U) {
1609 : // No input vectors were used! The result is undefined.
1610 4128 : Output = DAG.getUNDEF(NewVT);
1611 : } else {
1612 6045 : SDValue Op0 = Inputs[InputUsed[0]];
1613 : // If only one input was used, use an undefined vector for the other.
1614 6045 : SDValue Op1 = InputUsed[1] == -1U ?
1615 6045 : DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1616 : // At least one input vector was used. Create a new shuffle vector.
1617 12090 : Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1618 : }
1619 :
1620 : Ops.clear();
1621 : }
1622 5126 : }
1623 :
1624 :
1625 : //===----------------------------------------------------------------------===//
1626 : // Operand Vector Splitting
1627 : //===----------------------------------------------------------------------===//
1628 :
1629 : /// This method is called when the specified operand of the specified node is
1630 : /// found to need vector splitting. At this point, all of the result types of
1631 : /// the node are known to be legal, but other operands of the node may need
1632 : /// legalization as well as the specified one.
1633 73292 : bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1634 : LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1635 73292 : SDValue Res = SDValue();
1636 :
1637 : // See if the target wants to custom split this node.
1638 219876 : if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1639 : return false;
1640 :
1641 : if (!Res.getNode()) {
1642 143470 : switch (N->getOpcode()) {
1643 0 : default:
1644 : #ifndef NDEBUG
1645 : dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1646 : N->dump(&DAG);
1647 : dbgs() << "\n";
1648 : #endif
1649 0 : report_fatal_error("Do not know how to split this operator's "
1650 : "operand!\n");
1651 :
1652 142 : case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1653 5492 : case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1654 22998 : case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1655 22384 : case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1656 2 : case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1657 1899 : case ISD::TRUNCATE:
1658 1899 : Res = SplitVecOp_TruncateHelper(N);
1659 1899 : break;
1660 82 : case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1661 8 : case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1662 : case ISD::STORE:
1663 8932 : Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1664 8932 : break;
1665 : case ISD::MSTORE:
1666 8 : Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1667 8 : break;
1668 : case ISD::MSCATTER:
1669 22 : Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1670 22 : break;
1671 : case ISD::MGATHER:
1672 33 : Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1673 33 : break;
1674 136 : case ISD::VSELECT:
1675 136 : Res = SplitVecOp_VSELECT(N, OpNo);
1676 136 : break;
1677 51 : case ISD::FP_TO_SINT:
1678 : case ISD::FP_TO_UINT:
1679 153 : if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1680 39 : Res = SplitVecOp_TruncateHelper(N);
1681 : else
1682 12 : Res = SplitVecOp_UnaryOp(N);
1683 : break;
1684 249 : case ISD::SINT_TO_FP:
1685 : case ISD::UINT_TO_FP:
1686 747 : if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1687 143 : Res = SplitVecOp_TruncateHelper(N);
1688 : else
1689 106 : Res = SplitVecOp_UnaryOp(N);
1690 : break;
1691 9278 : case ISD::CTTZ:
1692 : case ISD::CTLZ:
1693 : case ISD::CTPOP:
1694 : case ISD::FP_EXTEND:
1695 : case ISD::SIGN_EXTEND:
1696 : case ISD::ZERO_EXTEND:
1697 : case ISD::ANY_EXTEND:
1698 : case ISD::FTRUNC:
1699 : case ISD::FCANONICALIZE:
1700 9278 : Res = SplitVecOp_UnaryOp(N);
1701 9278 : break;
1702 :
1703 6 : case ISD::ANY_EXTEND_VECTOR_INREG:
1704 : case ISD::SIGN_EXTEND_VECTOR_INREG:
1705 : case ISD::ZERO_EXTEND_VECTOR_INREG:
1706 6 : Res = SplitVecOp_ExtVecInRegOp(N);
1707 6 : break;
1708 :
1709 13 : case ISD::VECREDUCE_FADD:
1710 : case ISD::VECREDUCE_FMUL:
1711 : case ISD::VECREDUCE_ADD:
1712 : case ISD::VECREDUCE_MUL:
1713 : case ISD::VECREDUCE_AND:
1714 : case ISD::VECREDUCE_OR:
1715 : case ISD::VECREDUCE_XOR:
1716 : case ISD::VECREDUCE_SMAX:
1717 : case ISD::VECREDUCE_SMIN:
1718 : case ISD::VECREDUCE_UMAX:
1719 : case ISD::VECREDUCE_UMIN:
1720 : case ISD::VECREDUCE_FMAX:
1721 : case ISD::VECREDUCE_FMIN:
1722 13 : Res = SplitVecOp_VECREDUCE(N, OpNo);
1723 13 : break;
1724 : }
1725 : }
1726 :
1727 : // If the result is null, the sub-method took care of registering results etc.
1728 71735 : if (!Res.getNode()) return false;
1729 :
1730 : // If the result is N, the sub-method updated N in place. Tell the legalizer
1731 : // core about this.
1732 71702 : if (Res.getNode() == N)
1733 : return true;
1734 :
1735 : assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1736 : "Invalid operand expansion");
1737 :
1738 50112 : ReplaceValueWith(SDValue(N, 0), Res);
1739 50112 : return false;
1740 : }
1741 :
1742 140 : SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1743 : // The only possibility for an illegal operand is the mask, since result type
1744 : // legalization would have handled this node already otherwise.
1745 : assert(OpNo == 0 && "Illegal operand must be mask");
1746 :
1747 140 : SDValue Mask = N->getOperand(0);
1748 140 : SDValue Src0 = N->getOperand(1);
1749 140 : SDValue Src1 = N->getOperand(2);
1750 280 : EVT Src0VT = Src0.getValueType();
1751 : SDLoc DL(N);
1752 : assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
1753 :
1754 140 : SDValue Lo, Hi;
1755 280 : GetSplitVector(N->getOperand(0), Lo, Hi);
1756 : assert(Lo.getValueType() == Hi.getValueType() &&
1757 : "Lo and Hi have differing types");
1758 :
1759 : EVT LoOpVT, HiOpVT;
1760 140 : std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
1761 : assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
1762 :
1763 : SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1764 140 : std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
1765 140 : std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
1766 140 : std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
1767 :
1768 : SDValue LoSelect =
1769 280 : DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1770 : SDValue HiSelect =
1771 280 : DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1772 :
1773 280 : return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1774 : }
1775 :
1776 13 : SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
1777 13 : EVT ResVT = N->getValueType(0);
1778 13 : SDValue Lo, Hi;
1779 : SDLoc dl(N);
1780 :
1781 26 : SDValue VecOp = N->getOperand(OpNo);
1782 13 : EVT VecVT = VecOp.getValueType();
1783 : assert(VecVT.isVector() && "Can only split reduce vector operand");
1784 13 : GetSplitVector(VecOp, Lo, Hi);
1785 : EVT LoOpVT, HiOpVT;
1786 13 : std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
1787 :
1788 13 : bool NoNaN = N->getFlags().hasNoNaNs();
1789 : unsigned CombineOpc = 0;
1790 26 : switch (N->getOpcode()) {
1791 : case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
1792 0 : case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
1793 5 : case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
1794 0 : case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
1795 0 : case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
1796 0 : case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
1797 0 : case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
1798 2 : case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
1799 2 : case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
1800 2 : case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
1801 2 : case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
1802 0 : case ISD::VECREDUCE_FMAX:
1803 0 : CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXNAN;
1804 : break;
1805 0 : case ISD::VECREDUCE_FMIN:
1806 0 : CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINNAN;
1807 : break;
1808 0 : default:
1809 0 : llvm_unreachable("Unexpected reduce ISD node");
1810 : }
1811 :
1812 : // Use the appropriate scalar instruction on the split subvectors before
1813 : // reducing the now partially reduced smaller vector.
1814 26 : SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
1815 39 : return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
1816 : }
1817 :
1818 10803 : SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1819 : // The result has a legal vector type, but the input needs splitting.
1820 10803 : EVT ResVT = N->getValueType(0);
1821 10803 : SDValue Lo, Hi;
1822 : SDLoc dl(N);
1823 21606 : GetSplitVector(N->getOperand(0), Lo, Hi);
1824 21606 : EVT InVT = Lo.getValueType();
1825 :
1826 10803 : EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1827 10803 : InVT.getVectorNumElements());
1828 :
1829 32409 : Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1830 32409 : Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1831 :
1832 21606 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1833 : }
1834 :
1835 5492 : SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1836 : // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1837 : // end up being split all the way down to individual components. Convert the
1838 : // split pieces into integers and reassemble.
1839 5492 : SDValue Lo, Hi;
1840 10984 : GetSplitVector(N->getOperand(0), Lo, Hi);
1841 5492 : Lo = BitConvertToInteger(Lo);
1842 5492 : Hi = BitConvertToInteger(Hi);
1843 :
1844 5492 : if (DAG.getDataLayout().isBigEndian())
1845 : std::swap(Lo, Hi);
1846 :
1847 5492 : return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1848 5492 : JoinIntegers(Lo, Hi));
1849 : }
1850 :
1851 22998 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1852 : // We know that the extracted result type is legal.
1853 22998 : EVT SubVT = N->getValueType(0);
1854 22998 : SDValue Idx = N->getOperand(1);
1855 : SDLoc dl(N);
1856 22998 : SDValue Lo, Hi;
1857 45996 : GetSplitVector(N->getOperand(0), Lo, Hi);
1858 :
1859 45996 : uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1860 22998 : uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1861 :
1862 22998 : if (IdxVal < LoElts) {
1863 : assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1864 : "Extracted subvector crosses vector split!");
1865 23880 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1866 : } else {
1867 11058 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1868 : DAG.getConstant(IdxVal - LoElts, dl,
1869 11058 : Idx.getValueType()));
1870 : }
1871 : }
1872 :
1873 22384 : SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1874 22384 : SDValue Vec = N->getOperand(0);
1875 22384 : SDValue Idx = N->getOperand(1);
1876 22384 : EVT VecVT = Vec.getValueType();
1877 :
1878 : if (isa<ConstantSDNode>(Idx)) {
1879 22166 : uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1880 : assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1881 :
1882 22166 : SDValue Lo, Hi;
1883 22166 : GetSplitVector(Vec, Lo, Hi);
1884 :
1885 44332 : uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1886 :
1887 22166 : if (IdxVal < LoElts)
1888 12892 : return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1889 9274 : return SDValue(DAG.UpdateNodeOperands(N, Hi,
1890 9274 : DAG.getConstant(IdxVal - LoElts, SDLoc(N),
1891 9274 : Idx.getValueType())), 0);
1892 : }
1893 :
1894 : // See if the target wants to custom expand this node.
1895 436 : if (CustomLowerNode(N, N->getValueType(0), true))
1896 0 : return SDValue();
1897 :
1898 : // Make the vector elements byte-addressable if they aren't already.
1899 : SDLoc dl(N);
1900 218 : EVT EltVT = VecVT.getVectorElementType();
1901 218 : if (VecVT.getScalarSizeInBits() < 8) {
1902 2 : EltVT = MVT::i8;
1903 2 : VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1904 2 : VecVT.getVectorNumElements());
1905 4 : Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1906 : }
1907 :
1908 : // Store the vector to the stack.
1909 218 : SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1910 218 : auto &MF = DAG.getMachineFunction();
1911 218 : auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1912 218 : auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1913 436 : SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1914 :
1915 : // Load back the required element.
1916 218 : StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1917 218 : return DAG.getExtLoad(
1918 : ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1919 218 : MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
1920 : }
1921 :
1922 6 : SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
1923 6 : SDValue Lo, Hi;
1924 :
1925 : // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
1926 : // splitting the result has the same effect as splitting the input operand.
1927 6 : SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1928 :
1929 12 : return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
1930 : }
1931 :
1932 33 : SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
1933 : unsigned OpNo) {
1934 33 : EVT LoVT, HiVT;
1935 : SDLoc dl(MGT);
1936 66 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1937 :
1938 33 : SDValue Ch = MGT->getChain();
1939 33 : SDValue Ptr = MGT->getBasePtr();
1940 33 : SDValue Index = MGT->getIndex();
1941 33 : SDValue Scale = MGT->getScale();
1942 33 : SDValue Mask = MGT->getMask();
1943 33 : SDValue PassThru = MGT->getPassThru();
1944 33 : unsigned Alignment = MGT->getOriginalAlignment();
1945 :
1946 33 : SDValue MaskLo, MaskHi;
1947 66 : if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1948 : // Split Mask operand
1949 0 : GetSplitVector(Mask, MaskLo, MaskHi);
1950 : else
1951 33 : std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1952 :
1953 33 : EVT MemoryVT = MGT->getMemoryVT();
1954 33 : EVT LoMemVT, HiMemVT;
1955 33 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1956 :
1957 33 : SDValue PassThruLo, PassThruHi;
1958 66 : if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1959 0 : GetSplitVector(PassThru, PassThruLo, PassThruHi);
1960 : else
1961 33 : std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1962 :
1963 33 : SDValue IndexHi, IndexLo;
1964 66 : if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1965 33 : GetSplitVector(Index, IndexLo, IndexHi);
1966 : else
1967 0 : std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1968 :
1969 33 : MachineMemOperand *MMO = DAG.getMachineFunction().
1970 99 : getMachineMemOperand(MGT->getPointerInfo(),
1971 : MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1972 33 : Alignment, MGT->getAAInfo(), MGT->getRanges());
1973 :
1974 33 : SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1975 33 : SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
1976 33 : OpsLo, MMO);
1977 :
1978 33 : MMO = DAG.getMachineFunction().
1979 99 : getMachineMemOperand(MGT->getPointerInfo(),
1980 : MachineMemOperand::MOLoad, HiMemVT.getStoreSize(),
1981 33 : Alignment, MGT->getAAInfo(),
1982 : MGT->getRanges());
1983 :
1984 33 : SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1985 33 : SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
1986 33 : OpsHi, MMO);
1987 :
1988 : // Build a factor node to remember that this load is independent of the
1989 : // other one.
1990 33 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1991 66 : Hi.getValue(1));
1992 :
1993 : // Legalize the chain result - switch anything that used the old chain to
1994 : // use the new one.
1995 33 : ReplaceValueWith(SDValue(MGT, 1), Ch);
1996 :
1997 33 : SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
1998 66 : Hi);
1999 33 : ReplaceValueWith(SDValue(MGT, 0), Res);
2000 33 : return SDValue();
2001 : }
2002 :
2003 8 : SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2004 : unsigned OpNo) {
2005 8 : SDValue Ch = N->getChain();
2006 8 : SDValue Ptr = N->getBasePtr();
2007 8 : SDValue Mask = N->getMask();
2008 8 : SDValue Data = N->getValue();
2009 8 : EVT MemoryVT = N->getMemoryVT();
2010 8 : unsigned Alignment = N->getOriginalAlignment();
2011 : SDLoc DL(N);
2012 :
2013 8 : EVT LoMemVT, HiMemVT;
2014 8 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2015 :
2016 8 : SDValue DataLo, DataHi;
2017 16 : if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2018 : // Split Data operand
2019 8 : GetSplitVector(Data, DataLo, DataHi);
2020 : else
2021 0 : std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2022 :
2023 8 : SDValue MaskLo, MaskHi;
2024 16 : if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2025 : // Split Mask operand
2026 0 : GetSplitVector(Mask, MaskLo, MaskHi);
2027 : else
2028 8 : std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2029 :
2030 : // if Alignment is equal to the vector size,
2031 : // take the half of it for the second part
2032 : unsigned SecondHalfAlignment =
2033 16 : (Alignment == Data->getValueType(0).getSizeInBits()/8) ?
2034 : Alignment/2 : Alignment;
2035 :
2036 : SDValue Lo, Hi;
2037 8 : MachineMemOperand *MMO = DAG.getMachineFunction().
2038 24 : getMachineMemOperand(N->getPointerInfo(),
2039 : MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2040 8 : Alignment, N->getAAInfo(), N->getRanges());
2041 :
2042 8 : Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2043 8 : N->isTruncatingStore(),
2044 16 : N->isCompressingStore());
2045 :
2046 8 : Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2047 8 : N->isCompressingStore());
2048 : unsigned HiOffset = LoMemVT.getStoreSize();
2049 :
2050 24 : MMO = DAG.getMachineFunction().getMachineMemOperand(
2051 : N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
2052 8 : HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
2053 : N->getRanges());
2054 :
2055 8 : Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2056 16 : N->isTruncatingStore(), N->isCompressingStore());
2057 :
2058 : // Build a factor node to remember that this store is independent of the
2059 : // other one.
2060 16 : return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2061 : }
2062 :
2063 22 : SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2064 : unsigned OpNo) {
2065 22 : SDValue Ch = N->getChain();
2066 22 : SDValue Ptr = N->getBasePtr();
2067 22 : SDValue Mask = N->getMask();
2068 22 : SDValue Index = N->getIndex();
2069 22 : SDValue Scale = N->getScale();
2070 22 : SDValue Data = N->getValue();
2071 22 : EVT MemoryVT = N->getMemoryVT();
2072 22 : unsigned Alignment = N->getOriginalAlignment();
2073 : SDLoc DL(N);
2074 :
2075 : // Split all operands
2076 22 : EVT LoMemVT, HiMemVT;
2077 22 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2078 :
2079 22 : SDValue DataLo, DataHi;
2080 44 : if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2081 : // Split Data operand
2082 13 : GetSplitVector(Data, DataLo, DataHi);
2083 : else
2084 9 : std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2085 :
2086 22 : SDValue MaskLo, MaskHi;
2087 44 : if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2088 : // Split Mask operand
2089 0 : GetSplitVector(Mask, MaskLo, MaskHi);
2090 : else
2091 22 : std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2092 :
2093 22 : SDValue IndexHi, IndexLo;
2094 44 : if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2095 18 : GetSplitVector(Index, IndexLo, IndexHi);
2096 : else
2097 4 : std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2098 :
2099 : SDValue Lo;
2100 22 : MachineMemOperand *MMO = DAG.getMachineFunction().
2101 66 : getMachineMemOperand(N->getPointerInfo(),
2102 : MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2103 22 : Alignment, N->getAAInfo(), N->getRanges());
2104 :
2105 22 : SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2106 22 : Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2107 44 : DL, OpsLo, MMO);
2108 :
2109 22 : MMO = DAG.getMachineFunction().
2110 66 : getMachineMemOperand(N->getPointerInfo(),
2111 : MachineMemOperand::MOStore, HiMemVT.getStoreSize(),
2112 22 : Alignment, N->getAAInfo(), N->getRanges());
2113 :
2114 : // The order of the Scatter operation after split is well defined. The "Hi"
2115 : // part comes after the "Lo". So these two operations should be chained one
2116 : // after another.
2117 22 : SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2118 22 : return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2119 44 : DL, OpsHi, MMO);
2120 : }
2121 :
2122 8932 : SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2123 : assert(N->isUnindexed() && "Indexed store of vector?");
2124 : assert(OpNo == 1 && "Can only split the stored value");
2125 : SDLoc DL(N);
2126 :
2127 8932 : bool isTruncating = N->isTruncatingStore();
2128 8932 : SDValue Ch = N->getChain();
2129 8932 : SDValue Ptr = N->getBasePtr();
2130 8932 : EVT MemoryVT = N->getMemoryVT();
2131 8932 : unsigned Alignment = N->getOriginalAlignment();
2132 8932 : MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2133 : AAMDNodes AAInfo = N->getAAInfo();
2134 8932 : SDValue Lo, Hi;
2135 8932 : GetSplitVector(N->getOperand(1), Lo, Hi);
2136 :
2137 8932 : EVT LoMemVT, HiMemVT;
2138 8932 : std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2139 :
2140 : // Scalarize if the split halves are not byte-sized.
2141 8932 : if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2142 117 : return TLI.scalarizeVectorStore(N, DAG);
2143 :
2144 8815 : unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2145 :
2146 8815 : if (isTruncating)
2147 0 : Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2148 0 : Alignment, MMOFlags, AAInfo);
2149 : else
2150 8815 : Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2151 17630 : AAInfo);
2152 :
2153 : // Increment the pointer to the other half.
2154 8815 : Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2155 :
2156 8815 : if (isTruncating)
2157 0 : Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2158 0 : N->getPointerInfo().getWithOffset(IncrementSize),
2159 0 : HiMemVT, Alignment, MMOFlags, AAInfo);
2160 : else
2161 8815 : Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2162 8815 : N->getPointerInfo().getWithOffset(IncrementSize),
2163 17630 : Alignment, MMOFlags, AAInfo);
2164 :
2165 17630 : return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2166 : }
2167 :
2168 2 : SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2169 : SDLoc DL(N);
2170 :
2171 : // The input operands all must have the same type, and we know the result
2172 : // type is valid. Convert this to a buildvector which extracts all the
2173 : // input elements.
2174 : // TODO: If the input elements are power-two vectors, we could convert this to
2175 : // a new CONCAT_VECTORS node with elements that are half-wide.
2176 : SmallVector<SDValue, 32> Elts;
2177 4 : EVT EltVT = N->getValueType(0).getVectorElementType();
2178 10 : for (const SDValue &Op : N->op_values()) {
2179 24 : for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2180 56 : i != e; ++i) {
2181 96 : Elts.push_back(DAG.getNode(
2182 : ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2183 96 : DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2184 : }
2185 : }
2186 :
2187 6 : return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2188 : }
2189 :
2190 2081 : SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2191 : // The result type is legal, but the input type is illegal. If splitting
2192 : // ends up with the result type of each half still being legal, just
2193 : // do that. If, however, that would result in an illegal result type,
2194 : // we can try to get more clever with power-two vectors. Specifically,
2195 : // split the input type, but also widen the result element size, then
2196 : // concatenate the halves and truncate again. For example, consider a target
2197 : // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2198 : // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2199 : // %inlo = v4i32 extract_subvector %in, 0
2200 : // %inhi = v4i32 extract_subvector %in, 4
2201 : // %lo16 = v4i16 trunc v4i32 %inlo
2202 : // %hi16 = v4i16 trunc v4i32 %inhi
2203 : // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2204 : // %res = v8i8 trunc v8i16 %in16
2205 : //
2206 : // Without this transform, the original truncate would end up being
2207 : // scalarized, which is pretty much always a last resort.
2208 2081 : SDValue InVec = N->getOperand(0);
2209 2081 : EVT InVT = InVec->getValueType(0);
2210 4162 : EVT OutVT = N->getValueType(0);
2211 : unsigned NumElements = OutVT.getVectorNumElements();
2212 2081 : bool IsFloat = OutVT.isFloatingPoint();
2213 :
2214 : // Widening should have already made sure this is a power-two vector
2215 : // if we're trying to split it at all. assert() that's true, just in case.
2216 : assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2217 :
2218 : unsigned InElementSize = InVT.getScalarSizeInBits();
2219 : unsigned OutElementSize = OutVT.getScalarSizeInBits();
2220 :
2221 : // If the input elements are only 1/2 the width of the result elements,
2222 : // just use the normal splitting. Our trick only work if there's room
2223 : // to split more than once.
2224 2081 : if (InElementSize <= OutElementSize * 2)
2225 1407 : return SplitVecOp_UnaryOp(N);
2226 : SDLoc DL(N);
2227 :
2228 : // Get the split input vector.
2229 674 : SDValue InLoVec, InHiVec;
2230 674 : GetSplitVector(InVec, InLoVec, InHiVec);
2231 : // Truncate them to 1/2 the element size.
2232 : EVT HalfElementVT = IsFloat ?
2233 42 : EVT::getFloatingPointVT(InElementSize/2) :
2234 674 : EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2235 674 : EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2236 674 : NumElements/2);
2237 2022 : SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2238 2022 : SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2239 : // Concatenate them to get the full intermediate truncation result.
2240 674 : EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2241 674 : SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2242 674 : HalfHi);
2243 : // Now finish up by truncating all the way down to the original result
2244 : // type. This should normally be something that ends up being legal directly,
2245 : // but in theory if a target has very wide vectors and an annoyingly
2246 : // restricted set of legal types, this split can chain to build things up.
2247 : return IsFloat
2248 21 : ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2249 : DAG.getTargetConstant(
2250 42 : 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2251 695 : : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2252 : }
2253 :
2254 144 : SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2255 : assert(N->getValueType(0).isVector() &&
2256 : N->getOperand(0).getValueType().isVector() &&
2257 : "Operand types must be vectors");
2258 : // The result has a legal vector type, but the input needs splitting.
2259 144 : SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2260 : SDLoc DL(N);
2261 288 : GetSplitVector(N->getOperand(0), Lo0, Hi0);
2262 288 : GetSplitVector(N->getOperand(1), Lo1, Hi1);
2263 432 : unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2264 144 : EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2265 288 : EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2266 :
2267 432 : LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2268 432 : HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2269 288 : SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2270 288 : return PromoteTargetBoolean(Con, N->getValueType(0));
2271 : }
2272 :
2273 :
2274 82 : SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2275 : // The result has a legal vector type, but the input needs splitting.
2276 82 : EVT ResVT = N->getValueType(0);
2277 82 : SDValue Lo, Hi;
2278 : SDLoc DL(N);
2279 164 : GetSplitVector(N->getOperand(0), Lo, Hi);
2280 164 : EVT InVT = Lo.getValueType();
2281 :
2282 82 : EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2283 82 : InVT.getVectorNumElements());
2284 :
2285 246 : Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2286 246 : Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2287 :
2288 164 : return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2289 : }
2290 :
2291 8 : SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2292 : // The result (and the first input) has a legal vector type, but the second
2293 : // input needs splitting.
2294 24 : return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2295 : }
2296 :
2297 :
2298 : //===----------------------------------------------------------------------===//
2299 : // Result Vector Widening
2300 : //===----------------------------------------------------------------------===//
2301 :
2302 10716 : void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2303 : LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2304 : dbgs() << "\n");
2305 :
2306 : // See if the target wants to custom widen this node.
2307 21432 : if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2308 381 : return;
2309 :
2310 10335 : SDValue Res = SDValue();
2311 20670 : switch (N->getOpcode()) {
2312 0 : default:
2313 : #ifndef NDEBUG
2314 : dbgs() << "WidenVectorResult #" << ResNo << ": ";
2315 : N->dump(&DAG);
2316 : dbgs() << "\n";
2317 : #endif
2318 0 : llvm_unreachable("Do not know how to widen the result of this operator!");
2319 :
2320 0 : case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2321 2180 : case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2322 649 : case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2323 51 : case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2324 1955 : case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2325 0 : case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
2326 40 : case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2327 2672 : case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2328 6 : case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2329 15 : case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2330 111 : case ISD::VSELECT:
2331 111 : case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2332 0 : case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2333 114 : case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2334 845 : case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2335 : case ISD::VECTOR_SHUFFLE:
2336 116 : Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2337 116 : break;
2338 : case ISD::MLOAD:
2339 10 : Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2340 10 : break;
2341 : case ISD::MGATHER:
2342 18 : Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2343 18 : break;
2344 :
2345 223 : case ISD::ADD:
2346 : case ISD::AND:
2347 : case ISD::MUL:
2348 : case ISD::MULHS:
2349 : case ISD::MULHU:
2350 : case ISD::OR:
2351 : case ISD::SUB:
2352 : case ISD::XOR:
2353 : case ISD::FMINNUM:
2354 : case ISD::FMAXNUM:
2355 : case ISD::FMINNAN:
2356 : case ISD::FMAXNAN:
2357 : case ISD::SMIN:
2358 : case ISD::SMAX:
2359 : case ISD::UMIN:
2360 : case ISD::UMAX:
2361 223 : Res = WidenVecRes_Binary(N);
2362 223 : break;
2363 :
2364 185 : case ISD::FADD:
2365 : case ISD::FMUL:
2366 : case ISD::FPOW:
2367 : case ISD::FSUB:
2368 : case ISD::FDIV:
2369 : case ISD::FREM:
2370 : case ISD::SDIV:
2371 : case ISD::UDIV:
2372 : case ISD::SREM:
2373 : case ISD::UREM:
2374 185 : Res = WidenVecRes_BinaryCanTrap(N);
2375 185 : break;
2376 :
2377 72 : case ISD::STRICT_FADD:
2378 : case ISD::STRICT_FSUB:
2379 : case ISD::STRICT_FMUL:
2380 : case ISD::STRICT_FDIV:
2381 : case ISD::STRICT_FREM:
2382 : case ISD::STRICT_FSQRT:
2383 : case ISD::STRICT_FMA:
2384 : case ISD::STRICT_FPOW:
2385 : case ISD::STRICT_FPOWI:
2386 : case ISD::STRICT_FSIN:
2387 : case ISD::STRICT_FCOS:
2388 : case ISD::STRICT_FEXP:
2389 : case ISD::STRICT_FEXP2:
2390 : case ISD::STRICT_FLOG:
2391 : case ISD::STRICT_FLOG10:
2392 : case ISD::STRICT_FLOG2:
2393 : case ISD::STRICT_FRINT:
2394 : case ISD::STRICT_FNEARBYINT:
2395 72 : Res = WidenVecRes_StrictFP(N);
2396 72 : break;
2397 :
2398 5 : case ISD::FCOPYSIGN:
2399 5 : Res = WidenVecRes_FCOPYSIGN(N);
2400 5 : break;
2401 :
2402 1 : case ISD::FPOWI:
2403 1 : Res = WidenVecRes_POWI(N);
2404 1 : break;
2405 :
2406 37 : case ISD::SHL:
2407 : case ISD::SRA:
2408 : case ISD::SRL:
2409 37 : Res = WidenVecRes_Shift(N);
2410 37 : break;
2411 :
2412 16 : case ISD::ANY_EXTEND_VECTOR_INREG:
2413 : case ISD::SIGN_EXTEND_VECTOR_INREG:
2414 : case ISD::ZERO_EXTEND_VECTOR_INREG:
2415 16 : Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2416 16 : break;
2417 :
2418 956 : case ISD::ANY_EXTEND:
2419 : case ISD::FP_EXTEND:
2420 : case ISD::FP_ROUND:
2421 : case ISD::FP_TO_SINT:
2422 : case ISD::FP_TO_UINT:
2423 : case ISD::SIGN_EXTEND:
2424 : case ISD::SINT_TO_FP:
2425 : case ISD::TRUNCATE:
2426 : case ISD::UINT_TO_FP:
2427 : case ISD::ZERO_EXTEND:
2428 956 : Res = WidenVecRes_Convert(N);
2429 956 : break;
2430 :
2431 51 : case ISD::FABS:
2432 : case ISD::FCEIL:
2433 : case ISD::FCOS:
2434 : case ISD::FEXP:
2435 : case ISD::FEXP2:
2436 : case ISD::FFLOOR:
2437 : case ISD::FLOG:
2438 : case ISD::FLOG10:
2439 : case ISD::FLOG2:
2440 : case ISD::FNEARBYINT:
2441 : case ISD::FRINT:
2442 : case ISD::FROUND:
2443 : case ISD::FSIN:
2444 : case ISD::FSQRT:
2445 : case ISD::FTRUNC: {
2446 : // We're going to widen this vector op to a legal type by padding with undef
2447 : // elements. If the wide vector op is eventually going to be expanded to
2448 : // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2449 : // libcalls on the undef elements. We are assuming that if the scalar op
2450 : // requires expanding, then the vector op needs expanding too.
2451 51 : EVT VT = N->getValueType(0);
2452 51 : if (TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2453 24 : EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2454 : assert(!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2455 : "Target supports vector op, but scalar requires expansion?");
2456 48 : Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2457 : break;
2458 51 : }
2459 : }
2460 : // If the target has custom/legal support for the scalar FP intrinsic ops
2461 : // (they are probably not destined to become libcalls), then widen those like
2462 : // any other unary ops.
2463 : LLVM_FALLTHROUGH;
2464 :
2465 : case ISD::BITREVERSE:
2466 : case ISD::BSWAP:
2467 : case ISD::CTLZ:
2468 : case ISD::CTPOP:
2469 : case ISD::CTTZ:
2470 : case ISD::FNEG:
2471 : case ISD::FCANONICALIZE:
2472 32 : Res = WidenVecRes_Unary(N);
2473 32 : break;
2474 2 : case ISD::FMA:
2475 2 : Res = WidenVecRes_Ternary(N);
2476 2 : break;
2477 : }
2478 :
2479 : // If Res is null, the sub-method took care of registering the result.
2480 10335 : if (Res.getNode())
2481 10335 : SetWidenedVector(SDValue(N, ResNo), Res);
2482 : }
2483 :
2484 2 : SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2485 : // Ternary op widening.
2486 : SDLoc dl(N);
2487 4 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2488 4 : SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2489 4 : SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2490 4 : SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2491 6 : return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2492 : }
2493 :
2494 223 : SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2495 : // Binary op widening.
2496 : SDLoc dl(N);
2497 446 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2498 446 : SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2499 446 : SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2500 669 : return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2501 : }
2502 :
2503 : // Given a vector of operations that have been broken up to widen, see
2504 : // if we can collect them together into the next widest legal VT. This
2505 : // implementation is trap-safe.
2506 113 : static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
2507 : SmallVectorImpl<SDValue> &ConcatOps,
2508 : unsigned ConcatEnd, EVT VT, EVT MaxVT,
2509 : EVT WidenVT) {
2510 : // Check to see if we have a single operation with the widen type.
2511 113 : if (ConcatEnd == 1) {
2512 12 : VT = ConcatOps[0].getValueType();
2513 12 : if (VT == WidenVT)
2514 0 : return ConcatOps[0];
2515 : }
2516 :
2517 : SDLoc dl(ConcatOps[0]);
2518 113 : EVT WidenEltVT = WidenVT.getVectorElementType();
2519 : int Idx = 0;
2520 :
2521 : // while (Some element of ConcatOps is not of type MaxVT) {
2522 : // From the end of ConcatOps, collect elements of the same type and put
2523 : // them into an op of the next larger supported type
2524 : // }
2525 474 : while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2526 135 : Idx = ConcatEnd - 1;
2527 405 : VT = ConcatOps[Idx--].getValueType();
2528 253 : while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2529 118 : Idx--;
2530 :
2531 157 : int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2532 : EVT NextVT;
2533 : do {
2534 211 : NextSize *= 2;
2535 211 : NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2536 : } while (!TLI.isTypeLegal(NextVT));
2537 :
2538 135 : if (!VT.isVector()) {
2539 : // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2540 113 : SDValue VecOp = DAG.getUNDEF(NextVT);
2541 113 : unsigned NumToInsert = ConcatEnd - Idx - 1;
2542 326 : for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2543 213 : VecOp = DAG.getNode(
2544 213 : ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2545 213 : DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2546 : }
2547 113 : ConcatOps[Idx+1] = VecOp;
2548 113 : ConcatEnd = Idx + 2;
2549 : } else {
2550 : // Vector type, create a CONCAT_VECTORS of type NextVT
2551 22 : SDValue undefVec = DAG.getUNDEF(VT);
2552 22 : unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2553 22 : SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2554 22 : unsigned RealVals = ConcatEnd - Idx - 1;
2555 : unsigned SubConcatEnd = 0;
2556 22 : unsigned SubConcatIdx = Idx + 1;
2557 62 : while (SubConcatEnd < RealVals)
2558 120 : SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2559 26 : while (SubConcatEnd < OpsToConcat)
2560 8 : SubConcatOps[SubConcatEnd++] = undefVec;
2561 22 : ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2562 22 : NextVT, SubConcatOps);
2563 22 : ConcatEnd = SubConcatIdx + 1;
2564 : }
2565 : }
2566 :
2567 : // Check to see if we have a single operation with the widen type.
2568 113 : if (ConcatEnd == 1) {
2569 86 : VT = ConcatOps[0].getValueType();
2570 86 : if (VT == WidenVT)
2571 86 : return ConcatOps[0];
2572 : }
2573 :
2574 : // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2575 27 : unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2576 27 : if (NumOps != ConcatEnd ) {
2577 3 : SDValue UndefVal = DAG.getUNDEF(MaxVT);
2578 6 : for (unsigned j = ConcatEnd; j < NumOps; ++j)
2579 6 : ConcatOps[j] = UndefVal;
2580 : }
2581 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2582 27 : makeArrayRef(ConcatOps.data(), NumOps));
2583 : }
2584 :
2585 189 : SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2586 : // Binary op widening for operations that can trap.
2587 189 : unsigned Opcode = N->getOpcode();
2588 : SDLoc dl(N);
2589 378 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2590 189 : EVT WidenEltVT = WidenVT.getVectorElementType();
2591 189 : EVT VT = WidenVT;
2592 : unsigned NumElts = VT.getVectorNumElements();
2593 189 : const SDNodeFlags Flags = N->getFlags();
2594 245 : while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2595 27 : NumElts = NumElts / 2;
2596 27 : VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2597 : }
2598 :
2599 189 : if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2600 : // Operation doesn't trap so just widen as normal.
2601 288 : SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2602 288 : SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2603 288 : return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2604 : }
2605 :
2606 : // No legal vector version so unroll the vector operation and then widen.
2607 45 : if (NumElts == 1)
2608 8 : return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2609 :
2610 : // Since the operation can trap, apply operation on the original vector.
2611 41 : EVT MaxVT = VT;
2612 82 : SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2613 82 : SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2614 82 : unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2615 :
2616 41 : SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2617 : unsigned ConcatEnd = 0; // Current ConcatOps index.
2618 : int Idx = 0; // Current Idx into input vectors.
2619 :
2620 : // NumElts := greatest legal vector size (at most WidenVT)
2621 : // while (orig. vector has unhandled elements) {
2622 : // take munches of size NumElts from the beginning and add to ConcatOps
2623 : // NumElts := next smaller supported vector size or 1
2624 : // }
2625 86 : while (CurNumElts != 0) {
2626 57 : while (CurNumElts >= NumElts) {
2627 12 : SDValue EOp1 = DAG.getNode(
2628 : ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2629 12 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2630 12 : SDValue EOp2 = DAG.getNode(
2631 : ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2632 12 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2633 12 : ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2634 12 : Idx += NumElts;
2635 12 : CurNumElts -= NumElts;
2636 : }
2637 : do {
2638 85 : NumElts = NumElts / 2;
2639 85 : VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2640 166 : } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2641 :
2642 45 : if (NumElts == 1) {
2643 110 : for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2644 69 : SDValue EOp1 = DAG.getNode(
2645 : ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
2646 69 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2647 69 : SDValue EOp2 = DAG.getNode(
2648 : ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
2649 69 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2650 138 : ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
2651 69 : EOp1, EOp2, Flags);
2652 : }
2653 : CurNumElts = 0;
2654 : }
2655 : }
2656 :
2657 41 : return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
2658 : }
2659 :
2660 72 : SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
2661 : // StrictFP op widening for operations that can trap.
2662 72 : unsigned NumOpers = N->getNumOperands();
2663 72 : unsigned Opcode = N->getOpcode();
2664 : SDLoc dl(N);
2665 144 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2666 72 : EVT WidenEltVT = WidenVT.getVectorElementType();
2667 72 : EVT VT = WidenVT;
2668 : unsigned NumElts = VT.getVectorNumElements();
2669 108 : while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2670 18 : NumElts = NumElts / 2;
2671 18 : VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2672 : }
2673 :
2674 : // No legal vector version so unroll the vector operation and then widen.
2675 72 : if (NumElts == 1)
2676 0 : return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2677 :
2678 : // Since the operation can trap, apply operation on the original vector.
2679 72 : EVT MaxVT = VT;
2680 : SmallVector<SDValue, 4> InOps;
2681 144 : unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2682 :
2683 72 : SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2684 : SmallVector<SDValue, 16> Chains;
2685 : unsigned ConcatEnd = 0; // Current ConcatOps index.
2686 : int Idx = 0; // Current Idx into input vectors.
2687 :
2688 : // The Chain is the first operand.
2689 144 : InOps.push_back(N->getOperand(0));
2690 :
2691 : // Now process the remaining operands.
2692 180 : for (unsigned i = 1; i < NumOpers; ++i) {
2693 108 : SDValue Oper = N->getOperand(i);
2694 :
2695 324 : if (Oper.getValueType().isVector()) {
2696 : assert(Oper.getValueType() == N->getValueType(0) &&
2697 : "Invalid operand type to widen!");
2698 104 : Oper = GetWidenedVector(Oper);
2699 : }
2700 :
2701 108 : InOps.push_back(Oper);
2702 : }
2703 :
2704 : // NumElts := greatest legal vector size (at most WidenVT)
2705 : // while (orig. vector has unhandled elements) {
2706 : // take munches of size NumElts from the beginning and add to ConcatOps
2707 : // NumElts := next smaller supported vector size or 1
2708 : // }
2709 162 : while (CurNumElts != 0) {
2710 126 : while (CurNumElts >= NumElts) {
2711 : SmallVector<SDValue, 4> EOps;
2712 :
2713 126 : for (unsigned i = 0; i < NumOpers; ++i) {
2714 90 : SDValue Op = InOps[i];
2715 :
2716 270 : if (Op.getValueType().isVector())
2717 52 : Op = DAG.getNode(
2718 : ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
2719 52 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2720 :
2721 90 : EOps.push_back(Op);
2722 : }
2723 :
2724 36 : EVT OperVT[] = {VT, MVT::Other};
2725 72 : SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
2726 36 : ConcatOps[ConcatEnd++] = Oper;
2727 36 : Chains.push_back(Oper.getValue(1));
2728 36 : Idx += NumElts;
2729 36 : CurNumElts -= NumElts;
2730 : }
2731 : do {
2732 126 : NumElts = NumElts / 2;
2733 126 : VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2734 234 : } while (!TLI.isTypeLegal(VT) && NumElts != 1);
2735 :
2736 90 : if (NumElts == 1) {
2737 216 : for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2738 : SmallVector<SDValue, 4> EOps;
2739 :
2740 504 : for (unsigned i = 0; i < NumOpers; ++i) {
2741 360 : SDValue Op = InOps[i];
2742 :
2743 1080 : if (Op.getValueType().isVector())
2744 208 : Op = DAG.getNode(
2745 : ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
2746 : DAG.getConstant(Idx, dl,
2747 208 : TLI.getVectorIdxTy(DAG.getDataLayout())));
2748 :
2749 360 : EOps.push_back(Op);
2750 : }
2751 :
2752 144 : EVT WidenVT[] = {WidenEltVT, MVT::Other};
2753 288 : SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
2754 144 : ConcatOps[ConcatEnd++] = Oper;
2755 144 : Chains.push_back(Oper.getValue(1));
2756 : }
2757 : CurNumElts = 0;
2758 : }
2759 : }
2760 :
2761 : // Build a factor node to remember all the Ops that have been created.
2762 72 : SDValue NewChain;
2763 144 : if (Chains.size() == 1)
2764 0 : NewChain = Chains[0];
2765 : else
2766 144 : NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
2767 72 : ReplaceValueWith(SDValue(N, 1), NewChain);
2768 :
2769 72 : return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
2770 : }
2771 :
2772 956 : SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
2773 956 : SDValue InOp = N->getOperand(0);
2774 : SDLoc DL(N);
2775 :
2776 1912 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2777 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
2778 :
2779 956 : EVT InVT = InOp.getValueType();
2780 956 : EVT InEltVT = InVT.getVectorElementType();
2781 956 : EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2782 :
2783 956 : unsigned Opcode = N->getOpcode();
2784 : unsigned InVTNumElts = InVT.getVectorNumElements();
2785 956 : const SDNodeFlags Flags = N->getFlags();
2786 956 : if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2787 524 : InOp = GetWidenedVector(N->getOperand(0));
2788 262 : InVT = InOp.getValueType();
2789 : InVTNumElts = InVT.getVectorNumElements();
2790 262 : if (InVTNumElts == WidenNumElts) {
2791 249 : if (N->getNumOperands() == 1)
2792 474 : return DAG.getNode(Opcode, DL, WidenVT, InOp);
2793 24 : return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
2794 : }
2795 13 : if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
2796 : // If both input and result vector types are of same width, extend
2797 : // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
2798 : // accepts fewer elements in the result than in the input.
2799 13 : if (Opcode == ISD::SIGN_EXTEND)
2800 4 : return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2801 9 : if (Opcode == ISD::ZERO_EXTEND)
2802 5 : return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2803 : }
2804 : }
2805 :
2806 698 : if (TLI.isTypeLegal(InWidenVT)) {
2807 : // Because the result and the input are different vector types, widening
2808 : // the result could create a legal type but widening the input might make
2809 : // it an illegal type that might lead to repeatedly splitting the input
2810 : // and then widening it. To avoid this, we widen the input only if
2811 : // it results in a legal type.
2812 60 : if (WidenNumElts % InVTNumElts == 0) {
2813 : // Widen the input and call convert on the widened input vector.
2814 60 : unsigned NumConcat = WidenNumElts/InVTNumElts;
2815 60 : SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
2816 60 : Ops[0] = InOp;
2817 120 : SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
2818 60 : if (N->getNumOperands() == 1)
2819 120 : return DAG.getNode(Opcode, DL, WidenVT, InVec);
2820 0 : return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
2821 : }
2822 :
2823 0 : if (InVTNumElts % WidenNumElts == 0) {
2824 0 : SDValue InVal = DAG.getNode(
2825 : ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
2826 0 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2827 : // Extract the input and convert the shorten input vector.
2828 0 : if (N->getNumOperands() == 1)
2829 0 : return DAG.getNode(Opcode, DL, WidenVT, InVal);
2830 0 : return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
2831 : }
2832 : }
2833 :
2834 : // Otherwise unroll into some nasty scalar code and rebuild the vector.
2835 638 : EVT EltVT = WidenVT.getVectorElementType();
2836 638 : SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
2837 : // Use the original element count so we don't do more scalar opts than
2838 : // necessary.
2839 1914 : unsigned MinElts = N->getValueType(0).getVectorNumElements();
2840 2374 : for (unsigned i=0; i < MinElts; ++i) {
2841 1736 : SDValue Val = DAG.getNode(
2842 : ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
2843 1736 : DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2844 1736 : if (N->getNumOperands() == 1)
2845 3408 : Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
2846 : else
2847 64 : Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
2848 : }
2849 :
2850 1276 : return DAG.getBuildVector(WidenVT, DL, Ops);
2851 : }
2852 :
2853 16 : SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
2854 16 : unsigned Opcode = N->getOpcode();
2855 16 : SDValue InOp = N->getOperand(0);
2856 : SDLoc DL(N);
2857 :
2858 32 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2859 16 : EVT WidenSVT = WidenVT.getVectorElementType();
2860 16 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
2861 :
2862 16 : EVT InVT = InOp.getValueType();
2863 16 : EVT InSVT = InVT.getVectorElementType();
2864 16 : unsigned InVTNumElts = InVT.getVectorNumElements();
2865 :
2866 16 : if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2867 16 : InOp = GetWidenedVector(InOp);
2868 16 : InVT = InOp.getValueType();
2869 16 : if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
2870 16 : switch (Opcode) {
2871 0 : case ISD::ANY_EXTEND_VECTOR_INREG:
2872 0 : return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT);
2873 5 : case ISD::SIGN_EXTEND_VECTOR_INREG:
2874 5 : return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT);
2875 11 : case ISD::ZERO_EXTEND_VECTOR_INREG:
2876 11 : return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT);
2877 : }
2878 : }
2879 : }
2880 :
2881 : // Unroll, extend the scalars and rebuild the vector.
2882 : SmallVector<SDValue, 16> Ops;
2883 0 : for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2884 0 : SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
2885 0 : DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
2886 0 : switch (Opcode) {
2887 0 : case ISD::ANY_EXTEND_VECTOR_INREG:
2888 0 : Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
2889 0 : break;
2890 0 : case ISD::SIGN_EXTEND_VECTOR_INREG:
2891 0 : Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
2892 0 : break;
2893 0 : case ISD::ZERO_EXTEND_VECTOR_INREG:
2894 0 : Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
2895 0 : break;
2896 0 : default:
2897 0 : llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
2898 : }
2899 0 : Ops.push_back(Val);
2900 : }
2901 :
2902 0 : while (Ops.size() != WidenNumElts)
2903 0 : Ops.push_back(DAG.getUNDEF(WidenSVT));
2904 :
2905 0 : return DAG.getBuildVector(WidenVT, DL, Ops);
2906 : }
2907 :
2908 5 : SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
2909 : // If this is an FCOPYSIGN with same input types, we can treat it as a
2910 : // normal (can trap) binary op.
2911 13 : if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
2912 4 : return WidenVecRes_BinaryCanTrap(N);
2913 :
2914 : // If the types are different, fall back to unrolling.
2915 2 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2916 2 : return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2917 : }
2918 :
2919 1 : SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
2920 2 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2921 2 : SDValue InOp = GetWidenedVector(N->getOperand(0));
2922 1 : SDValue ShOp = N->getOperand(1);
2923 1 : return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2924 : }
2925 :
2926 37 : SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
2927 74 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2928 74 : SDValue InOp = GetWidenedVector(N->getOperand(0));
2929 37 : SDValue ShOp = N->getOperand(1);
2930 :
2931 37 : EVT ShVT = ShOp.getValueType();
2932 37 : if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
2933 37 : ShOp = GetWidenedVector(ShOp);
2934 37 : ShVT = ShOp.getValueType();
2935 : }
2936 37 : EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
2937 : ShVT.getVectorElementType(),
2938 37 : WidenVT.getVectorNumElements());
2939 37 : if (ShVT != ShWidenVT)
2940 0 : ShOp = ModifyToType(ShOp, ShWidenVT);
2941 :
2942 37 : return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
2943 : }
2944 :
2945 32 : SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
2946 : // Unary op widening.
2947 64 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2948 64 : SDValue InOp = GetWidenedVector(N->getOperand(0));
2949 32 : return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
2950 : }
2951 :
2952 15 : SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
2953 30 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2954 15 : EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
2955 15 : cast<VTSDNode>(N->getOperand(1))->getVT()
2956 : .getVectorElementType(),
2957 15 : WidenVT.getVectorNumElements());
2958 30 : SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
2959 15 : return DAG.getNode(N->getOpcode(), SDLoc(N),
2960 45 : WidenVT, WidenLHS, DAG.getValueType(ExtVT));
2961 : }
2962 :
2963 0 : SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
2964 0 : SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2965 0 : return GetWidenedVector(WidenVec);
2966 : }
2967 :
2968 2180 : SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
2969 2180 : SDValue InOp = N->getOperand(0);
2970 2180 : EVT InVT = InOp.getValueType();
2971 2180 : EVT VT = N->getValueType(0);
2972 2180 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2973 : SDLoc dl(N);
2974 :
2975 2180 : switch (getTypeAction(InVT)) {
2976 : case TargetLowering::TypeLegal:
2977 : break;
2978 : case TargetLowering::TypePromoteInteger:
2979 : // If the incoming type is a vector that is being promoted, then
2980 : // we know that the elements are arranged differently and that we
2981 : // must perform the conversion using a stack slot.
2982 39 : if (InVT.isVector())
2983 : break;
2984 :
2985 : // If the InOp is promoted to the same size, convert it. Otherwise,
2986 : // fall out of the switch and widen the promoted input.
2987 22 : InOp = GetPromotedInteger(InOp);
2988 22 : InVT = InOp.getValueType();
2989 22 : if (WidenVT.bitsEq(InVT))
2990 28 : return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
2991 : break;
2992 : case TargetLowering::TypeSoftenFloat:
2993 : case TargetLowering::TypePromoteFloat:
2994 : case TargetLowering::TypeExpandInteger:
2995 : case TargetLowering::TypeExpandFloat:
2996 : case TargetLowering::TypeScalarizeVector:
2997 : case TargetLowering::TypeSplitVector:
2998 : break;
2999 2052 : case TargetLowering::TypeWidenVector:
3000 : // If the InOp is widened to the same size, convert it. Otherwise, fall
3001 : // out of the switch and widen the widened input.
3002 2052 : InOp = GetWidenedVector(InOp);
3003 2052 : InVT = InOp.getValueType();
3004 2052 : if (WidenVT.bitsEq(InVT))
3005 : // The input widens to the same size. Convert to the widen value.
3006 4104 : return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3007 : break;
3008 : }
3009 :
3010 114 : unsigned WidenSize = WidenVT.getSizeInBits();
3011 114 : unsigned InSize = InVT.getSizeInBits();
3012 : // x86mmx is not an acceptable vector element type, so don't try.
3013 114 : if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3014 : // Determine new input vector type. The new input vector type will use
3015 : // the same element type (if its a vector) or use the input type as a
3016 : // vector. It is the same size as the type to widen to.
3017 : EVT NewInVT;
3018 89 : unsigned NewNumElts = WidenSize / InSize;
3019 89 : if (InVT.isVector()) {
3020 17 : EVT InEltVT = InVT.getVectorElementType();
3021 17 : NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3022 17 : WidenSize / InEltVT.getSizeInBits());
3023 : } else {
3024 72 : NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3025 : }
3026 :
3027 89 : if (TLI.isTypeLegal(NewInVT)) {
3028 87 : SDValue NewVec;
3029 87 : if (InVT.isVector()) {
3030 : // Because the result and the input are different vector types, widening
3031 : // the result could create a legal type but widening the input might make
3032 : // it an illegal type that might lead to repeatedly splitting the input
3033 : // and then widening it. To avoid this, we widen the input only if
3034 : // it results in a legal type.
3035 17 : SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3036 17 : Ops[0] = InOp;
3037 :
3038 34 : NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3039 : } else {
3040 140 : NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3041 : }
3042 174 : return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3043 : }
3044 : }
3045 :
3046 27 : return CreateStackStoreLoad(InOp, WidenVT);
3047 : }
3048 :
3049 649 : SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3050 : SDLoc dl(N);
3051 : // Build a vector with undefined for the new nodes.
3052 649 : EVT VT = N->getValueType(0);
3053 :
3054 : // Integer BUILD_VECTOR operands may be larger than the node's vector element
3055 : // type. The UNDEFs need to have the same type as the existing operands.
3056 1298 : EVT EltVT = N->getOperand(0).getValueType();
3057 : unsigned NumElts = VT.getVectorNumElements();
3058 :
3059 649 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3060 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
3061 :
3062 649 : SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3063 : assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3064 649 : NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3065 :
3066 1298 : return DAG.getBuildVector(WidenVT, dl, NewOps);
3067 : }
3068 :
3069 51 : SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3070 51 : EVT InVT = N->getOperand(0).getValueType();
3071 102 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3072 : SDLoc dl(N);
3073 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
3074 : unsigned NumInElts = InVT.getVectorNumElements();
3075 51 : unsigned NumOperands = N->getNumOperands();
3076 :
3077 : bool InputWidened = false; // Indicates we need to widen the input.
3078 51 : if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3079 16 : if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3080 : // Add undef vectors to widen to correct length.
3081 : unsigned NumConcat = WidenVT.getVectorNumElements() /
3082 16 : InVT.getVectorNumElements();
3083 16 : SDValue UndefVal = DAG.getUNDEF(InVT);
3084 16 : SmallVector<SDValue, 16> Ops(NumConcat);
3085 68 : for (unsigned i=0; i < NumOperands; ++i)
3086 104 : Ops[i] = N->getOperand(i);
3087 44 : for (unsigned i = NumOperands; i != NumConcat; ++i)
3088 56 : Ops[i] = UndefVal;
3089 32 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3090 : }
3091 : } else {
3092 : InputWidened = true;
3093 35 : if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3094 : // The inputs and the result are widen to the same value.
3095 : unsigned i;
3096 12 : for (i=1; i < NumOperands; ++i)
3097 24 : if (!N->getOperand(i).isUndef())
3098 : break;
3099 :
3100 8 : if (i == NumOperands)
3101 : // Everything but the first operand is an UNDEF so just return the
3102 : // widened first operand.
3103 8 : return GetWidenedVector(N->getOperand(0));
3104 :
3105 4 : if (NumOperands == 2) {
3106 : // Replace concat of two operands with a shuffle.
3107 4 : SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3108 16 : for (unsigned i = 0; i < NumInElts; ++i) {
3109 12 : MaskOps[i] = i;
3110 24 : MaskOps[i + NumInElts] = i + WidenNumElts;
3111 : }
3112 4 : return DAG.getVectorShuffle(WidenVT, dl,
3113 4 : GetWidenedVector(N->getOperand(0)),
3114 4 : GetWidenedVector(N->getOperand(1)),
3115 4 : MaskOps);
3116 : }
3117 : }
3118 : }
3119 :
3120 : // Fall back to use extracts and build vector.
3121 27 : EVT EltVT = WidenVT.getVectorElementType();
3122 27 : SmallVector<SDValue, 16> Ops(WidenNumElts);
3123 : unsigned Idx = 0;
3124 97 : for (unsigned i=0; i < NumOperands; ++i) {
3125 70 : SDValue InOp = N->getOperand(i);
3126 70 : if (InputWidened)
3127 70 : InOp = GetWidenedVector(InOp);
3128 684 : for (unsigned j=0; j < NumInElts; ++j)
3129 1228 : Ops[Idx++] = DAG.getNode(
3130 : ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3131 614 : DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3132 : }
3133 27 : SDValue UndefVal = DAG.getUNDEF(EltVT);
3134 261 : for (; Idx < WidenNumElts; ++Idx)
3135 468 : Ops[Idx] = UndefVal;
3136 54 : return DAG.getBuildVector(WidenVT, dl, Ops);
3137 : }
3138 :
3139 1955 : SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3140 1955 : EVT VT = N->getValueType(0);
3141 1955 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3142 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
3143 1955 : SDValue InOp = N->getOperand(0);
3144 1955 : SDValue Idx = N->getOperand(1);
3145 : SDLoc dl(N);
3146 :
3147 1955 : if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3148 52 : InOp = GetWidenedVector(InOp);
3149 :
3150 1955 : EVT InVT = InOp.getValueType();
3151 :
3152 : // Check if we can just return the input vector after widening.
3153 1955 : uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3154 1955 : if (IdxVal == 0 && InVT == WidenVT)
3155 1690 : return InOp;
3156 :
3157 : // Check if we can extract from the vector.
3158 : unsigned InNumElts = InVT.getVectorNumElements();
3159 265 : if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3160 12 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3161 :
3162 : // We could try widening the input to the right length but for now, extract
3163 : // the original elements, fill the rest with undefs and build a vector.
3164 259 : SmallVector<SDValue, 16> Ops(WidenNumElts);
3165 259 : EVT EltVT = VT.getVectorElementType();
3166 : unsigned NumElts = VT.getVectorNumElements();
3167 : unsigned i;
3168 1007 : for (i=0; i < NumElts; ++i)
3169 748 : Ops[i] =
3170 748 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3171 : DAG.getConstant(IdxVal + i, dl,
3172 748 : TLI.getVectorIdxTy(DAG.getDataLayout())));
3173 :
3174 259 : SDValue UndefVal = DAG.getUNDEF(EltVT);
3175 1199 : for (; i < WidenNumElts; ++i)
3176 1880 : Ops[i] = UndefVal;
3177 518 : return DAG.getBuildVector(WidenVT, dl, Ops);
3178 : }
3179 :
3180 40 : SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3181 80 : SDValue InOp = GetWidenedVector(N->getOperand(0));
3182 40 : return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3183 : InOp.getValueType(), InOp,
3184 80 : N->getOperand(1), N->getOperand(2));
3185 : }
3186 :
3187 2672 : SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3188 : LoadSDNode *LD = cast<LoadSDNode>(N);
3189 : ISD::LoadExtType ExtType = LD->getExtensionType();
3190 :
3191 : SDValue Result;
3192 : SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3193 2672 : if (ExtType != ISD::NON_EXTLOAD)
3194 0 : Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3195 : else
3196 2672 : Result = GenWidenVectorLoads(LdChain, LD);
3197 :
3198 : // If we generate a single load, we can use that for the chain. Otherwise,
3199 : // build a factor node to remember the multiple loads are independent and
3200 : // chain to that.
3201 2672 : SDValue NewChain;
3202 5344 : if (LdChain.size() == 1)
3203 366 : NewChain = LdChain[0];
3204 : else
3205 2306 : NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3206 :
3207 : // Modified the chain - switch anything that used the old chain to use
3208 : // the new one.
3209 2672 : ReplaceValueWith(SDValue(N, 1), NewChain);
3210 :
3211 2672 : return Result;
3212 : }
3213 :
3214 10 : SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3215 :
3216 20 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3217 10 : SDValue Mask = N->getMask();
3218 10 : EVT MaskVT = Mask.getValueType();
3219 10 : SDValue PassThru = GetWidenedVector(N->getPassThru());
3220 : ISD::LoadExtType ExtType = N->getExtensionType();
3221 : SDLoc dl(N);
3222 :
3223 : // The mask should be widened as well
3224 10 : EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3225 : MaskVT.getVectorElementType(),
3226 10 : WidenVT.getVectorNumElements());
3227 10 : Mask = ModifyToType(Mask, WideMaskVT, true);
3228 :
3229 10 : SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3230 : Mask, PassThru, N->getMemoryVT(),
3231 : N->getMemOperand(), ExtType,
3232 20 : N->isExpandingLoad());
3233 : // Legalize the chain result - switch anything that used the old chain to
3234 : // use the new one.
3235 10 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3236 10 : return Res;
3237 : }
3238 :
3239 18 : SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3240 :
3241 36 : EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3242 18 : SDValue Mask = N->getMask();
3243 18 : EVT MaskVT = Mask.getValueType();
3244 18 : SDValue PassThru = GetWidenedVector(N->getPassThru());
3245 18 : SDValue Scale = N->getScale();
3246 : unsigned NumElts = WideVT.getVectorNumElements();
3247 : SDLoc dl(N);
3248 :
3249 : // The mask should be widened as well
3250 18 : EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3251 : MaskVT.getVectorElementType(),
3252 18 : WideVT.getVectorNumElements());
3253 18 : Mask = ModifyToType(Mask, WideMaskVT, true);
3254 :
3255 : // Widen the Index operand
3256 18 : SDValue Index = N->getIndex();
3257 18 : EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3258 36 : Index.getValueType().getScalarType(),
3259 18 : NumElts);
3260 18 : Index = ModifyToType(Index, WideIndexVT);
3261 : SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3262 18 : Scale };
3263 18 : SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3264 : N->getMemoryVT(), dl, Ops,
3265 36 : N->getMemOperand());
3266 :
3267 : // Legalize the chain result - switch anything that used the old chain to
3268 : // use the new one.
3269 18 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3270 18 : return Res;
3271 : }
3272 :
3273 6 : SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3274 12 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3275 6 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3276 6 : WidenVT, N->getOperand(0));
3277 : }
3278 :
3279 : // Return true if this is a node that could have two SETCCs as operands.
3280 : static inline bool isLogicalMaskOp(unsigned Opcode) {
3281 860 : switch (Opcode) {
3282 : case ISD::AND:
3283 : case ISD::OR:
3284 : case ISD::XOR:
3285 : return true;
3286 : }
3287 : return false;
3288 : }
3289 :
3290 : // This is used just for the assert in convertMask(). Check that this either
3291 : // a SETCC or a previously handled SETCC by convertMask().
3292 : #ifndef NDEBUG
3293 : static inline bool isSETCCorConvertedSETCC(SDValue N) {
3294 : if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3295 : N = N.getOperand(0);
3296 : else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3297 : for (unsigned i = 1; i < N->getNumOperands(); ++i)
3298 : if (!N->getOperand(i)->isUndef())
3299 : return false;
3300 : N = N.getOperand(0);
3301 : }
3302 :
3303 : if (N.getOpcode() == ISD::TRUNCATE)
3304 : N = N.getOperand(0);
3305 : else if (N.getOpcode() == ISD::SIGN_EXTEND)
3306 : N = N.getOperand(0);
3307 :
3308 : if (isLogicalMaskOp(N.getOpcode()))
3309 : return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3310 : isSETCCorConvertedSETCC(N.getOperand(1));
3311 :
3312 : return (N.getOpcode() == ISD::SETCC ||
3313 : ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
3314 : }
3315 : #endif
3316 :
3317 : // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3318 : // to ToMaskVT if needed with vector extension or truncation.
3319 1121 : SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3320 : EVT ToMaskVT) {
3321 : // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3322 : // FIXME: This code seems to be too restrictive, we might consider
3323 : // generalizing it or dropping it.
3324 : assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3325 :
3326 : // Make a new Mask node, with a legal result VT.
3327 : SmallVector<SDValue, 4> Ops;
3328 4414 : for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3329 6586 : Ops.push_back(InMask->getOperand(i));
3330 1153 : SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3331 :
3332 : // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3333 : // extend or truncate is needed.
3334 1121 : LLVMContext &Ctx = *DAG.getContext();
3335 : unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3336 : unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3337 1121 : if (MaskScalarBits < ToMaskScalBits) {
3338 : EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3339 59 : MaskVT.getVectorNumElements());
3340 59 : Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3341 1062 : } else if (MaskScalarBits > ToMaskScalBits) {
3342 : EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3343 87 : MaskVT.getVectorNumElements());
3344 87 : Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3345 : }
3346 :
3347 : assert(Mask->getValueType(0).getScalarSizeInBits() ==
3348 : ToMaskVT.getScalarSizeInBits() &&
3349 : "Mask should have the right element size by now.");
3350 :
3351 : // Adjust Mask to the right number of elements.
3352 3363 : unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3353 1121 : if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3354 28 : MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3355 28 : SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3356 56 : Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3357 56 : ZeroIdx);
3358 1093 : } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3359 39 : unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3360 39 : EVT SubVT = Mask->getValueType(0);
3361 39 : SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3362 39 : SubOps[0] = Mask;
3363 39 : Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3364 : }
3365 :
3366 : assert((Mask->getValueType(0) == ToMaskVT) &&
3367 : "A mask of ToMaskVT should have been produced by now.");
3368 :
3369 1121 : return Mask;
3370 : }
3371 :
3372 : // Get the target mask VT, and widen if needed.
3373 1051 : EVT DAGTypeLegalizer::getSETCCWidenedResultTy(SDValue SetCC) {
3374 : assert(SetCC->getOpcode() == ISD::SETCC);
3375 1051 : LLVMContext &Ctx = *DAG.getContext();
3376 2102 : EVT MaskVT = getSetCCResultType(SetCC->getOperand(0).getValueType());
3377 1051 : if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
3378 62 : MaskVT = TLI.getTypeToTransformTo(Ctx, MaskVT);
3379 1051 : return MaskVT;
3380 : }
3381 :
3382 : // This method tries to handle VSELECT and its mask by legalizing operands
3383 : // (which may require widening) and if needed adjusting the mask vector type
3384 : // to match that of the VSELECT. Without it, many cases end up with
3385 : // scalarization of the SETCC, with many unnecessary instructions.
3386 1809 : SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3387 1809 : LLVMContext &Ctx = *DAG.getContext();
3388 1809 : SDValue Cond = N->getOperand(0);
3389 :
3390 1809 : if (N->getOpcode() != ISD::VSELECT)
3391 0 : return SDValue();
3392 :
3393 3618 : if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
3394 716 : return SDValue();
3395 :
3396 : // If this is a splitted VSELECT that was previously already handled, do
3397 : // nothing.
3398 2186 : EVT CondVT = Cond->getValueType(0);
3399 1093 : if (CondVT.getScalarSizeInBits() != 1)
3400 1 : return SDValue();
3401 :
3402 1092 : EVT VSelVT = N->getValueType(0);
3403 : // Only handle vector types which are a power of 2.
3404 1092 : if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3405 28 : return SDValue();
3406 :
3407 : // Don't touch if this will be scalarized.
3408 1064 : EVT FinalVT = VSelVT;
3409 1697 : while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3410 633 : FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3411 :
3412 1064 : if (FinalVT.getVectorNumElements() == 1)
3413 70 : return SDValue();
3414 :
3415 : // If there is support for an i1 vector mask, don't touch.
3416 994 : if (Cond.getOpcode() == ISD::SETCC) {
3417 920 : EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3418 1492 : while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3419 572 : SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3420 920 : EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3421 920 : if (SetCCResVT.getScalarSizeInBits() == 1)
3422 9 : return SDValue();
3423 148 : } else if (CondVT.getScalarType() == MVT::i1) {
3424 : // If there is support for an i1 vector mask (or only scalar i1 conditions),
3425 : // don't touch.
3426 152 : while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3427 78 : CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3428 :
3429 144 : if (CondVT.getScalarType() == MVT::i1)
3430 4 : return SDValue();
3431 : }
3432 :
3433 : // Get the VT and operands for VSELECT, and widen if needed.
3434 981 : SDValue VSelOp1 = N->getOperand(1);
3435 981 : SDValue VSelOp2 = N->getOperand(2);
3436 981 : if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3437 61 : VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3438 61 : VSelOp1 = GetWidenedVector(VSelOp1);
3439 61 : VSelOp2 = GetWidenedVector(VSelOp2);
3440 : }
3441 :
3442 : // The mask of the VSELECT should have integer elements.
3443 981 : EVT ToMaskVT = VSelVT;
3444 981 : if (!ToMaskVT.getScalarType().isInteger())
3445 452 : ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3446 :
3447 981 : SDValue Mask;
3448 1962 : if (Cond->getOpcode() == ISD::SETCC) {
3449 911 : EVT MaskVT = getSETCCWidenedResultTy(Cond);
3450 911 : Mask = convertMask(Cond, MaskVT, ToMaskVT);
3451 70 : } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3452 70 : Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3453 70 : Cond->getOperand(1).getOpcode() == ISD::SETCC) {
3454 : // Cond is (AND/OR/XOR (SETCC, SETCC))
3455 70 : SDValue SETCC0 = Cond->getOperand(0);
3456 70 : SDValue SETCC1 = Cond->getOperand(1);
3457 70 : EVT VT0 = getSETCCWidenedResultTy(SETCC0);
3458 70 : EVT VT1 = getSETCCWidenedResultTy(SETCC1);
3459 : unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3460 : unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3461 : unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3462 : EVT MaskVT;
3463 : // If the two SETCCs have different VTs, either extend/truncate one of
3464 : // them to the other "towards" ToMaskVT, or truncate one and extend the
3465 : // other to ToMaskVT.
3466 70 : if (ScalarBits0 != ScalarBits1) {
3467 40 : EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3468 80 : EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3469 40 : if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3470 18 : MaskVT = WideVT;
3471 22 : else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3472 22 : MaskVT = NarrowVT;
3473 : else
3474 0 : MaskVT = ToMaskVT;
3475 : } else
3476 : // If the two SETCCs have the same VT, don't change it.
3477 30 : MaskVT = VT0;
3478 :
3479 : // Make new SETCCs and logical nodes.
3480 70 : SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3481 70 : SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3482 70 : Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3483 :
3484 : // Convert the logical op for VSELECT if needed.
3485 70 : Mask = convertMask(Cond, MaskVT, ToMaskVT);
3486 : } else
3487 0 : return SDValue();
3488 :
3489 1978 : return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3490 : }
3491 :
3492 111 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3493 222 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3494 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
3495 :
3496 111 : SDValue Cond1 = N->getOperand(0);
3497 111 : EVT CondVT = Cond1.getValueType();
3498 111 : if (CondVT.isVector()) {
3499 99 : if (SDValue Res = WidenVSELECTAndMask(N))
3500 61 : return Res;
3501 :
3502 38 : EVT CondEltVT = CondVT.getVectorElementType();
3503 38 : EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
3504 38 : CondEltVT, WidenNumElts);
3505 38 : if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3506 28 : Cond1 = GetWidenedVector(Cond1);
3507 :
3508 : // If we have to split the condition there is no point in widening the
3509 : // select. This would result in an cycle of widening the select ->
3510 : // widening the condition operand -> splitting the condition operand ->
3511 : // splitting the select -> widening the select. Instead split this select
3512 : // further and widen the resulting type.
3513 38 : if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3514 4 : SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3515 4 : SDValue Res = ModifyToType(SplitSelect, WidenVT);
3516 4 : return Res;
3517 : }
3518 :
3519 34 : if (Cond1.getValueType() != CondWidenVT)
3520 6 : Cond1 = ModifyToType(Cond1, CondWidenVT);
3521 : }
3522 :
3523 92 : SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3524 92 : SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3525 : assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3526 46 : return DAG.getNode(N->getOpcode(), SDLoc(N),
3527 138 : WidenVT, Cond1, InOp1, InOp2);
3528 : }
3529 :
3530 0 : SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3531 0 : SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3532 0 : SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3533 0 : return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3534 : InOp1.getValueType(), N->getOperand(0),
3535 0 : N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3536 : }
3537 :
3538 845 : SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3539 1690 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3540 845 : return DAG.getUNDEF(WidenVT);
3541 : }
3542 :
3543 116 : SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3544 232 : EVT VT = N->getValueType(0);
3545 : SDLoc dl(N);
3546 :
3547 116 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3548 : unsigned NumElts = VT.getVectorNumElements();
3549 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
3550 :
3551 232 : SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3552 232 : SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3553 :
3554 : // Adjust mask based on new input vector length.
3555 : SmallVector<int, 16> NewMask;
3556 479 : for (unsigned i = 0; i != NumElts; ++i) {
3557 363 : int Idx = N->getMaskElt(i);
3558 363 : if (Idx < (int)NumElts)
3559 302 : NewMask.push_back(Idx);
3560 : else
3561 61 : NewMask.push_back(Idx - NumElts + WidenNumElts);
3562 : }
3563 517 : for (unsigned i = NumElts; i != WidenNumElts; ++i)
3564 401 : NewMask.push_back(-1);
3565 232 : return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
3566 : }
3567 :
3568 114 : SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
3569 : assert(N->getValueType(0).isVector() &&
3570 : N->getOperand(0).getValueType().isVector() &&
3571 : "Operands must be vectors");
3572 228 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3573 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
3574 :
3575 114 : SDValue InOp1 = N->getOperand(0);
3576 114 : EVT InVT = InOp1.getValueType();
3577 : assert(InVT.isVector() && "can not widen non-vector type");
3578 114 : EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
3579 114 : InVT.getVectorElementType(), WidenNumElts);
3580 :
3581 : // The input and output types often differ here, and it could be that while
3582 : // we'd prefer to widen the result type, the input operands have been split.
3583 : // In this case, we also need to split the result of this node as well.
3584 114 : if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
3585 2 : SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3586 2 : SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3587 2 : return Res;
3588 : }
3589 :
3590 112 : InOp1 = GetWidenedVector(InOp1);
3591 224 : SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3592 :
3593 : // Assume that the input and output will be widen appropriately. If not,
3594 : // we will have to unroll it at some point.
3595 : assert(InOp1.getValueType() == WidenInVT &&
3596 : InOp2.getValueType() == WidenInVT &&
3597 : "Input not widened to expected type!");
3598 : (void)WidenInVT;
3599 112 : return DAG.getNode(ISD::SETCC, SDLoc(N),
3600 224 : WidenVT, InOp1, InOp2, N->getOperand(2));
3601 : }
3602 :
3603 :
3604 : //===----------------------------------------------------------------------===//
3605 : // Widen Vector Operand
3606 : //===----------------------------------------------------------------------===//
3607 12278 : bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
3608 : LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
3609 : dbgs() << "\n");
3610 12278 : SDValue Res = SDValue();
3611 :
3612 : // See if the target wants to custom widen this node.
3613 36834 : if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3614 : return false;
3615 :
3616 23982 : switch (N->getOpcode()) {
3617 0 : default:
3618 : #ifndef NDEBUG
3619 : dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
3620 : N->dump(&DAG);
3621 : dbgs() << "\n";
3622 : #endif
3623 0 : llvm_unreachable("Do not know how to widen this operator's operand!");
3624 :
3625 13 : case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
3626 1378 : case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
3627 72 : case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
3628 9353 : case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
3629 729 : case ISD::STORE: Res = WidenVecOp_STORE(N); break;
3630 10 : case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
3631 3 : case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
3632 5 : case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
3633 75 : case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
3634 3 : case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
3635 :
3636 235 : case ISD::ANY_EXTEND:
3637 : case ISD::SIGN_EXTEND:
3638 : case ISD::ZERO_EXTEND:
3639 235 : Res = WidenVecOp_EXTEND(N);
3640 235 : break;
3641 :
3642 115 : case ISD::FP_EXTEND:
3643 : case ISD::FP_TO_SINT:
3644 : case ISD::FP_TO_UINT:
3645 : case ISD::SINT_TO_FP:
3646 : case ISD::UINT_TO_FP:
3647 : case ISD::TRUNCATE:
3648 115 : Res = WidenVecOp_Convert(N);
3649 115 : break;
3650 : }
3651 :
3652 : // If Res is null, the sub-method took care of registering the result.
3653 11991 : if (!Res.getNode()) return false;
3654 :
3655 : // If the result is N, the sub-method updated N in place. Tell the legalizer
3656 : // core about this.
3657 11988 : if (Res.getNode() == N)
3658 : return true;
3659 :
3660 :
3661 : assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3662 : "Invalid operand expansion");
3663 :
3664 11988 : ReplaceValueWith(SDValue(N, 0), Res);
3665 11988 : return false;
3666 : }
3667 :
3668 235 : SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
3669 : SDLoc DL(N);
3670 235 : EVT VT = N->getValueType(0);
3671 :
3672 235 : SDValue InOp = N->getOperand(0);
3673 : assert(getTypeAction(InOp.getValueType()) ==
3674 : TargetLowering::TypeWidenVector &&
3675 : "Unexpected type action");
3676 235 : InOp = GetWidenedVector(InOp);
3677 : assert(VT.getVectorNumElements() <
3678 : InOp.getValueType().getVectorNumElements() &&
3679 : "Input wasn't widened!");
3680 :
3681 : // We may need to further widen the operand until it has the same total
3682 : // vector size as the result.
3683 235 : EVT InVT = InOp.getValueType();
3684 235 : if (InVT.getSizeInBits() != VT.getSizeInBits()) {
3685 0 : EVT InEltVT = InVT.getVectorElementType();
3686 0 : for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
3687 0 : EVT FixedVT = (MVT::SimpleValueType)i;
3688 0 : EVT FixedEltVT = FixedVT.getVectorElementType();
3689 0 : if (TLI.isTypeLegal(FixedVT) &&
3690 0 : FixedVT.getSizeInBits() == VT.getSizeInBits() &&
3691 0 : FixedEltVT == InEltVT) {
3692 : assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
3693 : "Not enough elements in the fixed type for the operand!");
3694 : assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
3695 : "We can't have the same type as we started with!");
3696 0 : if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
3697 0 : InOp = DAG.getNode(
3698 0 : ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
3699 0 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3700 : else
3701 0 : InOp = DAG.getNode(
3702 : ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
3703 0 : DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3704 0 : break;
3705 : }
3706 : }
3707 0 : InVT = InOp.getValueType();
3708 0 : if (InVT.getSizeInBits() != VT.getSizeInBits())
3709 : // We couldn't find a legal vector type that was a widening of the input
3710 : // and could be extended in-register to the result type, so we have to
3711 : // scalarize.
3712 0 : return WidenVecOp_Convert(N);
3713 : }
3714 :
3715 : // Use special DAG nodes to represent the operation of extending the
3716 : // low lanes.
3717 470 : switch (N->getOpcode()) {
3718 0 : default:
3719 0 : llvm_unreachable("Extend legalization on extend operation!");
3720 2 : case ISD::ANY_EXTEND:
3721 2 : return DAG.getAnyExtendVectorInReg(InOp, DL, VT);
3722 197 : case ISD::SIGN_EXTEND:
3723 197 : return DAG.getSignExtendVectorInReg(InOp, DL, VT);
3724 36 : case ISD::ZERO_EXTEND:
3725 36 : return DAG.getZeroExtendVectorInReg(InOp, DL, VT);
3726 : }
3727 : }
3728 :
3729 0 : SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
3730 : // The result (and first input) is legal, but the second input is illegal.
3731 : // We can't do much to fix that, so just unroll and let the extracts off of
3732 : // the second input be widened as needed later.
3733 3 : return DAG.UnrollVectorOp(N);
3734 : }
3735 :
3736 115 : SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
3737 : // Since the result is legal and the input is illegal.
3738 115 : EVT VT = N->getValueType(0);
3739 115 : EVT EltVT = VT.getVectorElementType();
3740 : SDLoc dl(N);
3741 : unsigned NumElts = VT.getVectorNumElements();
3742 115 : SDValue InOp = N->getOperand(0);
3743 : assert(getTypeAction(InOp.getValueType()) ==
3744 : TargetLowering::TypeWidenVector &&
3745 : "Unexpected type action");
3746 115 : InOp = GetWidenedVector(InOp);
3747 115 : EVT InVT = InOp.getValueType();
3748 115 : unsigned Opcode = N->getOpcode();
3749 :
3750 : // See if a widened result type would be legal, if so widen the node.
3751 115 : EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3752 115 : InVT.getVectorNumElements());
3753 115 : if (TLI.isTypeLegal(WideVT)) {
3754 70 : SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp);
3755 35 : return DAG.getNode(
3756 : ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
3757 35 : DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3758 : }
3759 :
3760 80 : EVT InEltVT = InVT.getVectorElementType();
3761 :
3762 : // Unroll the convert into some scalar code and create a nasty build vector.
3763 80 : SmallVector<SDValue, 16> Ops(NumElts);
3764 244 : for (unsigned i=0; i < NumElts; ++i)
3765 164 : Ops[i] = DAG.getNode(
3766 : Opcode, dl, EltVT,
3767 : DAG.getNode(
3768 : ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
3769 164 : DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3770 :
3771 160 : return DAG.getBuildVector(VT, dl, Ops);
3772 : }
3773 :
3774 13 : SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
3775 13 : EVT VT = N->getValueType(0);
3776 26 : SDValue InOp = GetWidenedVector(N->getOperand(0));
3777 26 : EVT InWidenVT = InOp.getValueType();
3778 : SDLoc dl(N);
3779 :
3780 : // Check if we can convert between two legal vector types and extract.
3781 13 : unsigned InWidenSize = InWidenVT.getSizeInBits();
3782 13 : unsigned Size = VT.getSizeInBits();
3783 : // x86mmx is not an acceptable vector element type, so don't try.
3784 26 : if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
3785 13 : unsigned NewNumElts = InWidenSize / Size;
3786 13 : EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
3787 13 : if (TLI.isTypeLegal(NewVT)) {
3788 26 : SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
3789 13 : return DAG.getNode(
3790 : ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
3791 13 : DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3792 : }
3793 : }
3794 :
3795 0 : return CreateStackStoreLoad(InOp, VT);
3796 : }
3797 :
3798 1378 : SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
3799 1378 : EVT VT = N->getValueType(0);
3800 1378 : EVT EltVT = VT.getVectorElementType();
3801 2756 : EVT InVT = N->getOperand(0).getValueType();
3802 : SDLoc dl(N);
3803 :
3804 : // If the widen width for this operand is the same as the width of the concat
3805 : // and all but the first operand is undef, just use the widened operand.
3806 1378 : unsigned NumOperands = N->getNumOperands();
3807 1378 : if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3808 : unsigned i;
3809 2512 : for (i = 1; i < NumOperands; ++i)
3810 4668 : if (!N->getOperand(i).isUndef())
3811 : break;
3812 :
3813 1338 : if (i == NumOperands)
3814 1912 : return GetWidenedVector(N->getOperand(0));
3815 : }
3816 :
3817 : // Otherwise, fall back to a nasty build vector.
3818 : unsigned NumElts = VT.getVectorNumElements();
3819 422 : SmallVector<SDValue, 16> Ops(NumElts);
3820 :
3821 : unsigned NumInElts = InVT.getVectorNumElements();
3822 :
3823 : unsigned Idx = 0;
3824 1412 : for (unsigned i=0; i < NumOperands; ++i) {
3825 990 : SDValue InOp = N->getOperand(i);
3826 : assert(getTypeAction(InOp.getValueType()) ==
3827 : TargetLowering::TypeWidenVector &&
3828 : "Unexpected type action");
3829 990 : InOp = GetWidenedVector(InOp);
3830 3540 : for (unsigned j=0; j < NumInElts; ++j)
3831 5100 : Ops[Idx++] = DAG.getNode(
3832 : ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3833 2550 : DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3834 : }
3835 844 : return DAG.getBuildVector(VT, dl, Ops);
3836 : }
3837 :
3838 72 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3839 144 : SDValue InOp = GetWidenedVector(N->getOperand(0));
3840 72 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
3841 144 : N->getValueType(0), InOp, N->getOperand(1));
3842 : }
3843 :
3844 9353 : SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3845 18706 : SDValue InOp = GetWidenedVector(N->getOperand(0));
3846 9353 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3847 18709 : N->getValueType(0), InOp, N->getOperand(1));
3848 : }
3849 :
3850 729 : SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
3851 : // We have to widen the value, but we want only to store the original
3852 : // vector type.
3853 : StoreSDNode *ST = cast<StoreSDNode>(N);
3854 :
3855 729 : if (!ST->getMemoryVT().getScalarType().isByteSized())
3856 12 : return TLI.scalarizeVectorStore(ST, DAG);
3857 :
3858 : SmallVector<SDValue, 16> StChain;
3859 717 : if (ST->isTruncatingStore())
3860 0 : GenWidenVectorTruncStores(StChain, ST);
3861 : else
3862 717 : GenWidenVectorStores(StChain, ST);
3863 :
3864 1434 : if (StChain.size() == 1)
3865 49 : return StChain[0];
3866 : else
3867 1336 : return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
3868 : }
3869 :
3870 10 : SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
3871 : assert((OpNo == 1 || OpNo == 3) &&
3872 : "Can widen only data or mask operand of mstore");
3873 : MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
3874 10 : SDValue Mask = MST->getMask();
3875 10 : EVT MaskVT = Mask.getValueType();
3876 10 : SDValue StVal = MST->getValue();
3877 : SDLoc dl(N);
3878 :
3879 10 : if (OpNo == 1) {
3880 : // Widen the value.
3881 10 : StVal = GetWidenedVector(StVal);
3882 :
3883 : // The mask should be widened as well.
3884 10 : EVT WideVT = StVal.getValueType();
3885 10 : EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3886 : MaskVT.getVectorElementType(),
3887 10 : WideVT.getVectorNumElements());
3888 10 : Mask = ModifyToType(Mask, WideMaskVT, true);
3889 : } else {
3890 : // Widen the mask.
3891 0 : EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
3892 0 : Mask = ModifyToType(Mask, WideMaskVT, true);
3893 :
3894 0 : EVT ValueVT = StVal.getValueType();
3895 0 : EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
3896 : ValueVT.getVectorElementType(),
3897 0 : WideMaskVT.getVectorNumElements());
3898 0 : StVal = ModifyToType(StVal, WideVT);
3899 : }
3900 :
3901 : assert(Mask.getValueType().getVectorNumElements() ==
3902 : StVal.getValueType().getVectorNumElements() &&
3903 : "Mask and data vectors should have the same number of elements");
3904 10 : return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
3905 : Mask, MST->getMemoryVT(), MST->getMemOperand(),
3906 20 : false, MST->isCompressingStore());
3907 : }
3908 :
3909 3 : SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
3910 : assert(OpNo == 4 && "Can widen only the index of mgather");
3911 : auto *MG = cast<MaskedGatherSDNode>(N);
3912 3 : SDValue DataOp = MG->getPassThru();
3913 3 : SDValue Mask = MG->getMask();
3914 3 : SDValue Scale = MG->getScale();
3915 :
3916 : // Just widen the index. It's allowed to have extra elements.
3917 3 : SDValue Index = GetWidenedVector(MG->getIndex());
3918 :
3919 : SDLoc dl(N);
3920 : SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
3921 3 : Scale};
3922 3 : SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
3923 9 : MG->getMemOperand());
3924 3 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3925 3 : ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
3926 3 : return SDValue();
3927 : }
3928 :
3929 5 : SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
3930 : MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
3931 5 : SDValue DataOp = MSC->getValue();
3932 5 : SDValue Mask = MSC->getMask();
3933 5 : SDValue Index = MSC->getIndex();
3934 5 : SDValue Scale = MSC->getScale();
3935 :
3936 : unsigned NumElts;
3937 5 : if (OpNo == 1) {
3938 3 : DataOp = GetWidenedVector(DataOp);
3939 6 : NumElts = DataOp.getValueType().getVectorNumElements();
3940 :
3941 : // Widen index.
3942 3 : EVT IndexVT = Index.getValueType();
3943 3 : EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3944 3 : IndexVT.getVectorElementType(), NumElts);
3945 3 : Index = ModifyToType(Index, WideIndexVT);
3946 :
3947 : // The mask should be widened as well.
3948 3 : EVT MaskVT = Mask.getValueType();
3949 3 : EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3950 3 : MaskVT.getVectorElementType(), NumElts);
3951 3 : Mask = ModifyToType(Mask, WideMaskVT, true);
3952 2 : } else if (OpNo == 4) {
3953 : // Just widen the index. It's allowed to have extra elements.
3954 2 : Index = GetWidenedVector(Index);
3955 : } else
3956 0 : llvm_unreachable("Can't widen this operand of mscatter");
3957 :
3958 : SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
3959 5 : Scale};
3960 5 : return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
3961 5 : MSC->getMemoryVT(), SDLoc(N), Ops,
3962 5 : MSC->getMemOperand());
3963 : }
3964 :
3965 75 : SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
3966 150 : SDValue InOp0 = GetWidenedVector(N->getOperand(0));
3967 150 : SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3968 : SDLoc dl(N);
3969 75 : EVT VT = N->getValueType(0);
3970 :
3971 : // WARNING: In this code we widen the compare instruction with garbage.
3972 : // This garbage may contain denormal floats which may be slow. Is this a real
3973 : // concern ? Should we zero the unused lanes if this is a float compare ?
3974 :
3975 : // Get a new SETCC node to compare the newly widened operands.
3976 : // Only some of the compared elements are legal.
3977 150 : EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3978 150 : InOp0.getValueType());
3979 : // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
3980 150 : if (VT.getScalarType() == MVT::i1)
3981 4 : SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
3982 4 : SVT.getVectorNumElements());
3983 :
3984 75 : SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
3985 150 : SVT, InOp0, InOp1, N->getOperand(2));
3986 :
3987 : // Extract the needed results from the result vector.
3988 75 : EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
3989 : SVT.getVectorElementType(),
3990 75 : VT.getVectorNumElements());
3991 75 : SDValue CC = DAG.getNode(
3992 : ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
3993 75 : DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3994 :
3995 75 : return PromoteTargetBoolean(CC, VT);
3996 : }
3997 :
3998 :
3999 : //===----------------------------------------------------------------------===//
4000 : // Vector Widening Utilities
4001 : //===----------------------------------------------------------------------===//
4002 :
4003 : // Utility function to find the type to chop up a widen vector for load/store
4004 : // TLI: Target lowering used to determine legal types.
4005 : // Width: Width left need to load/store.
4006 : // WidenVT: The widen vector type to load to/store from
4007 : // Align: If 0, don't allow use of a wider type
4008 : // WidenEx: If Align is not 0, the amount additional we can load/store from.
4009 :
4010 6212 : static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4011 : unsigned Width, EVT WidenVT,
4012 : unsigned Align = 0, unsigned WidenEx = 0) {
4013 6212 : EVT WidenEltVT = WidenVT.getVectorElementType();
4014 6212 : unsigned WidenWidth = WidenVT.getSizeInBits();
4015 6212 : unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4016 6212 : unsigned AlignInBits = Align*8;
4017 :
4018 : // If we have one element to load/store, return it.
4019 6212 : EVT RetVT = WidenEltVT;
4020 6212 : if (Width == WidenEltWidth)
4021 1168 : return RetVT;
4022 :
4023 : // See if there is larger legal integer than the element type to load/store.
4024 : unsigned VT;
4025 5772 : for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4026 10816 : VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4027 10816 : EVT MemVT((MVT::SimpleValueType) VT);
4028 10816 : unsigned MemVTWidth = MemVT.getSizeInBits();
4029 10816 : if (MemVT.getSizeInBits() <= WidenEltWidth)
4030 : break;
4031 10467 : auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4032 : if ((Action == TargetLowering::TypeLegal ||
4033 5133 : Action == TargetLowering::TypePromoteInteger) &&
4034 5133 : (WidenWidth % MemVTWidth) == 0 &&
4035 20263 : isPowerOf2_32(WidenWidth / MemVTWidth) &&
4036 348 : (MemVTWidth <= Width ||
4037 348 : (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4038 4695 : RetVT = MemVT;
4039 4695 : break;
4040 : }
4041 : }
4042 :
4043 : // See if there is a larger vector type to load/store that has the same vector
4044 : // element type and is evenly divisible with the WidenVT.
4045 412953 : for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4046 417997 : VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4047 415130 : EVT MemVT = (MVT::SimpleValueType) VT;
4048 415130 : unsigned MemVTWidth = MemVT.getSizeInBits();
4049 53397 : if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
4050 15017 : (WidenWidth % MemVTWidth) == 0 &&
4051 20430 : isPowerOf2_32(WidenWidth / MemVTWidth) &&
4052 6077 : (MemVTWidth <= Width ||
4053 6077 : (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4054 4434 : if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
4055 2177 : return MemVT;
4056 : }
4057 : }
4058 :
4059 2867 : return RetVT;
4060 : }
4061 :
4062 : // Builds a vector type from scalar loads
4063 : // VecTy: Resulting Vector type
4064 : // LDOps: Load operators to build a vector type
4065 : // [Start,End) the list of loads to use.
4066 2168 : static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4067 : SmallVectorImpl<SDValue> &LdOps,
4068 : unsigned Start, unsigned End) {
4069 2168 : const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4070 2168 : SDLoc dl(LdOps[Start]);
4071 2168 : EVT LdTy = LdOps[Start].getValueType();
4072 2168 : unsigned Width = VecTy.getSizeInBits();
4073 2168 : unsigned NumElts = Width / LdTy.getSizeInBits();
4074 2168 : EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4075 :
4076 : unsigned Idx = 1;
4077 2168 : SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4078 :
4079 3013 : for (unsigned i = Start + 1; i != End; ++i) {
4080 1690 : EVT NewLdTy = LdOps[i].getValueType();
4081 871 : if (NewLdTy != LdTy) {
4082 664 : NumElts = Width / NewLdTy.getSizeInBits();
4083 664 : NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4084 664 : VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4085 : // Readjust position and vector position based on new load type.
4086 664 : Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4087 664 : LdTy = NewLdTy;
4088 : }
4089 845 : VecOp = DAG.getNode(
4090 : ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4091 845 : DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4092 : }
4093 2168 : return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4094 : }
4095 :
4096 2672 : SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4097 : LoadSDNode *LD) {
4098 : // The strategy assumes that we can efficiently load power-of-two widths.
4099 : // The routine chops the vector into the largest vector loads with the same
4100 : // element type or scalar loads and then recombines it to the widen vector
4101 : // type.
4102 5344 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4103 2672 : unsigned WidenWidth = WidenVT.getSizeInBits();
4104 2672 : EVT LdVT = LD->getMemoryVT();
4105 : SDLoc dl(LD);
4106 : assert(LdVT.isVector() && WidenVT.isVector());
4107 : assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4108 :
4109 : // Load information
4110 2672 : SDValue Chain = LD->getChain();
4111 2672 : SDValue BasePtr = LD->getBasePtr();
4112 2672 : unsigned Align = LD->getAlignment();
4113 2672 : MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4114 : AAMDNodes AAInfo = LD->getAAInfo();
4115 :
4116 2672 : int LdWidth = LdVT.getSizeInBits();
4117 2672 : int WidthDiff = WidenWidth - LdWidth;
4118 2672 : unsigned LdAlign = LD->isVolatile() ? 0 : Align; // Allow wider loads.
4119 :
4120 : // Find the vector type that can load from.
4121 2672 : EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4122 2672 : int NewVTWidth = NewVT.getSizeInBits();
4123 5344 : SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4124 5344 : Align, MMOFlags, AAInfo);
4125 2672 : LdChain.push_back(LdOp.getValue(1));
4126 :
4127 : // Check if we can load the element with one instruction.
4128 2672 : if (LdWidth <= NewVTWidth) {
4129 366 : if (!NewVT.isVector()) {
4130 181 : unsigned NumElts = WidenWidth / NewVTWidth;
4131 181 : EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4132 362 : SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4133 362 : return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4134 : }
4135 185 : if (NewVT == WidenVT)
4136 185 : return LdOp;
4137 :
4138 : assert(WidenWidth % NewVTWidth == 0);
4139 0 : unsigned NumConcat = WidenWidth / NewVTWidth;
4140 0 : SmallVector<SDValue, 16> ConcatOps(NumConcat);
4141 0 : SDValue UndefVal = DAG.getUNDEF(NewVT);
4142 0 : ConcatOps[0] = LdOp;
4143 0 : for (unsigned i = 1; i != NumConcat; ++i)
4144 0 : ConcatOps[i] = UndefVal;
4145 0 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4146 : }
4147 :
4148 : // Load vector by using multiple loads from largest vector to scalar.
4149 : SmallVector<SDValue, 16> LdOps;
4150 2306 : LdOps.push_back(LdOp);
4151 :
4152 2306 : LdWidth -= NewVTWidth;
4153 : unsigned Offset = 0;
4154 :
4155 5126 : while (LdWidth > 0) {
4156 2820 : unsigned Increment = NewVTWidth / 8;
4157 2820 : Offset += Increment;
4158 2820 : BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4159 :
4160 2820 : SDValue L;
4161 2820 : if (LdWidth < NewVTWidth) {
4162 : // The current type we are using is too large. Find a better size.
4163 2211 : NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4164 2211 : NewVTWidth = NewVT.getSizeInBits();
4165 4422 : L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4166 2211 : LD->getPointerInfo().getWithOffset(Offset),
4167 6633 : MinAlign(Align, Increment), MMOFlags, AAInfo);
4168 2211 : LdChain.push_back(L.getValue(1));
4169 6633 : if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
4170 : // Later code assumes the vector loads produced will be mergeable, so we
4171 : // must pad the final entry up to the previous width. Scalars are
4172 : // combined separately.
4173 : SmallVector<SDValue, 16> Loads;
4174 119 : Loads.push_back(L);
4175 119 : unsigned size = L->getValueSizeInBits(0);
4176 256 : while (size < LdOp->getValueSizeInBits(0)) {
4177 18 : Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4178 18 : size += L->getValueSizeInBits(0);
4179 : }
4180 238 : L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4181 : }
4182 : } else {
4183 1218 : L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4184 609 : LD->getPointerInfo().getWithOffset(Offset),
4185 1827 : MinAlign(Align, Increment), MMOFlags, AAInfo);
4186 609 : LdChain.push_back(L.getValue(1));
4187 : }
4188 :
4189 2820 : LdOps.push_back(L);
4190 2820 : LdOp = L;
4191 :
4192 2820 : LdWidth -= NewVTWidth;
4193 : }
4194 :
4195 : // Build the vector from the load operations.
4196 2306 : unsigned End = LdOps.size();
4197 6918 : if (!LdOps[0].getValueType().isVector())
4198 : // All the loads are scalar loads.
4199 751 : return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4200 :
4201 : // If the load contains vectors, build the vector using concat vector.
4202 : // All of the vectors used to load are power-of-2, and the scalar loads can be
4203 : // combined to make a power-of-2 vector.
4204 1555 : SmallVector<SDValue, 16> ConcatOps(End);
4205 1555 : int i = End - 1;
4206 1555 : int Idx = End;
4207 4665 : EVT LdTy = LdOps[i].getValueType();
4208 : // First, combine the scalar loads to a vector.
4209 1555 : if (!LdTy.isVector()) {
4210 1417 : for (--i; i >= 0; --i) {
4211 4251 : LdTy = LdOps[i].getValueType();
4212 1417 : if (LdTy.isVector())
4213 : break;
4214 : }
4215 1417 : ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4216 : }
4217 3110 : ConcatOps[--Idx] = LdOps[i];
4218 2113 : for (--i; i >= 0; --i) {
4219 1116 : EVT NewLdTy = LdOps[i].getValueType();
4220 558 : if (NewLdTy != LdTy) {
4221 : // Create a larger vector.
4222 11 : ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4223 22 : makeArrayRef(&ConcatOps[Idx], End - Idx));
4224 : Idx = End - 1;
4225 11 : LdTy = NewLdTy;
4226 : }
4227 1116 : ConcatOps[--Idx] = LdOps[i];
4228 : }
4229 :
4230 1555 : if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
4231 1521 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4232 3042 : makeArrayRef(&ConcatOps[Idx], End - Idx));
4233 :
4234 : // We need to fill the rest with undefs to build the vector.
4235 34 : unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
4236 34 : SmallVector<SDValue, 16> WidenOps(NumOps);
4237 34 : SDValue UndefVal = DAG.getUNDEF(LdTy);
4238 : {
4239 : unsigned i = 0;
4240 503 : for (; i != End-Idx; ++i)
4241 1407 : WidenOps[i] = ConcatOps[Idx+i];
4242 229 : for (; i != NumOps; ++i)
4243 390 : WidenOps[i] = UndefVal;
4244 : }
4245 68 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4246 : }
4247 :
4248 : SDValue
4249 0 : DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4250 : LoadSDNode *LD,
4251 : ISD::LoadExtType ExtType) {
4252 : // For extension loads, it may not be more efficient to chop up the vector
4253 : // and then extend it. Instead, we unroll the load and build a new vector.
4254 0 : EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4255 0 : EVT LdVT = LD->getMemoryVT();
4256 : SDLoc dl(LD);
4257 : assert(LdVT.isVector() && WidenVT.isVector());
4258 :
4259 : // Load information
4260 0 : SDValue Chain = LD->getChain();
4261 0 : SDValue BasePtr = LD->getBasePtr();
4262 0 : unsigned Align = LD->getAlignment();
4263 0 : MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4264 : AAMDNodes AAInfo = LD->getAAInfo();
4265 :
4266 0 : EVT EltVT = WidenVT.getVectorElementType();
4267 0 : EVT LdEltVT = LdVT.getVectorElementType();
4268 : unsigned NumElts = LdVT.getVectorNumElements();
4269 :
4270 : // Load each element and widen.
4271 : unsigned WidenNumElts = WidenVT.getVectorNumElements();
4272 0 : SmallVector<SDValue, 16> Ops(WidenNumElts);
4273 0 : unsigned Increment = LdEltVT.getSizeInBits() / 8;
4274 0 : Ops[0] =
4275 0 : DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
4276 0 : LdEltVT, Align, MMOFlags, AAInfo);
4277 0 : LdChain.push_back(Ops[0].getValue(1));
4278 : unsigned i = 0, Offset = Increment;
4279 0 : for (i=1; i < NumElts; ++i, Offset += Increment) {
4280 0 : SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4281 0 : Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
4282 0 : LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
4283 0 : Align, MMOFlags, AAInfo);
4284 0 : LdChain.push_back(Ops[i].getValue(1));
4285 : }
4286 :
4287 : // Fill the rest with undefs.
4288 0 : SDValue UndefVal = DAG.getUNDEF(EltVT);
4289 0 : for (; i != WidenNumElts; ++i)
4290 0 : Ops[i] = UndefVal;
4291 :
4292 0 : return DAG.getBuildVector(WidenVT, dl, Ops);
4293 : }
4294 :
4295 717 : void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
4296 : StoreSDNode *ST) {
4297 : // The strategy assumes that we can efficiently store power-of-two widths.
4298 : // The routine chops the vector into the largest vector stores with the same
4299 : // element type or scalar stores.
4300 717 : SDValue Chain = ST->getChain();
4301 717 : SDValue BasePtr = ST->getBasePtr();
4302 717 : unsigned Align = ST->getAlignment();
4303 717 : MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4304 : AAMDNodes AAInfo = ST->getAAInfo();
4305 717 : SDValue ValOp = GetWidenedVector(ST->getValue());
4306 : SDLoc dl(ST);
4307 :
4308 717 : EVT StVT = ST->getMemoryVT();
4309 717 : unsigned StWidth = StVT.getSizeInBits();
4310 717 : EVT ValVT = ValOp.getValueType();
4311 717 : unsigned ValWidth = ValVT.getSizeInBits();
4312 717 : EVT ValEltVT = ValVT.getVectorElementType();
4313 717 : unsigned ValEltWidth = ValEltVT.getSizeInBits();
4314 : assert(StVT.getVectorElementType() == ValEltVT);
4315 :
4316 : int Idx = 0; // current index to store
4317 : unsigned Offset = 0; // offset from base to store
4318 2046 : while (StWidth != 0) {
4319 : // Find the largest vector type we can store with.
4320 1329 : EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
4321 1329 : unsigned NewVTWidth = NewVT.getSizeInBits();
4322 1329 : unsigned Increment = NewVTWidth / 8;
4323 1329 : if (NewVT.isVector()) {
4324 : unsigned NumVTElts = NewVT.getVectorNumElements();
4325 : do {
4326 989 : SDValue EOp = DAG.getNode(
4327 : ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
4328 989 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4329 989 : StChain.push_back(DAG.getStore(
4330 989 : Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4331 2967 : MinAlign(Align, Offset), MMOFlags, AAInfo));
4332 989 : StWidth -= NewVTWidth;
4333 989 : Offset += Increment;
4334 989 : Idx += NumVTElts;
4335 :
4336 989 : BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4337 989 : } while (StWidth != 0 && StWidth >= NewVTWidth);
4338 : } else {
4339 : // Cast the vector to the scalar type we can store.
4340 1022 : unsigned NumElts = ValWidth / NewVTWidth;
4341 1022 : EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4342 2044 : SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4343 : // Readjust index position based on new vector type.
4344 1022 : Idx = Idx * ValEltWidth / NewVTWidth;
4345 : do {
4346 1203 : SDValue EOp = DAG.getNode(
4347 : ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4348 1203 : DAG.getConstant(Idx++, dl,
4349 1203 : TLI.getVectorIdxTy(DAG.getDataLayout())));
4350 1203 : StChain.push_back(DAG.getStore(
4351 1203 : Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4352 3609 : MinAlign(Align, Offset), MMOFlags, AAInfo));
4353 1203 : StWidth -= NewVTWidth;
4354 1203 : Offset += Increment;
4355 1203 : BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4356 1203 : } while (StWidth != 0 && StWidth >= NewVTWidth);
4357 : // Restore index back to be relative to the original widen element type.
4358 1022 : Idx = Idx * NewVTWidth / ValEltWidth;
4359 : }
4360 : }
4361 717 : }
4362 :
4363 : void
4364 0 : DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4365 : StoreSDNode *ST) {
4366 : // For extension loads, it may not be more efficient to truncate the vector
4367 : // and then store it. Instead, we extract each element and then store it.
4368 0 : SDValue Chain = ST->getChain();
4369 0 : SDValue BasePtr = ST->getBasePtr();
4370 0 : unsigned Align = ST->getAlignment();
4371 0 : MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4372 : AAMDNodes AAInfo = ST->getAAInfo();
4373 0 : SDValue ValOp = GetWidenedVector(ST->getValue());
4374 : SDLoc dl(ST);
4375 :
4376 0 : EVT StVT = ST->getMemoryVT();
4377 0 : EVT ValVT = ValOp.getValueType();
4378 :
4379 : // It must be true that the wide vector type is bigger than where we need to
4380 : // store.
4381 : assert(StVT.isVector() && ValOp.getValueType().isVector());
4382 : assert(StVT.bitsLT(ValOp.getValueType()));
4383 :
4384 : // For truncating stores, we can not play the tricks of chopping legal vector
4385 : // types and bitcast it to the right type. Instead, we unroll the store.
4386 0 : EVT StEltVT = StVT.getVectorElementType();
4387 0 : EVT ValEltVT = ValVT.getVectorElementType();
4388 0 : unsigned Increment = ValEltVT.getSizeInBits() / 8;
4389 : unsigned NumElts = StVT.getVectorNumElements();
4390 0 : SDValue EOp = DAG.getNode(
4391 : ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4392 0 : DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4393 0 : StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4394 0 : ST->getPointerInfo(), StEltVT, Align,
4395 0 : MMOFlags, AAInfo));
4396 : unsigned Offset = Increment;
4397 0 : for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4398 0 : SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4399 0 : SDValue EOp = DAG.getNode(
4400 : ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4401 0 : DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4402 0 : StChain.push_back(DAG.getTruncStore(
4403 0 : Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4404 0 : StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4405 : }
4406 0 : }
4407 :
4408 : /// Modifies a vector input (widen or narrows) to a vector of NVT. The
4409 : /// input vector must have the same element type as NVT.
4410 : /// FillWithZeroes specifies that the vector should be widened with zeroes.
4411 74 : SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
4412 : bool FillWithZeroes) {
4413 : // Note that InOp might have been widened so it might already have
4414 : // the right width or it might need be narrowed.
4415 74 : EVT InVT = InOp.getValueType();
4416 : assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
4417 : "input and widen element type must match");
4418 : SDLoc dl(InOp);
4419 :
4420 : // Check if InOp already has the right width.
4421 74 : if (InVT == NVT)
4422 0 : return InOp;
4423 :
4424 74 : unsigned InNumElts = InVT.getVectorNumElements();
4425 74 : unsigned WidenNumElts = NVT.getVectorNumElements();
4426 74 : if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4427 70 : unsigned NumConcat = WidenNumElts / InNumElts;
4428 70 : SmallVector<SDValue, 16> Ops(NumConcat);
4429 37 : SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
4430 70 : DAG.getUNDEF(InVT);
4431 70 : Ops[0] = InOp;
4432 140 : for (unsigned i = 1; i != NumConcat; ++i)
4433 140 : Ops[i] = FillVal;
4434 :
4435 140 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
4436 : }
4437 :
4438 4 : if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4439 0 : return DAG.getNode(
4440 : ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
4441 0 : DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4442 :
4443 : // Fall back to extract and build.
4444 4 : SmallVector<SDValue, 16> Ops(WidenNumElts);
4445 4 : EVT EltVT = NVT.getVectorElementType();
4446 4 : unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4447 : unsigned Idx;
4448 16 : for (Idx = 0; Idx < MinNumElts; ++Idx)
4449 12 : Ops[Idx] = DAG.getNode(
4450 : ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4451 12 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4452 :
4453 4 : SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
4454 4 : DAG.getUNDEF(EltVT);
4455 8 : for ( ; Idx < WidenNumElts; ++Idx)
4456 8 : Ops[Idx] = FillVal;
4457 8 : return DAG.getBuildVector(NVT, dl, Ops);
4458 : }
|
