LLVM 17.0.0git
LegalizeTypesGeneric.cpp
Go to the documentation of this file.
1//===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements generic type expansion and splitting for LegalizeTypes.
10// The routines here perform legalization when the details of the type (such as
11// whether it is an integer or a float) do not matter.
12// Expansion is the act of changing a computation in an illegal type to be a
13// computation in two identical registers of a smaller type. The Lo/Hi part
14// is required to be stored first in memory on little/big-endian machines.
15// Splitting is the act of changing a computation in an illegal type to be a
16// computation in two not necessarily identical registers of a smaller type.
17// There are no requirements on how the type is represented in memory.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/IR/DataLayout.h"
23using namespace llvm;
24
25#define DEBUG_TYPE "legalize-types"
26
27//===----------------------------------------------------------------------===//
28// Generic Result Expansion.
29//===----------------------------------------------------------------------===//
30
31// These routines assume that the Lo/Hi part is stored first in memory on
32// little/big-endian machines, followed by the Hi/Lo part. This means that
33// they cannot be used as is on vectors, for which Lo is always stored first.
34void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
35 SDValue &Lo, SDValue &Hi) {
36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
37 GetExpandedOp(Op, Lo, Hi);
38}
39
40void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
41 EVT OutVT = N->getValueType(0);
42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
43 SDValue InOp = N->getOperand(0);
44 EVT InVT = InOp.getValueType();
45 SDLoc dl(N);
46
47 // Handle some special cases efficiently.
48 switch (getTypeAction(InVT)) {
51 break;
54 llvm_unreachable("Bitcast of a promotion-needing float should never need"
55 "expansion");
57 SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
58 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
59 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
60 return;
63 auto &DL = DAG.getDataLayout();
64 // Convert the expanded pieces of the input.
65 GetExpandedOp(InOp, Lo, Hi);
66 if (TLI.hasBigEndianPartOrdering(InVT, DL) !=
67 TLI.hasBigEndianPartOrdering(OutVT, DL))
68 std::swap(Lo, Hi);
69 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
70 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
71 return;
72 }
74 GetSplitVector(InOp, Lo, Hi);
75 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
76 std::swap(Lo, Hi);
77 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
78 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
79 return;
81 // Convert the element instead.
82 SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
83 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
84 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
85 return;
87 report_fatal_error("Scalarization of scalable vectors is not supported.");
89 assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
90 InOp = GetWidenedVector(InOp);
91 EVT LoVT, HiVT;
92 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
93 std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
94 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
95 std::swap(Lo, Hi);
96 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
97 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
98 return;
99 }
100 }
101
102 if (InVT.isVector() && OutVT.isInteger()) {
103 // Handle cases like i64 = BITCAST v1i64 on x86, where the operand
104 // is legal but the result is not.
105 unsigned NumElems = 2;
106 EVT ElemVT = NOutVT;
107 EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
108
109 // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
110 while (!isTypeLegal(NVT)) {
111 unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
112 // If the element size is smaller than byte, bail.
113 if (NewSizeInBits < 8)
114 break;
115 NumElems *= 2;
116 ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
117 NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
118 }
119
120 if (isTypeLegal(NVT)) {
121 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
122
124 for (unsigned i = 0; i < NumElems; ++i)
125 Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
126 CastInOp, DAG.getVectorIdxConstant(i, dl)));
127
128 // Build Lo, Hi pair by pairing extracted elements if needed.
129 unsigned Slot = 0;
130 for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
131 // Each iteration will BUILD_PAIR two nodes and append the result until
132 // there are only two nodes left, i.e. Lo and Hi.
133 SDValue LHS = Vals[Slot];
134 SDValue RHS = Vals[Slot + 1];
135
136 if (DAG.getDataLayout().isBigEndian())
137 std::swap(LHS, RHS);
138
139 Vals.push_back(DAG.getNode(
140 ISD::BUILD_PAIR, dl,
141 EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1),
142 LHS, RHS));
143 }
144 Lo = Vals[Slot++];
145 Hi = Vals[Slot++];
146
147 if (DAG.getDataLayout().isBigEndian())
148 std::swap(Lo, Hi);
149
150 return;
151 }
152 }
153
154 // Lower the bit-convert to a store/load from the stack.
155 assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
156
157 // Create the stack frame object. Make sure it is aligned for both
158 // the source and expanded destination types.
159
160 // In cases where the vector is illegal it will be broken down into parts
161 // and stored in parts - we should use the alignment for the smallest part.
162 Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false);
163 Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false);
164 Align Align = std::max(InAlign, NOutAlign);
166 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
167 MachinePointerInfo PtrInfo =
169
170 // Emit a store to the stack slot.
171 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
172
173 // Load the first half from the stack slot.
174 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign);
175
176 // Increment the pointer to the other half.
177 unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
178 StackPtr =
179 DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(IncrementSize), dl);
180
181 // Load the second half from the stack slot.
182 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
183 PtrInfo.getWithOffset(IncrementSize), NOutAlign);
184
185 // Handle endianness of the load.
186 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
187 std::swap(Lo, Hi);
188}
189
190void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,
191 SDValue &Hi) {
192 // Return the operands.
193 Lo = N->getOperand(0);
194 Hi = N->getOperand(1);
195}
196
197void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,
198 SDValue &Hi) {
199 GetExpandedOp(N->getOperand(0), Lo, Hi);
200 SDValue Part = N->getConstantOperandVal(1) ? Hi : Lo;
201
202 assert(Part.getValueType() == N->getValueType(0) &&
203 "Type twice as big as expanded type not itself expanded!");
204
205 GetPairElements(Part, Lo, Hi);
206}
207
208void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
209 SDValue &Hi) {
210 SDValue OldVec = N->getOperand(0);
211 ElementCount OldEltCount = OldVec.getValueType().getVectorElementCount();
212 EVT OldEltVT = OldVec.getValueType().getVectorElementType();
213 SDLoc dl(N);
214
215 // Convert to a vector of the expanded element type, for example
216 // <3 x i64> -> <6 x i32>.
217 EVT OldVT = N->getValueType(0);
218 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
219
220 if (OldVT != OldEltVT) {
221 // The result of EXTRACT_VECTOR_ELT may be larger than the element type of
222 // the input vector. If so, extend the elements of the input vector to the
223 // same bitwidth as the result before expanding.
224 assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!");
225 EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldEltCount);
226 OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0));
227 }
228
229 SDValue NewVec = DAG.getNode(
230 ISD::BITCAST, dl,
231 EVT::getVectorVT(*DAG.getContext(), NewVT, OldEltCount * 2), OldVec);
232
233 // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
234 SDValue Idx = N->getOperand(1);
235
236 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
237 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
238
239 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
240 DAG.getConstant(1, dl, Idx.getValueType()));
241 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
242
243 if (DAG.getDataLayout().isBigEndian())
244 std::swap(Lo, Hi);
245}
246
247void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
248 SDValue &Hi) {
249 assert(ISD::isNormalLoad(N) && "This routine only for normal loads!");
250 SDLoc dl(N);
251
252 LoadSDNode *LD = cast<LoadSDNode>(N);
253 assert(!LD->isAtomic() && "Atomics can not be split");
254 EVT ValueVT = LD->getValueType(0);
255 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
256 SDValue Chain = LD->getChain();
257 SDValue Ptr = LD->getBasePtr();
258 AAMDNodes AAInfo = LD->getAAInfo();
259
260 assert(NVT.isByteSized() && "Expanded type not byte sized!");
261
262 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(),
263 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
264 AAInfo);
265
266 // Increment the pointer to the other half.
267 unsigned IncrementSize = NVT.getSizeInBits() / 8;
268 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
269 Hi = DAG.getLoad(
270 NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize),
271 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo);
272
273 // Build a factor node to remember that this load is independent of the
274 // other one.
275 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
276 Hi.getValue(1));
277
278 // Handle endianness of the load.
279 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
280 std::swap(Lo, Hi);
281
282 // Modified the chain - switch anything that used the old chain to use
283 // the new one.
284 ReplaceValueWith(SDValue(N, 1), Chain);
285}
286
287void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
288 EVT OVT = N->getValueType(0);
289 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
290 SDValue Chain = N->getOperand(0);
291 SDValue Ptr = N->getOperand(1);
292 SDLoc dl(N);
293 const unsigned Align = N->getConstantOperandVal(3);
294
295 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);
296 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
297 Chain = Hi.getValue(1);
298
299 // Handle endianness of the load.
300 if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
301 std::swap(Lo, Hi);
302
303 // Modified the chain - switch anything that used the old chain to use
304 // the new one.
305 ReplaceValueWith(SDValue(N, 1), Chain);
306}
307
308
309//===--------------------------------------------------------------------===//
310// Generic Operand Expansion.
311//===--------------------------------------------------------------------===//
312
313void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
315 EVT EltVT) {
316 assert(Op.getValueType().isInteger());
317 SDLoc DL(Op);
318 SDValue Parts[2];
319
320 if (NumElements > 1) {
321 NumElements >>= 1;
322 SplitInteger(Op, Parts[0], Parts[1]);
323 if (DAG.getDataLayout().isBigEndian())
324 std::swap(Parts[0], Parts[1]);
325 IntegerToVector(Parts[0], NumElements, Ops, EltVT);
326 IntegerToVector(Parts[1], NumElements, Ops, EltVT);
327 } else {
328 Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));
329 }
330}
331
332SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {
333 SDLoc dl(N);
334 if (N->getValueType(0).isVector() &&
335 N->getOperand(0).getValueType().isInteger()) {
336 // An illegal expanding type is being converted to a legal vector type.
337 // Make a two element vector out of the expanded parts and convert that
338 // instead, but only if the new vector type is legal (otherwise there
339 // is no point, and it might create expansion loops). For example, on
340 // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32.
341 //
342 // FIXME: I'm not sure why we are first trying to split the input into
343 // a 2 element vector, so I'm leaving it here to maintain the current
344 // behavior.
345 unsigned NumElts = 2;
346 EVT OVT = N->getOperand(0).getValueType();
347 EVT NVT = EVT::getVectorVT(*DAG.getContext(),
348 TLI.getTypeToTransformTo(*DAG.getContext(), OVT),
349 NumElts);
350 if (!isTypeLegal(NVT)) {
351 // If we can't find a legal type by splitting the integer in half,
352 // then we can use the node's value type.
353 NumElts = N->getValueType(0).getVectorNumElements();
354 NVT = N->getValueType(0);
355 }
356
358 IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType());
359
360 SDValue Vec = DAG.getBuildVector(NVT, dl, ArrayRef(Ops.data(), NumElts));
361 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);
362 }
363
364 // Otherwise, store to a temporary and load out again as the new type.
365 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
366}
367
368SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
369 // The vector type is legal but the element type needs expansion.
370 EVT VecVT = N->getValueType(0);
371 unsigned NumElts = VecVT.getVectorNumElements();
372 EVT OldVT = N->getOperand(0).getValueType();
373 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
374 SDLoc dl(N);
375
376 assert(OldVT == VecVT.getVectorElementType() &&
377 "BUILD_VECTOR operand type doesn't match vector element type!");
378
379 // Build a vector of twice the length out of the expanded elements.
380 // For example <3 x i64> -> <6 x i32>.
382 NewElts.reserve(NumElts*2);
383
384 for (unsigned i = 0; i < NumElts; ++i) {
385 SDValue Lo, Hi;
386 GetExpandedOp(N->getOperand(i), Lo, Hi);
387 if (DAG.getDataLayout().isBigEndian())
388 std::swap(Lo, Hi);
389 NewElts.push_back(Lo);
390 NewElts.push_back(Hi);
391 }
392
393 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size());
394 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
395
396 // Convert the new vector to the old vector type.
397 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
398}
399
400SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {
401 SDValue Lo, Hi;
402 GetExpandedOp(N->getOperand(0), Lo, Hi);
403 return N->getConstantOperandVal(1) ? Hi : Lo;
404}
405
406SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
407 // The vector type is legal but the element type needs expansion.
408 EVT VecVT = N->getValueType(0);
409 unsigned NumElts = VecVT.getVectorNumElements();
410 SDLoc dl(N);
411
412 SDValue Val = N->getOperand(1);
413 EVT OldEVT = Val.getValueType();
414 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);
415
416 assert(OldEVT == VecVT.getVectorElementType() &&
417 "Inserted element type doesn't match vector element type!");
418
419 // Bitconvert to a vector of twice the length with elements of the expanded
420 // type, insert the expanded vector elements, and then convert back.
421 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);
422 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
423 NewVecVT, N->getOperand(0));
424
425 SDValue Lo, Hi;
426 GetExpandedOp(Val, Lo, Hi);
427 if (DAG.getDataLayout().isBigEndian())
428 std::swap(Lo, Hi);
429
430 SDValue Idx = N->getOperand(2);
431 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
432 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
433 Idx = DAG.getNode(ISD::ADD, dl,
434 Idx.getValueType(), Idx,
435 DAG.getConstant(1, dl, Idx.getValueType()));
436 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
437
438 // Convert the new vector to the old vector type.
439 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
440}
441
442SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
443 SDLoc dl(N);
444 EVT VT = N->getValueType(0);
445 assert(VT.getVectorElementType() == N->getOperand(0).getValueType() &&
446 "SCALAR_TO_VECTOR operand type doesn't match vector element type!");
447 unsigned NumElts = VT.getVectorNumElements();
448 SmallVector<SDValue, 16> Ops(NumElts);
449 Ops[0] = N->getOperand(0);
450 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
451 for (unsigned i = 1; i < NumElts; ++i)
452 Ops[i] = UndefVal;
453 return DAG.getBuildVector(VT, dl, Ops);
454}
455
456SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
457 assert(ISD::isNormalStore(N) && "This routine only for normal stores!");
458 assert(OpNo == 1 && "Can only expand the stored value so far");
459 SDLoc dl(N);
460
461 StoreSDNode *St = cast<StoreSDNode>(N);
462 assert(!St->isAtomic() && "Atomics can not be split");
463 EVT ValueVT = St->getValue().getValueType();
464 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
465 SDValue Chain = St->getChain();
466 SDValue Ptr = St->getBasePtr();
467 AAMDNodes AAInfo = St->getAAInfo();
468
469 assert(NVT.isByteSized() && "Expanded type not byte sized!");
470 unsigned IncrementSize = NVT.getSizeInBits() / 8;
471
472 SDValue Lo, Hi;
473 GetExpandedOp(St->getValue(), Lo, Hi);
474
475 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
476 std::swap(Lo, Hi);
477
478 Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(),
480 AAInfo);
481
482 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
483 Hi = DAG.getStore(
484 Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize),
485 St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo);
486
487 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
488}
489
490
491//===--------------------------------------------------------------------===//
492// Generic Result Splitting.
493//===--------------------------------------------------------------------===//
494
495// Be careful to make no assumptions about which of Lo/Hi is stored first in
496// memory (for vectors it is always Lo first followed by Hi in the following
497// bytes; for integers and floats it is Lo first if and only if the machine is
498// little-endian).
499
500void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
501 SDValue &Lo, SDValue &Hi) {
502 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
503 GetSplitOp(Op, Lo, Hi);
504}
505
506void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
507 SDValue LL, LH, RL, RH, CL, CH;
508 SDLoc dl(N);
509 unsigned Opcode = N->getOpcode();
510 GetSplitOp(N->getOperand(1), LL, LH);
511 GetSplitOp(N->getOperand(2), RL, RH);
512
513 SDValue Cond = N->getOperand(0);
514 CL = CH = Cond;
515 if (Cond.getValueType().isVector()) {
516 if (SDValue Res = WidenVSELECTMask(N))
517 std::tie(CL, CH) = DAG.SplitVector(Res, dl);
518 // Check if there are already splitted versions of the vector available and
519 // use those instead of splitting the mask operand again.
520 else if (getTypeAction(Cond.getValueType()) ==
522 GetSplitVector(Cond, CL, CH);
523 // It seems to improve code to generate two narrow SETCCs as opposed to
524 // splitting a wide result vector.
525 else if (Cond.getOpcode() == ISD::SETCC) {
526 // If the condition is a vXi1 vector, and the LHS of the setcc is a legal
527 // type and the setcc result type is the same vXi1, then leave the setcc
528 // alone.
529 EVT CondLHSVT = Cond.getOperand(0).getValueType();
530 if (Cond.getValueType().getVectorElementType() == MVT::i1 &&
531 isTypeLegal(CondLHSVT) &&
532 getSetCCResultType(CondLHSVT) == Cond.getValueType())
533 std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
534 else
535 SplitVecRes_SETCC(Cond.getNode(), CL, CH);
536 } else
537 std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
538 }
539
540 if (Opcode != ISD::VP_SELECT && Opcode != ISD::VP_MERGE) {
541 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL);
542 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH);
543 return;
544 }
545
546 SDValue EVLLo, EVLHi;
547 std::tie(EVLLo, EVLHi) =
548 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
549
550 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL, EVLLo);
551 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);
552}
553
554void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
555 SDValue &Hi) {
556 SDValue LL, LH, RL, RH;
557 SDLoc dl(N);
558 GetSplitOp(N->getOperand(2), LL, LH);
559 GetSplitOp(N->getOperand(3), RL, RH);
560
561 Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0),
562 N->getOperand(1), LL, RL, N->getOperand(4));
563 Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0),
564 N->getOperand(1), LH, RH, N->getOperand(4));
565}
566
567void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
568 EVT LoVT, HiVT;
569 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
570 Lo = DAG.getUNDEF(LoVT);
571 Hi = DAG.getUNDEF(HiVT);
572}
573
574void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) {
575 SDValue L, H;
576 SDLoc dl(N);
577 GetSplitOp(N->getOperand(0), L, H);
578
579 Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L);
580 Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H);
581}
582
583void DAGTypeLegalizer::SplitRes_ARITH_FENCE(SDNode *N, SDValue &Lo,
584 SDValue &Hi) {
585 SDValue L, H;
586 SDLoc DL(N);
587 GetSplitOp(N->getOperand(0), L, H);
588
589 Lo = DAG.getNode(ISD::ARITH_FENCE, DL, L.getValueType(), L);
590 Hi = DAG.getNode(ISD::ARITH_FENCE, DL, H.getValueType(), H);
591}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
SmallVector< MachineOperand, 4 > Cond
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define H(x, y, z)
Definition: MD5.cpp:57
#define CH(x, y, z)
Definition: SHA256.cpp:34
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
bool isBigEndian() const
Definition: DataLayout.h:246
This class is used to represent ISD::LOAD nodes.
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isAtomic() const
Return true if the memory operation ordering is Unordered or higher.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
EVT getValueType() const
Return the ValueType of the referenced return value.
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:824
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:468
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:465
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:481
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:550
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void reserve(size_type N)
Definition: SmallVector.h:667
void push_back(const T &Elt)
Definition: SmallVector.h:416
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:289
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
static constexpr TypeSize Fixed(ScalarTy ExactSize)
Definition: TypeSize.h:331
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:885
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1149
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:523
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:651
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:362
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:272
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:322
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:215
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:154
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.