Line data Source code
1 : //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements integer type expansion and promotion for LegalizeTypes.
11 : // Promotion is the act of changing a computation in an illegal type into a
12 : // computation in a larger type. For example, implementing i8 arithmetic in an
13 : // i32 register (often needed on powerpc).
14 : // Expansion is the act of changing a computation in an illegal type into a
15 : // computation in two identical registers of a smaller type. For example,
16 : // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 : // targets).
18 : //
19 : //===----------------------------------------------------------------------===//
20 :
21 : #include "LegalizeTypes.h"
22 : #include "llvm/IR/DerivedTypes.h"
23 : #include "llvm/Support/ErrorHandling.h"
24 : #include "llvm/Support/KnownBits.h"
25 : #include "llvm/Support/raw_ostream.h"
26 : using namespace llvm;
27 :
28 : #define DEBUG_TYPE "legalize-types"
29 :
30 : //===----------------------------------------------------------------------===//
31 : // Integer Result Promotion
32 : //===----------------------------------------------------------------------===//
33 :
34 : /// PromoteIntegerResult - This method is called when a result of a node is
35 : /// found to be in need of promotion to a larger type. At this point, the node
36 : /// may also have invalid operands or may have other results that need
37 : /// expansion, we just know that (at least) one result needs promotion.
38 239437 : void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39 : LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
40 : dbgs() << "\n");
41 239437 : SDValue Res = SDValue();
42 :
43 : // See if the target wants to custom expand this node.
44 478874 : if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
45 : LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
46 465 : return;
47 : }
48 :
49 477944 : switch (N->getOpcode()) {
50 0 : default:
51 : #ifndef NDEBUG
52 : dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
53 : N->dump(&DAG); dbgs() << "\n";
54 : #endif
55 0 : llvm_unreachable("Do not know how to promote this operator!");
56 0 : case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
57 63 : case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
58 63 : case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
59 5173 : case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
60 18 : case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
61 23 : case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
62 0 : case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
63 26226 : case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64 8 : case ISD::CTLZ_ZERO_UNDEF:
65 8 : case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
66 80 : case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67 35 : case ISD::CTTZ_ZERO_UNDEF:
68 35 : case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69 9330 : case ISD::EXTRACT_VECTOR_ELT:
70 9330 : Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
71 6699 : case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
72 4 : case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
73 4 : break;
74 0 : case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
75 0 : break;
76 831 : case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
77 11 : case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
78 0 : case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
79 97660 : case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
80 102 : case ISD::SMIN:
81 102 : case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
82 56 : case ISD::UMIN:
83 56 : case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
84 :
85 1132 : case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86 70 : case ISD::SIGN_EXTEND_INREG:
87 70 : Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
88 161 : case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
89 5593 : case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
90 48258 : case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
91 2497 : case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
92 37 : case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
93 :
94 2842 : case ISD::EXTRACT_SUBVECTOR:
95 2842 : Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96 108 : case ISD::VECTOR_SHUFFLE:
97 108 : Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98 213 : case ISD::INSERT_VECTOR_ELT:
99 213 : Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
100 1249 : case ISD::BUILD_VECTOR:
101 1249 : Res = PromoteIntRes_BUILD_VECTOR(N); break;
102 27 : case ISD::SCALAR_TO_VECTOR:
103 27 : Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
104 189 : case ISD::CONCAT_VECTORS:
105 189 : Res = PromoteIntRes_CONCAT_VECTORS(N); break;
106 :
107 3 : case ISD::ANY_EXTEND_VECTOR_INREG:
108 : case ISD::SIGN_EXTEND_VECTOR_INREG:
109 : case ISD::ZERO_EXTEND_VECTOR_INREG:
110 3 : Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
111 :
112 2789 : case ISD::SIGN_EXTEND:
113 : case ISD::ZERO_EXTEND:
114 2789 : case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
115 :
116 179 : case ISD::FP_TO_SINT:
117 179 : case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
118 :
119 1077 : case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
120 :
121 17164 : case ISD::AND:
122 : case ISD::OR:
123 : case ISD::XOR:
124 : case ISD::ADD:
125 : case ISD::SUB:
126 17164 : case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
127 :
128 316 : case ISD::SDIV:
129 316 : case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
130 :
131 280 : case ISD::UDIV:
132 280 : case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
133 :
134 575 : case ISD::SADDO:
135 575 : case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
136 544 : case ISD::UADDO:
137 544 : case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
138 144 : case ISD::SMULO:
139 144 : case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
140 :
141 45 : case ISD::ADDCARRY:
142 45 : case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
143 :
144 2 : case ISD::SADDSAT: Res = PromoteIntRes_SADDSAT(N); break;
145 :
146 : case ISD::ATOMIC_LOAD:
147 134 : Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
148 :
149 : case ISD::ATOMIC_LOAD_ADD:
150 : case ISD::ATOMIC_LOAD_SUB:
151 : case ISD::ATOMIC_LOAD_AND:
152 : case ISD::ATOMIC_LOAD_CLR:
153 : case ISD::ATOMIC_LOAD_OR:
154 : case ISD::ATOMIC_LOAD_XOR:
155 : case ISD::ATOMIC_LOAD_NAND:
156 : case ISD::ATOMIC_LOAD_MIN:
157 : case ISD::ATOMIC_LOAD_MAX:
158 : case ISD::ATOMIC_LOAD_UMIN:
159 : case ISD::ATOMIC_LOAD_UMAX:
160 : case ISD::ATOMIC_SWAP:
161 1207 : Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
162 :
163 : case ISD::ATOMIC_CMP_SWAP:
164 : case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
165 5755 : Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
166 5755 : break;
167 : }
168 :
169 : // If the result is null then the sub-method took care of registering it.
170 238972 : if (Res.getNode())
171 238972 : SetPromotedInteger(SDValue(N, ResNo), Res);
172 : }
173 :
174 0 : SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
175 : unsigned ResNo) {
176 0 : SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
177 0 : return GetPromotedInteger(Op);
178 : }
179 :
180 63 : SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
181 : // Sign-extend the new bits, and continue the assertion.
182 126 : SDValue Op = SExtPromotedInteger(N->getOperand(0));
183 63 : return DAG.getNode(ISD::AssertSext, SDLoc(N),
184 126 : Op.getValueType(), Op, N->getOperand(1));
185 : }
186 :
187 63 : SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
188 : // Zero the new bits, and continue the assertion.
189 126 : SDValue Op = ZExtPromotedInteger(N->getOperand(0));
190 63 : return DAG.getNode(ISD::AssertZext, SDLoc(N),
191 126 : Op.getValueType(), Op, N->getOperand(1));
192 : }
193 :
194 134 : SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
195 268 : EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
196 134 : SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
197 : N->getMemoryVT(), ResVT,
198 : N->getChain(), N->getBasePtr(),
199 134 : N->getMemOperand());
200 : // Legalize the chain result - switch anything that used the old chain to
201 : // use the new one.
202 134 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
203 134 : return Res;
204 : }
205 :
206 1207 : SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
207 2414 : SDValue Op2 = GetPromotedInteger(N->getOperand(2));
208 1207 : SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
209 : N->getMemoryVT(),
210 : N->getChain(), N->getBasePtr(),
211 1215 : Op2, N->getMemOperand());
212 : // Legalize the chain result - switch anything that used the old chain to
213 : // use the new one.
214 1207 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
215 1207 : return Res;
216 : }
217 :
218 5755 : SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
219 : unsigned ResNo) {
220 5755 : if (ResNo == 1) {
221 : assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
222 11096 : EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
223 11096 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
224 :
225 : // Only use the result of getSetCCResultType if it is legal,
226 : // otherwise just use the promoted result type (NVT).
227 5548 : if (!TLI.isTypeLegal(SVT))
228 0 : SVT = NVT;
229 :
230 16644 : SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
231 5548 : SDValue Res = DAG.getAtomicCmpSwap(
232 5548 : ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
233 5548 : N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
234 5585 : N->getMemOperand());
235 5548 : ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
236 5548 : ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
237 : return Res.getValue(1);
238 : }
239 :
240 414 : SDValue Op2 = GetPromotedInteger(N->getOperand(2));
241 414 : SDValue Op3 = GetPromotedInteger(N->getOperand(3));
242 : SDVTList VTs =
243 828 : DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
244 207 : SDValue Res = DAG.getAtomicCmpSwap(
245 207 : N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
246 207 : N->getBasePtr(), Op2, Op3, N->getMemOperand());
247 : // Update the use to N with the newly created Res.
248 621 : for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
249 414 : ReplaceValueWith(SDValue(N, i), Res.getValue(i));
250 207 : return Res;
251 : }
252 :
253 5173 : SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
254 5173 : SDValue InOp = N->getOperand(0);
255 : EVT InVT = InOp.getValueType();
256 5173 : EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
257 5173 : EVT OutVT = N->getValueType(0);
258 5173 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
259 : SDLoc dl(N);
260 :
261 5173 : switch (getTypeAction(InVT)) {
262 : case TargetLowering::TypeLegal:
263 : break;
264 491 : case TargetLowering::TypePromoteInteger:
265 556 : if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
266 : // The input promotes to the same size. Convert the promoted value.
267 0 : return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
268 : break;
269 0 : case TargetLowering::TypeSoftenFloat:
270 : // Promote the integer operand by hand.
271 0 : return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
272 : case TargetLowering::TypePromoteFloat: {
273 : // Convert the promoted float by hand.
274 695 : if (!NOutVT.isVector())
275 1384 : return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
276 : break;
277 : }
278 : case TargetLowering::TypeExpandInteger:
279 : case TargetLowering::TypeExpandFloat:
280 : break;
281 : case TargetLowering::TypeScalarizeVector:
282 : // Convert the element to an integer and promote it by hand.
283 2313 : if (!NOutVT.isVector())
284 2276 : return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
285 2276 : BitConvertToInteger(GetScalarizedVector(InOp)));
286 : break;
287 247 : case TargetLowering::TypeSplitVector: {
288 : // For example, i32 = BITCAST v2i16 on alpha. Convert the split
289 : // pieces of the input into integers and reassemble in the final type.
290 247 : SDValue Lo, Hi;
291 494 : GetSplitVector(N->getOperand(0), Lo, Hi);
292 247 : Lo = BitConvertToInteger(Lo);
293 247 : Hi = BitConvertToInteger(Hi);
294 :
295 247 : if (DAG.getDataLayout().isBigEndian())
296 : std::swap(Lo, Hi);
297 :
298 247 : InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
299 247 : EVT::getIntegerVT(*DAG.getContext(),
300 : NOutVT.getSizeInBits()),
301 247 : JoinIntegers(Lo, Hi));
302 494 : return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
303 : }
304 9 : case TargetLowering::TypeWidenVector:
305 : // The input is widened to the same size. Convert to the widened value.
306 : // Make sure that the outgoing value is not a vector, because this would
307 : // make us bitcast between two vectors which are legalized in different ways.
308 18 : if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
309 18 : return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
310 : }
311 :
312 1949 : return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
313 1949 : CreateStackStoreLoad(InOp, OutVT));
314 : }
315 :
316 : // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
317 : // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
318 41 : static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
319 : const TargetLowering &TLI,
320 : SelectionDAG &DAG) {
321 41 : EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
322 : // If the value won't fit in the prefered type, just use something safe. It
323 : // will be legalized when the shift is expanded.
324 41 : if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
325 4 : ShiftVT = MVT::i32;
326 41 : return ShiftVT;
327 : }
328 :
329 23 : SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
330 46 : SDValue Op = GetPromotedInteger(N->getOperand(0));
331 23 : EVT OVT = N->getValueType(0);
332 46 : EVT NVT = Op.getValueType();
333 : SDLoc dl(N);
334 :
335 23 : unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
336 23 : EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
337 23 : return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
338 23 : DAG.getConstant(DiffBits, dl, ShiftVT));
339 : }
340 :
341 18 : SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
342 36 : SDValue Op = GetPromotedInteger(N->getOperand(0));
343 18 : EVT OVT = N->getValueType(0);
344 36 : EVT NVT = Op.getValueType();
345 : SDLoc dl(N);
346 :
347 18 : unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
348 18 : EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
349 18 : return DAG.getNode(ISD::SRL, dl, NVT,
350 18 : DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
351 18 : DAG.getConstant(DiffBits, dl, ShiftVT));
352 : }
353 :
354 0 : SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
355 : // The pair element type may be legal, or may not promote to the same type as
356 : // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
357 0 : return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
358 0 : TLI.getTypeToTransformTo(*DAG.getContext(),
359 : N->getValueType(0)), JoinIntegers(N->getOperand(0),
360 0 : N->getOperand(1)));
361 : }
362 :
363 26226 : SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
364 52452 : EVT VT = N->getValueType(0);
365 : // FIXME there is no actual debug info here
366 : SDLoc dl(N);
367 : // Zero extend things like i1, sign extend everything else. It shouldn't
368 : // matter in theory which one we pick, but this tends to give better code?
369 26226 : unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
370 26226 : SDValue Result = DAG.getNode(Opc, dl,
371 52452 : TLI.getTypeToTransformTo(*DAG.getContext(), VT),
372 26226 : SDValue(N, 0));
373 : assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
374 26226 : return Result;
375 : }
376 :
377 8 : SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
378 : // Zero extend to the promoted type and do the count there.
379 16 : SDValue Op = ZExtPromotedInteger(N->getOperand(0));
380 : SDLoc dl(N);
381 16 : EVT OVT = N->getValueType(0);
382 8 : EVT NVT = Op.getValueType();
383 24 : Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
384 : // Subtract off the extra leading bits in the bigger type.
385 8 : return DAG.getNode(
386 : ISD::SUB, dl, NVT, Op,
387 8 : DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
388 8 : NVT));
389 : }
390 :
391 80 : SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
392 : // Zero extend to the promoted type and do the count there.
393 160 : SDValue Op = ZExtPromotedInteger(N->getOperand(0));
394 160 : return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
395 : }
396 :
397 35 : SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
398 70 : SDValue Op = GetPromotedInteger(N->getOperand(0));
399 70 : EVT OVT = N->getValueType(0);
400 35 : EVT NVT = Op.getValueType();
401 : SDLoc dl(N);
402 35 : if (N->getOpcode() == ISD::CTTZ) {
403 : // The count is the same in the promoted type except if the original
404 : // value was zero. This can be handled by setting the bit just off
405 : // the top of the original type.
406 : auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
407 15 : OVT.getScalarSizeInBits());
408 30 : Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
409 : }
410 105 : return DAG.getNode(N->getOpcode(), dl, NVT, Op);
411 : }
412 :
413 9330 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
414 : SDLoc dl(N);
415 18660 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
416 9330 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
417 18660 : N->getOperand(1));
418 : }
419 :
420 179 : SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
421 358 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
422 179 : unsigned NewOpc = N->getOpcode();
423 : SDLoc dl(N);
424 :
425 : // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
426 : // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
427 : // and SINT conversions are Custom, there is no way to tell which is
428 : // preferable. We choose SINT because that's the right thing on PPC.)
429 179 : if (N->getOpcode() == ISD::FP_TO_UINT &&
430 179 : !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
431 : TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
432 : NewOpc = ISD::FP_TO_SINT;
433 :
434 537 : SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
435 :
436 : // Assert that the converted value fits in the original type. If it doesn't
437 : // (eg: because the value being converted is too big), then the result of the
438 : // original operation was undefined anyway, so the assert is still correct.
439 : //
440 : // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
441 : // before legalization: fp-to-uint16, 65534. -> 0xfffe
442 : // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
443 179 : return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
444 : ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
445 434 : DAG.getValueType(N->getValueType(0).getScalarType()));
446 : }
447 :
448 1077 : SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
449 2154 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
450 : SDLoc dl(N);
451 :
452 4308 : return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
453 : }
454 :
455 2789 : SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
456 5578 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
457 : SDLoc dl(N);
458 :
459 5578 : if (getTypeAction(N->getOperand(0).getValueType())
460 : == TargetLowering::TypePromoteInteger) {
461 4046 : SDValue Res = GetPromotedInteger(N->getOperand(0));
462 : assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
463 :
464 : // If the result and operand types are the same after promotion, simplify
465 : // to an in-register extension.
466 4 : if (NVT == Res.getValueType()) {
467 : // The high bits are not guaranteed to be anything. Insert an extend.
468 4002 : if (N->getOpcode() == ISD::SIGN_EXTEND)
469 558 : return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
470 1116 : DAG.getValueType(N->getOperand(0).getValueType()));
471 1443 : if (N->getOpcode() == ISD::ZERO_EXTEND)
472 1184 : return DAG.getZeroExtendInReg(Res, dl,
473 2368 : N->getOperand(0).getValueType().getScalarType());
474 : assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
475 259 : return Res;
476 : }
477 : }
478 :
479 : // Otherwise, just extend the original operand all the way to the larger type.
480 3152 : return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
481 : }
482 :
483 6699 : SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
484 : assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
485 13398 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
486 : ISD::LoadExtType ExtType =
487 : ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
488 : SDLoc dl(N);
489 6699 : SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
490 13398 : N->getMemoryVT(), N->getMemOperand());
491 :
492 : // Legalize the chain result - switch anything that used the old chain to
493 : // use the new one.
494 6699 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
495 6699 : return Res;
496 : }
497 :
498 4 : SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
499 8 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
500 4 : SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
501 :
502 : SDLoc dl(N);
503 4 : SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
504 : N->getMask(), ExtPassThru, N->getMemoryVT(),
505 8 : N->getMemOperand(), ISD::SEXTLOAD);
506 : // Legalize the chain result - switch anything that used the old chain to
507 : // use the new one.
508 4 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
509 4 : return Res;
510 : }
511 :
512 0 : SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
513 0 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
514 0 : SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
515 : assert(NVT == ExtPassThru.getValueType() &&
516 : "Gather result type and the passThru agrument type should be the same");
517 :
518 : SDLoc dl(N);
519 : SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
520 0 : N->getIndex(), N->getScale() };
521 0 : SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
522 : N->getMemoryVT(), dl, Ops,
523 0 : N->getMemOperand());
524 : // Legalize the chain result - switch anything that used the old chain to
525 : // use the new one.
526 0 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
527 0 : return Res;
528 : }
529 :
530 : /// Promote the overflow flag of an overflowing arithmetic node.
531 1289 : SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
532 : // Simply change the return type of the boolean result.
533 2578 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
534 1289 : EVT ValueVTs[] = { N->getValueType(0), NVT };
535 2578 : SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
536 1289 : unsigned NumOps = N->getNumOperands();
537 : assert(NumOps <= 3 && "Too many operands");
538 1289 : if (NumOps == 3)
539 40 : Ops[2] = N->getOperand(2);
540 :
541 2578 : SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
542 6445 : DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
543 :
544 : // Modified the sum result - switch anything that used the old sum to use
545 : // the new one.
546 1289 : ReplaceValueWith(SDValue(N, 0), Res);
547 :
548 1289 : return SDValue(Res.getNode(), 1);
549 : }
550 :
551 2 : SDValue DAGTypeLegalizer::PromoteIntRes_SADDSAT(SDNode *N) {
552 : // For promoting iN -> iM, this can be expanded by
553 : // 1. ANY_EXTEND iN to iM
554 : // 2. SHL by M-N
555 : // 3. SADDSAT
556 : // 4. ASHR by M-N
557 : SDLoc dl(N);
558 2 : SDValue Op1 = N->getOperand(0);
559 2 : SDValue Op2 = N->getOperand(1);
560 2 : unsigned OldBits = Op1.getValueSizeInBits();
561 :
562 2 : SDValue Op1Promoted = GetPromotedInteger(Op1);
563 2 : SDValue Op2Promoted = GetPromotedInteger(Op2);
564 :
565 2 : EVT PromotedType = Op1Promoted.getValueType();
566 2 : unsigned NewBits = Op1Promoted.getValueSizeInBits();
567 2 : unsigned SHLAmount = NewBits - OldBits;
568 2 : EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
569 2 : SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
570 2 : Op1Promoted =
571 4 : DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
572 2 : Op2Promoted =
573 4 : DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
574 :
575 : SDValue Result =
576 4 : DAG.getNode(ISD::SADDSAT, dl, PromotedType, Op1Promoted, Op2Promoted);
577 4 : return DAG.getNode(ISD::SRA, dl, PromotedType, Result, ShiftAmount);
578 : }
579 :
580 575 : SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
581 575 : if (ResNo == 1)
582 573 : return PromoteIntRes_Overflow(N);
583 :
584 : // The operation overflowed iff the result in the larger type is not the
585 : // sign extension of its truncation to the original type.
586 4 : SDValue LHS = SExtPromotedInteger(N->getOperand(0));
587 4 : SDValue RHS = SExtPromotedInteger(N->getOperand(1));
588 2 : EVT OVT = N->getOperand(0).getValueType();
589 4 : EVT NVT = LHS.getValueType();
590 : SDLoc dl(N);
591 :
592 : // Do the arithmetic in the larger type.
593 2 : unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
594 4 : SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
595 :
596 : // Calculate the overflow flag: sign extend the arithmetic result from
597 : // the original type.
598 2 : SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
599 2 : DAG.getValueType(OVT));
600 : // Overflowed if and only if this is not equal to Res.
601 4 : Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
602 :
603 : // Use the calculated overflow everywhere.
604 2 : ReplaceValueWith(SDValue(N, 1), Ofl);
605 :
606 2 : return Res;
607 : }
608 :
609 831 : SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
610 1662 : SDValue LHS = GetPromotedInteger(N->getOperand(1));
611 1662 : SDValue RHS = GetPromotedInteger(N->getOperand(2));
612 1662 : return DAG.getSelect(SDLoc(N),
613 1663 : LHS.getValueType(), N->getOperand(0), LHS, RHS);
614 : }
615 :
616 11 : SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
617 11 : SDValue Mask = N->getOperand(0);
618 :
619 11 : SDValue LHS = GetPromotedInteger(N->getOperand(1));
620 22 : SDValue RHS = GetPromotedInteger(N->getOperand(2));
621 11 : return DAG.getNode(ISD::VSELECT, SDLoc(N),
622 17 : LHS.getValueType(), Mask, LHS, RHS);
623 : }
624 :
625 0 : SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
626 0 : SDValue LHS = GetPromotedInteger(N->getOperand(2));
627 0 : SDValue RHS = GetPromotedInteger(N->getOperand(3));
628 0 : return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
629 : LHS.getValueType(), N->getOperand(0),
630 0 : N->getOperand(1), LHS, RHS, N->getOperand(4));
631 : }
632 :
633 97660 : SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
634 97660 : EVT InVT = N->getOperand(0).getValueType();
635 195320 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
636 :
637 97660 : EVT SVT = getSetCCResultType(InVT);
638 :
639 : // If we got back a type that needs to be promoted, this likely means the
640 : // the input type also needs to be promoted. So get the promoted type for
641 : // the input and try the query again.
642 97660 : if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
643 132 : if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
644 123 : InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
645 123 : SVT = getSetCCResultType(InVT);
646 : } else {
647 : // Input type isn't promoted, just use the default promoted type.
648 9 : SVT = NVT;
649 : }
650 : }
651 :
652 : SDLoc dl(N);
653 : assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
654 : "Vector compare must return a vector result!");
655 :
656 : // Get the SETCC result using the canonical SETCC type.
657 97660 : SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
658 292980 : N->getOperand(1), N->getOperand(2));
659 :
660 : // Convert to the expected type.
661 97660 : return DAG.getSExtOrTrunc(SetCC, dl, NVT);
662 : }
663 :
664 1132 : SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
665 2264 : SDValue LHS = GetPromotedInteger(N->getOperand(0));
666 1132 : SDValue RHS = N->getOperand(1);
667 1132 : if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
668 618 : RHS = ZExtPromotedInteger(RHS);
669 2266 : return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
670 : }
671 :
672 70 : SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
673 140 : SDValue Op = GetPromotedInteger(N->getOperand(0));
674 70 : return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
675 140 : Op.getValueType(), Op, N->getOperand(1));
676 : }
677 :
678 17164 : SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
679 : // The input may have strange things in the top bits of the registers, but
680 : // these operations don't care. They may have weird bits going out, but
681 : // that too is okay if they are integer operations.
682 34328 : SDValue LHS = GetPromotedInteger(N->getOperand(0));
683 34328 : SDValue RHS = GetPromotedInteger(N->getOperand(1));
684 17164 : return DAG.getNode(N->getOpcode(), SDLoc(N),
685 18262 : LHS.getValueType(), LHS, RHS);
686 : }
687 :
688 418 : SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
689 : // Sign extend the input.
690 836 : SDValue LHS = SExtPromotedInteger(N->getOperand(0));
691 836 : SDValue RHS = SExtPromotedInteger(N->getOperand(1));
692 418 : return DAG.getNode(N->getOpcode(), SDLoc(N),
693 420 : LHS.getValueType(), LHS, RHS);
694 : }
695 :
696 336 : SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
697 : // Zero extend the input.
698 672 : SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
699 672 : SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
700 336 : return DAG.getNode(N->getOpcode(), SDLoc(N),
701 338 : LHS.getValueType(), LHS, RHS);
702 : }
703 :
704 161 : SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
705 : // The input value must be properly sign extended.
706 322 : SDValue LHS = SExtPromotedInteger(N->getOperand(0));
707 161 : SDValue RHS = N->getOperand(1);
708 161 : if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
709 94 : RHS = ZExtPromotedInteger(RHS);
710 322 : return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
711 : }
712 :
713 5593 : SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
714 : // The input value must be properly zero extended.
715 11186 : SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
716 5593 : SDValue RHS = N->getOperand(1);
717 5593 : if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
718 1751 : RHS = ZExtPromotedInteger(RHS);
719 11190 : return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
720 : }
721 :
722 48258 : SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
723 96516 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
724 48258 : SDValue Res;
725 48258 : SDValue InOp = N->getOperand(0);
726 : SDLoc dl(N);
727 :
728 48258 : switch (getTypeAction(InOp.getValueType())) {
729 0 : default: llvm_unreachable("Unknown type action!");
730 43411 : case TargetLowering::TypeLegal:
731 : case TargetLowering::TypeExpandInteger:
732 43411 : Res = InOp;
733 43411 : break;
734 4826 : case TargetLowering::TypePromoteInteger:
735 4826 : Res = GetPromotedInteger(InOp);
736 4826 : break;
737 20 : case TargetLowering::TypeSplitVector: {
738 20 : EVT InVT = InOp.getValueType();
739 : assert(InVT.isVector() && "Cannot split scalar types");
740 : unsigned NumElts = InVT.getVectorNumElements();
741 : assert(NumElts == NVT.getVectorNumElements() &&
742 : "Dst and Src must have the same number of elements");
743 : assert(isPowerOf2_32(NumElts) &&
744 : "Promoted vector type must be a power of two");
745 :
746 20 : SDValue EOp1, EOp2;
747 20 : GetSplitVector(InOp, EOp1, EOp2);
748 :
749 20 : EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
750 20 : NumElts/2);
751 40 : EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
752 40 : EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
753 :
754 40 : return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
755 : }
756 1 : case TargetLowering::TypeWidenVector: {
757 1 : SDValue WideInOp = GetWidenedVector(InOp);
758 :
759 : // Truncate widened InOp.
760 2 : unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
761 1 : EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
762 2 : N->getValueType(0).getScalarType(), NumElem);
763 2 : SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
764 :
765 : // Zero extend so that the elements are of same type as those of NVT
766 1 : EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
767 1 : NumElem);
768 2 : SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
769 :
770 : // Extract the low NVT subvector.
771 1 : MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
772 2 : SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
773 2 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
774 : }
775 : }
776 :
777 : // Truncate to NVT instead of VT
778 96474 : return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
779 : }
780 :
781 544 : SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
782 544 : if (ResNo == 1)
783 534 : return PromoteIntRes_Overflow(N);
784 :
785 : // The operation overflowed iff the result in the larger type is not the
786 : // zero extension of its truncation to the original type.
787 20 : SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
788 20 : SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
789 10 : EVT OVT = N->getOperand(0).getValueType();
790 20 : EVT NVT = LHS.getValueType();
791 : SDLoc dl(N);
792 :
793 : // Do the arithmetic in the larger type.
794 10 : unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
795 20 : SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
796 :
797 : // Calculate the overflow flag: zero extend the arithmetic result from
798 : // the original type.
799 10 : SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
800 : // Overflowed if and only if this is not equal to Res.
801 20 : Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
802 :
803 : // Use the calculated overflow everywhere.
804 10 : ReplaceValueWith(SDValue(N, 1), Ofl);
805 :
806 10 : return Res;
807 : }
808 :
809 45 : SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
810 45 : if (ResNo == 1)
811 40 : return PromoteIntRes_Overflow(N);
812 :
813 : // We need to sign-extend the operands so the carry value computed by the
814 : // wide operation will be equivalent to the carry value computed by the
815 : // narrow operation.
816 : // An ADDCARRY can generate carry only if any of the operands has its
817 : // most significant bit set. Sign extension propagates the most significant
818 : // bit into the higher bits which means the extra bit that the narrow
819 : // addition would need (i.e. the carry) will be propagated through the higher
820 : // bits of the wide addition.
821 : // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
822 : // preserved by sign extension.
823 10 : SDValue LHS = SExtPromotedInteger(N->getOperand(0));
824 10 : SDValue RHS = SExtPromotedInteger(N->getOperand(1));
825 :
826 10 : EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
827 :
828 : // Do the arithmetic in the wide type.
829 10 : SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
830 20 : LHS, RHS, N->getOperand(2));
831 :
832 : // Update the users of the original carry/borrow value.
833 5 : ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
834 :
835 5 : return SDValue(Res.getNode(), 0);
836 : }
837 :
838 144 : SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
839 : // Promote the overflow bit trivially.
840 144 : if (ResNo == 1)
841 142 : return PromoteIntRes_Overflow(N);
842 :
843 2 : SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
844 : SDLoc DL(N);
845 2 : EVT SmallVT = LHS.getValueType();
846 :
847 : // To determine if the result overflowed in a larger type, we extend the
848 : // input to the larger type, do the multiply (checking if it overflows),
849 : // then also check the high bits of the result to see if overflow happened
850 : // there.
851 2 : if (N->getOpcode() == ISD::SMULO) {
852 1 : LHS = SExtPromotedInteger(LHS);
853 1 : RHS = SExtPromotedInteger(RHS);
854 : } else {
855 1 : LHS = ZExtPromotedInteger(LHS);
856 1 : RHS = ZExtPromotedInteger(RHS);
857 : }
858 4 : SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
859 4 : SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
860 :
861 : // Overflow occurred if it occurred in the larger type, or if the high part
862 : // of the result does not zero/sign-extend the low part. Check this second
863 : // possibility first.
864 2 : SDValue Overflow;
865 2 : if (N->getOpcode() == ISD::UMULO) {
866 : // Unsigned overflow occurred if the high part is non-zero.
867 1 : SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
868 1 : DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
869 1 : DL));
870 1 : Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
871 : DAG.getConstant(0, DL, Hi.getValueType()),
872 2 : ISD::SETNE);
873 : } else {
874 : // Signed overflow occurred if the high part does not sign extend the low.
875 1 : SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
876 1 : Mul, DAG.getValueType(SmallVT));
877 2 : Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
878 : }
879 :
880 : // The only other way for overflow to occur is if the multiplication in the
881 : // larger type itself overflowed.
882 2 : Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
883 4 : SDValue(Mul.getNode(), 1));
884 :
885 : // Use the calculated overflow everywhere.
886 2 : ReplaceValueWith(SDValue(N, 1), Overflow);
887 2 : return Mul;
888 : }
889 :
890 2497 : SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
891 2497 : return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
892 2497 : N->getValueType(0)));
893 : }
894 :
895 37 : SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
896 37 : SDValue Chain = N->getOperand(0); // Get the chain.
897 37 : SDValue Ptr = N->getOperand(1); // Get the pointer.
898 74 : EVT VT = N->getValueType(0);
899 : SDLoc dl(N);
900 :
901 37 : MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
902 37 : unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
903 : // The argument is passed as NumRegs registers of type RegVT.
904 :
905 37 : SmallVector<SDValue, 8> Parts(NumRegs);
906 74 : for (unsigned i = 0; i < NumRegs; ++i) {
907 74 : Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
908 74 : N->getConstantOperandVal(3));
909 37 : Chain = Parts[i].getValue(1);
910 : }
911 :
912 : // Handle endianness of the load.
913 37 : if (DAG.getDataLayout().isBigEndian())
914 : std::reverse(Parts.begin(), Parts.end());
915 :
916 : // Assemble the parts in the promoted type.
917 74 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
918 74 : SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
919 37 : for (unsigned i = 1; i < NumRegs; ++i) {
920 0 : SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
921 : // Shift it to the right position and "or" it in.
922 0 : Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
923 0 : DAG.getConstant(i * RegVT.getSizeInBits(), dl,
924 0 : TLI.getPointerTy(DAG.getDataLayout())));
925 0 : Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
926 : }
927 :
928 : // Modified the chain result - switch anything that used the old chain to
929 : // use the new one.
930 37 : ReplaceValueWith(SDValue(N, 1), Chain);
931 :
932 37 : return Res;
933 : }
934 :
935 : //===----------------------------------------------------------------------===//
936 : // Integer Operand Promotion
937 : //===----------------------------------------------------------------------===//
938 :
939 : /// PromoteIntegerOperand - This method is called when the specified operand of
940 : /// the specified node is found to need promotion. At this point, all of the
941 : /// result types of the node are known to be legal, but other operands of the
942 : /// node may need promotion or expansion as well as the specified one.
943 265317 : bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
944 : LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
945 : dbgs() << "\n");
946 265317 : SDValue Res = SDValue();
947 :
948 795951 : if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
949 : LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
950 : return false;
951 : }
952 :
953 527994 : switch (N->getOpcode()) {
954 0 : default:
955 : #ifndef NDEBUG
956 : dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
957 : N->dump(&DAG); dbgs() << "\n";
958 : #endif
959 0 : llvm_unreachable("Do not know how to promote this operator's operand!");
960 :
961 17188 : case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
962 : case ISD::ATOMIC_STORE:
963 122 : Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
964 122 : break;
965 408 : case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
966 27 : case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
967 79395 : case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
968 0 : case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
969 4137 : case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
970 3055 : case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
971 6375 : case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
972 290 : case ISD::INSERT_VECTOR_ELT:
973 290 : Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
974 39 : case ISD::SCALAR_TO_VECTOR:
975 39 : Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
976 5664 : case ISD::VSELECT:
977 5664 : case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
978 143 : case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
979 3013 : case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
980 8060 : case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
981 133 : case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
982 11681 : case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
983 11681 : OpNo); break;
984 24 : case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
985 24 : OpNo); break;
986 38 : case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
987 38 : OpNo); break;
988 52 : case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
989 52 : OpNo); break;
990 8 : case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
991 8 : OpNo); break;
992 3505 : case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
993 2632 : case ISD::FP16_TO_FP:
994 2632 : case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
995 117968 : case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
996 0 : case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
997 :
998 0 : case ISD::SHL:
999 : case ISD::SRA:
1000 : case ISD::SRL:
1001 : case ISD::ROTL:
1002 0 : case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1003 :
1004 40 : case ISD::ADDCARRY:
1005 40 : case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1006 : }
1007 :
1008 : // If the result is null, the sub-method took care of registering results etc.
1009 263997 : if (!Res.getNode()) return false;
1010 :
1011 : // If the result is N, the sub-method updated N in place. Tell the legalizer
1012 : // core about this.
1013 263997 : if (Res.getNode() == N)
1014 : return true;
1015 :
1016 : assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1017 : "Invalid operand expansion");
1018 :
1019 168954 : ReplaceValueWith(SDValue(N, 0), Res);
1020 168954 : return false;
1021 : }
1022 :
1023 : /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1024 : /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1025 3183 : void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1026 : ISD::CondCode CCCode) {
1027 : // We have to insert explicit sign or zero extends. Note that we could
1028 : // insert sign extends for ALL conditions, but zero extend is cheaper on
1029 : // many machines (an AND instead of two shifts), so prefer it.
1030 : switch (CCCode) {
1031 0 : default: llvm_unreachable("Unknown integer comparison!");
1032 2044 : case ISD::SETEQ:
1033 : case ISD::SETNE: {
1034 2044 : SDValue OpL = GetPromotedInteger(NewLHS);
1035 2044 : SDValue OpR = GetPromotedInteger(NewRHS);
1036 :
1037 : // We would prefer to promote the comparison operand with sign extension.
1038 : // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1039 : // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1040 : // instruction, which is redundant eventually.
1041 : unsigned OpLEffectiveBits =
1042 2044 : OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1043 : unsigned OpREffectiveBits =
1044 2044 : OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1045 2293 : if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1046 249 : OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1047 232 : NewLHS = OpL;
1048 232 : NewRHS = OpR;
1049 : } else {
1050 1812 : NewLHS = ZExtPromotedInteger(NewLHS);
1051 1812 : NewRHS = ZExtPromotedInteger(NewRHS);
1052 : }
1053 : break;
1054 : }
1055 519 : case ISD::SETUGE:
1056 : case ISD::SETUGT:
1057 : case ISD::SETULE:
1058 : case ISD::SETULT:
1059 : // ALL of these operations will work if we either sign or zero extend
1060 : // the operands (including the unsigned comparisons!). Zero extend is
1061 : // usually a simpler/cheaper operation, so prefer it.
1062 519 : NewLHS = ZExtPromotedInteger(NewLHS);
1063 519 : NewRHS = ZExtPromotedInteger(NewRHS);
1064 519 : break;
1065 620 : case ISD::SETGE:
1066 : case ISD::SETGT:
1067 : case ISD::SETLT:
1068 : case ISD::SETLE:
1069 620 : NewLHS = SExtPromotedInteger(NewLHS);
1070 620 : NewRHS = SExtPromotedInteger(NewRHS);
1071 620 : break;
1072 : }
1073 3183 : }
1074 :
1075 17188 : SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1076 34376 : SDValue Op = GetPromotedInteger(N->getOperand(0));
1077 34797 : return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1078 : }
1079 :
1080 122 : SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1081 244 : SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1082 244 : return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1083 122 : N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1084 : }
1085 :
1086 0 : SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1087 : // This should only occur in unusual situations like bitcasting to an
1088 : // x86_fp80, so just turn it into a store+load
1089 1224 : return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1090 : }
1091 :
1092 27 : SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1093 : assert(OpNo == 2 && "Don't know how to promote this operand!");
1094 :
1095 27 : SDValue LHS = N->getOperand(2);
1096 27 : SDValue RHS = N->getOperand(3);
1097 27 : PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1098 :
1099 : // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1100 : // legal types.
1101 27 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1102 27 : N->getOperand(1), LHS, RHS, N->getOperand(4)),
1103 27 : 0);
1104 : }
1105 :
1106 79395 : SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1107 : assert(OpNo == 1 && "only know how to promote condition");
1108 :
1109 : // Promote all the way up to the canonical SetCC type.
1110 158790 : SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1111 :
1112 : // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1113 79395 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1114 79395 : N->getOperand(2)), 0);
1115 : }
1116 :
1117 0 : SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1118 : // Since the result type is legal, the operands must promote to it.
1119 0 : EVT OVT = N->getOperand(0).getValueType();
1120 0 : SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1121 0 : SDValue Hi = GetPromotedInteger(N->getOperand(1));
1122 : assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1123 : SDLoc dl(N);
1124 :
1125 0 : Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1126 0 : DAG.getConstant(OVT.getSizeInBits(), dl,
1127 0 : TLI.getPointerTy(DAG.getDataLayout())));
1128 0 : return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1129 : }
1130 :
1131 4137 : SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1132 : // The vector type is legal but the element type is not. This implies
1133 : // that the vector is a power-of-two in length and that the element
1134 : // type does not have a strange size (eg: it is not i1).
1135 8274 : EVT VecVT = N->getValueType(0);
1136 : unsigned NumElts = VecVT.getVectorNumElements();
1137 : assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1138 : "Legal vector of one illegal element?");
1139 :
1140 : // Promote the inserted value. The type does not need to match the
1141 : // vector element type. Check that any extra bits introduced will be
1142 : // truncated away.
1143 : assert(N->getOperand(0).getValueSizeInBits() >=
1144 : N->getValueType(0).getScalarSizeInBits() &&
1145 : "Type of inserted value narrower than vector element type!");
1146 :
1147 : SmallVector<SDValue, 16> NewOps;
1148 43152 : for (unsigned i = 0; i < NumElts; ++i)
1149 78030 : NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1150 :
1151 8274 : return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1152 : }
1153 :
1154 290 : SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1155 : unsigned OpNo) {
1156 290 : if (OpNo == 1) {
1157 : // Promote the inserted value. This is valid because the type does not
1158 : // have to match the vector element type.
1159 :
1160 : // Check that any extra bits introduced will be truncated away.
1161 : assert(N->getOperand(1).getValueSizeInBits() >=
1162 : N->getValueType(0).getScalarSizeInBits() &&
1163 : "Type of inserted value narrower than vector element type!");
1164 290 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1165 : GetPromotedInteger(N->getOperand(1)),
1166 290 : N->getOperand(2)),
1167 290 : 0);
1168 : }
1169 :
1170 : assert(OpNo == 2 && "Different operand and result vector types?");
1171 :
1172 : // Promote the index.
1173 0 : SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1174 0 : TLI.getVectorIdxTy(DAG.getDataLayout()));
1175 0 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1176 0 : N->getOperand(1), Idx), 0);
1177 : }
1178 :
1179 39 : SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1180 : // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1181 : // the operand in place.
1182 39 : return SDValue(DAG.UpdateNodeOperands(N,
1183 39 : GetPromotedInteger(N->getOperand(0))), 0);
1184 : }
1185 :
1186 5664 : SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1187 : assert(OpNo == 0 && "Only know how to promote the condition!");
1188 5664 : SDValue Cond = N->getOperand(0);
1189 5664 : EVT OpTy = N->getOperand(1).getValueType();
1190 :
1191 5664 : if (N->getOpcode() == ISD::VSELECT)
1192 643 : if (SDValue Res = WidenVSELECTAndMask(N))
1193 578 : return Res;
1194 :
1195 : // Promote all the way up to the canonical SetCC type.
1196 5086 : EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1197 5086 : Cond = PromoteTargetBoolean(Cond, OpVT);
1198 :
1199 5086 : return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1200 5086 : N->getOperand(2)), 0);
1201 : }
1202 :
1203 143 : SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1204 : assert(OpNo == 0 && "Don't know how to promote this operand!");
1205 :
1206 143 : SDValue LHS = N->getOperand(0);
1207 143 : SDValue RHS = N->getOperand(1);
1208 143 : PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1209 :
1210 : // The CC (#4) and the possible return values (#2 and #3) have legal types.
1211 143 : return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1212 143 : N->getOperand(3), N->getOperand(4)), 0);
1213 : }
1214 :
1215 3013 : SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1216 : assert(OpNo == 0 && "Don't know how to promote this operand!");
1217 :
1218 3013 : SDValue LHS = N->getOperand(0);
1219 3013 : SDValue RHS = N->getOperand(1);
1220 3013 : PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1221 :
1222 : // The CC (#2) is always legal.
1223 6026 : return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1224 : }
1225 :
1226 0 : SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1227 0 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1228 0 : ZExtPromotedInteger(N->getOperand(1))), 0);
1229 : }
1230 :
1231 8060 : SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1232 16120 : SDValue Op = GetPromotedInteger(N->getOperand(0));
1233 : SDLoc dl(N);
1234 24180 : Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1235 8060 : return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1236 16120 : Op, DAG.getValueType(N->getOperand(0).getValueType()));
1237 : }
1238 :
1239 133 : SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1240 133 : return SDValue(DAG.UpdateNodeOperands(N,
1241 133 : SExtPromotedInteger(N->getOperand(0))), 0);
1242 : }
1243 :
1244 11681 : SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1245 : assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1246 11681 : SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1247 : SDLoc dl(N);
1248 :
1249 11681 : SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1250 :
1251 : // Truncate the value and store the result.
1252 11681 : return DAG.getTruncStore(Ch, dl, Val, Ptr,
1253 11681 : N->getMemoryVT(), N->getMemOperand());
1254 : }
1255 :
1256 24 : SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1257 : unsigned OpNo) {
1258 :
1259 24 : SDValue DataOp = N->getValue();
1260 24 : EVT DataVT = DataOp.getValueType();
1261 24 : SDValue Mask = N->getMask();
1262 : SDLoc dl(N);
1263 :
1264 : bool TruncateStore = false;
1265 24 : if (OpNo == 3) {
1266 20 : Mask = PromoteTargetBoolean(Mask, DataVT);
1267 : // Update in place.
1268 20 : SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1269 20 : NewOps[3] = Mask;
1270 40 : return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1271 : } else { // Data operand
1272 : assert(OpNo == 1 && "Unexpected operand for promotion");
1273 4 : DataOp = GetPromotedInteger(DataOp);
1274 : TruncateStore = true;
1275 : }
1276 :
1277 4 : return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1278 : N->getMemoryVT(), N->getMemOperand(),
1279 8 : TruncateStore, N->isCompressingStore());
1280 : }
1281 :
1282 38 : SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1283 : unsigned OpNo) {
1284 : assert(OpNo == 2 && "Only know how to promote the mask!");
1285 38 : EVT DataVT = N->getValueType(0);
1286 76 : SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1287 38 : SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1288 38 : NewOps[OpNo] = Mask;
1289 76 : return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1290 : }
1291 :
1292 52 : SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1293 : unsigned OpNo) {
1294 :
1295 52 : SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1296 52 : if (OpNo == 2) {
1297 : // The Mask
1298 32 : EVT DataVT = N->getValueType(0);
1299 64 : NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1300 20 : } else if (OpNo == 4) {
1301 : // Need to sign extend the index since the bits will likely be used.
1302 40 : NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1303 : } else
1304 0 : NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1305 :
1306 104 : return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1307 : }
1308 :
1309 8 : SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1310 : unsigned OpNo) {
1311 8 : SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1312 8 : if (OpNo == 2) {
1313 : // The Mask
1314 0 : EVT DataVT = N->getValue().getValueType();
1315 0 : NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1316 8 : } else if (OpNo == 4) {
1317 : // Need to sign extend the index since the bits will likely be used.
1318 16 : NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1319 : } else
1320 0 : NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1321 16 : return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1322 : }
1323 :
1324 3505 : SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1325 7010 : SDValue Op = GetPromotedInteger(N->getOperand(0));
1326 7015 : return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1327 : }
1328 :
1329 2632 : SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1330 2632 : return SDValue(DAG.UpdateNodeOperands(N,
1331 2632 : ZExtPromotedInteger(N->getOperand(0))), 0);
1332 : }
1333 :
1334 117968 : SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1335 : SDLoc dl(N);
1336 235936 : SDValue Op = GetPromotedInteger(N->getOperand(0));
1337 353904 : Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1338 117968 : return DAG.getZeroExtendInReg(Op, dl,
1339 235936 : N->getOperand(0).getValueType().getScalarType());
1340 : }
1341 :
1342 40 : SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1343 : assert(OpNo == 2 && "Don't know how to promote this operand!");
1344 :
1345 40 : SDValue LHS = N->getOperand(0);
1346 40 : SDValue RHS = N->getOperand(1);
1347 40 : SDValue Carry = N->getOperand(2);
1348 : SDLoc DL(N);
1349 :
1350 40 : auto VT = getSetCCResultType(LHS.getValueType());
1351 40 : TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT);
1352 40 : switch (BoolType) {
1353 0 : case TargetLoweringBase::UndefinedBooleanContent:
1354 0 : Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1355 0 : break;
1356 40 : case TargetLoweringBase::ZeroOrOneBooleanContent:
1357 40 : Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1358 40 : break;
1359 0 : case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
1360 0 : Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1361 0 : break;
1362 : }
1363 :
1364 40 : return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1365 : }
1366 :
1367 : //===----------------------------------------------------------------------===//
1368 : // Integer Result Expansion
1369 : //===----------------------------------------------------------------------===//
1370 :
1371 : /// ExpandIntegerResult - This method is called when the specified result of the
1372 : /// specified node is found to need expansion. At this point, the node may also
1373 : /// have invalid operands or may have other results that need promotion, we just
1374 : /// know that (at least) one result needs expansion.
1375 505622 : void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1376 : LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1377 : dbgs() << "\n");
1378 : SDValue Lo, Hi;
1379 505622 : Lo = Hi = SDValue();
1380 :
1381 : // See if the target wants to custom expand this node.
1382 1011244 : if (CustomLowerNode(N, N->getValueType(ResNo), true))
1383 798 : return;
1384 :
1385 1009648 : switch (N->getOpcode()) {
1386 0 : default:
1387 : #ifndef NDEBUG
1388 : dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1389 : N->dump(&DAG); dbgs() << "\n";
1390 : #endif
1391 0 : llvm_unreachable("Do not know how to expand the result of this operator!");
1392 :
1393 53 : case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1394 543 : case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1395 623 : case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1396 1833 : case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1397 :
1398 12776 : case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1399 17901 : case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1400 2770 : case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1401 1983 : case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1402 16 : case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1403 :
1404 4088 : case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1405 317 : case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1406 315 : case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1407 32 : case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1408 75 : case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1409 100839 : case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1410 54 : case ISD::CTLZ_ZERO_UNDEF:
1411 54 : case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1412 9 : case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1413 14 : case ISD::CTTZ_ZERO_UNDEF:
1414 14 : case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1415 1 : case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1416 96 : case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1417 88 : case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1418 113656 : case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1419 1854 : case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1420 2 : case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1421 81 : case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1422 1695 : case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1423 234 : case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1424 78 : case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1425 18320 : case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1426 74 : case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1427 78 : case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1428 5893 : case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1429 13 : case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1430 :
1431 49 : case ISD::ATOMIC_LOAD_ADD:
1432 : case ISD::ATOMIC_LOAD_SUB:
1433 : case ISD::ATOMIC_LOAD_AND:
1434 : case ISD::ATOMIC_LOAD_CLR:
1435 : case ISD::ATOMIC_LOAD_OR:
1436 : case ISD::ATOMIC_LOAD_XOR:
1437 : case ISD::ATOMIC_LOAD_NAND:
1438 : case ISD::ATOMIC_LOAD_MIN:
1439 : case ISD::ATOMIC_LOAD_MAX:
1440 : case ISD::ATOMIC_LOAD_UMIN:
1441 : case ISD::ATOMIC_LOAD_UMAX:
1442 : case ISD::ATOMIC_SWAP:
1443 : case ISD::ATOMIC_CMP_SWAP: {
1444 49 : std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1445 49 : SplitInteger(Tmp.first, Lo, Hi);
1446 49 : ReplaceValueWith(SDValue(N, 1), Tmp.second);
1447 : break;
1448 : }
1449 : case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1450 : AtomicSDNode *AN = cast<AtomicSDNode>(N);
1451 129 : SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1452 43 : SDValue Tmp = DAG.getAtomicCmpSwap(
1453 43 : ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1454 43 : N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1455 43 : AN->getMemOperand());
1456 :
1457 : // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1458 : // success simply by comparing the loaded value against the ingoing
1459 : // comparison.
1460 86 : SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1461 86 : N->getOperand(2), ISD::SETEQ);
1462 :
1463 43 : SplitInteger(Tmp, Lo, Hi);
1464 43 : ReplaceValueWith(SDValue(N, 1), Success);
1465 86 : ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1466 : break;
1467 : }
1468 :
1469 20529 : case ISD::AND:
1470 : case ISD::OR:
1471 20529 : case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1472 :
1473 9 : case ISD::UMAX:
1474 : case ISD::SMAX:
1475 : case ISD::UMIN:
1476 9 : case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1477 :
1478 110111 : case ISD::ADD:
1479 110111 : case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1480 :
1481 19 : case ISD::ADDC:
1482 19 : case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1483 :
1484 19 : case ISD::ADDE:
1485 19 : case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1486 :
1487 798 : case ISD::ADDCARRY:
1488 798 : case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1489 :
1490 86172 : case ISD::SHL:
1491 : case ISD::SRA:
1492 86172 : case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1493 :
1494 21 : case ISD::SADDO:
1495 21 : case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1496 609 : case ISD::UADDO:
1497 609 : case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1498 40 : case ISD::UMULO:
1499 40 : case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1500 :
1501 1 : case ISD::SADDSAT: ExpandIntRes_SADDSAT(N, Lo, Hi); break;
1502 : }
1503 :
1504 : // If Lo/Hi is null, the sub-method took care of registering results etc.
1505 504823 : if (Lo.getNode())
1506 504810 : SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1507 : }
1508 :
1509 : /// Lower an atomic node to the appropriate builtin call.
1510 49 : std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1511 49 : unsigned Opc = Node->getOpcode();
1512 49 : MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1513 49 : RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1514 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1515 :
1516 49 : return ExpandChainLibCall(LC, Node, false);
1517 : }
1518 :
1519 : /// N is a shift by a value that needs to be expanded,
1520 : /// and the shift amount is a constant 'Amt'. Expand the operation.
1521 85043 : void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1522 : SDValue &Lo, SDValue &Hi) {
1523 : SDLoc DL(N);
1524 : // Expand the incoming operand to be shifted, so that we have its parts
1525 85043 : SDValue InL, InH;
1526 170086 : GetExpandedInteger(N->getOperand(0), InL, InH);
1527 :
1528 : // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1529 : // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1530 85043 : if (!Amt) {
1531 0 : Lo = InL;
1532 0 : Hi = InH;
1533 0 : return;
1534 : }
1535 :
1536 85043 : EVT NVT = InL.getValueType();
1537 85043 : unsigned VTBits = N->getValueType(0).getSizeInBits();
1538 85043 : unsigned NVTBits = NVT.getSizeInBits();
1539 85043 : EVT ShTy = N->getOperand(1).getValueType();
1540 :
1541 170086 : if (N->getOpcode() == ISD::SHL) {
1542 16907 : if (Amt.ugt(VTBits)) {
1543 2 : Lo = Hi = DAG.getConstant(0, DL, NVT);
1544 16905 : } else if (Amt.ugt(NVTBits)) {
1545 3231 : Lo = DAG.getConstant(0, DL, NVT);
1546 6462 : Hi = DAG.getNode(ISD::SHL, DL,
1547 6462 : NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1548 13674 : } else if (Amt == NVTBits) {
1549 12827 : Lo = DAG.getConstant(0, DL, NVT);
1550 12827 : Hi = InL;
1551 : } else {
1552 1694 : Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1553 847 : Hi = DAG.getNode(ISD::OR, DL, NVT,
1554 847 : DAG.getNode(ISD::SHL, DL, NVT, InH,
1555 : DAG.getConstant(Amt, DL, ShTy)),
1556 : DAG.getNode(ISD::SRL, DL, NVT, InL,
1557 897 : DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1558 : }
1559 16907 : return;
1560 : }
1561 :
1562 68136 : if (N->getOpcode() == ISD::SRL) {
1563 68056 : if (Amt.ugt(VTBits)) {
1564 2 : Lo = Hi = DAG.getConstant(0, DL, NVT);
1565 68054 : } else if (Amt.ugt(NVTBits)) {
1566 754 : Lo = DAG.getNode(ISD::SRL, DL,
1567 754 : NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1568 377 : Hi = DAG.getConstant(0, DL, NVT);
1569 67677 : } else if (Amt == NVTBits) {
1570 43062 : Lo = InH;
1571 43062 : Hi = DAG.getConstant(0, DL, NVT);
1572 : } else {
1573 24615 : Lo = DAG.getNode(ISD::OR, DL, NVT,
1574 24615 : DAG.getNode(ISD::SRL, DL, NVT, InL,
1575 : DAG.getConstant(Amt, DL, ShTy)),
1576 : DAG.getNode(ISD::SHL, DL, NVT, InH,
1577 24760 : DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1578 49230 : Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1579 : }
1580 68056 : return;
1581 : }
1582 :
1583 : assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1584 80 : if (Amt.ugt(VTBits)) {
1585 2 : Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1586 2 : DAG.getConstant(NVTBits - 1, DL, ShTy));
1587 78 : } else if (Amt.ugt(NVTBits)) {
1588 126 : Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1589 126 : DAG.getConstant(Amt - NVTBits, DL, ShTy));
1590 63 : Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1591 63 : DAG.getConstant(NVTBits - 1, DL, ShTy));
1592 15 : } else if (Amt == NVTBits) {
1593 3 : Lo = InH;
1594 3 : Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1595 3 : DAG.getConstant(NVTBits - 1, DL, ShTy));
1596 : } else {
1597 12 : Lo = DAG.getNode(ISD::OR, DL, NVT,
1598 12 : DAG.getNode(ISD::SRL, DL, NVT, InL,
1599 : DAG.getConstant(Amt, DL, ShTy)),
1600 : DAG.getNode(ISD::SHL, DL, NVT, InH,
1601 17 : DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1602 24 : Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1603 : }
1604 : }
1605 :
1606 : /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1607 : /// this shift based on knowledge of the high bit of the shift amount. If we
1608 : /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1609 : /// shift amount.
1610 1129 : bool DAGTypeLegalizer::
1611 : ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1612 1129 : SDValue Amt = N->getOperand(1);
1613 2258 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1614 1129 : EVT ShTy = Amt.getValueType();
1615 : unsigned ShBits = ShTy.getScalarSizeInBits();
1616 : unsigned NVTBits = NVT.getScalarSizeInBits();
1617 : assert(isPowerOf2_32(NVTBits) &&
1618 : "Expanded integer type size not a power of two!");
1619 : SDLoc dl(N);
1620 :
1621 1129 : APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1622 1129 : KnownBits Known;
1623 2258 : DAG.computeKnownBits(N->getOperand(1), Known);
1624 :
1625 : // If we don't know anything about the high bits, exit.
1626 1177 : if (((Known.Zero|Known.One) & HighBitMask) == 0)
1627 : return false;
1628 :
1629 : // Get the incoming operand to be shifted.
1630 58 : SDValue InL, InH;
1631 116 : GetExpandedInteger(N->getOperand(0), InL, InH);
1632 :
1633 : // If we know that any of the high bits of the shift amount are one, then we
1634 : // can do this as a couple of simple shifts.
1635 58 : if (Known.One.intersects(HighBitMask)) {
1636 : // Mask out the high bit, which we know is set.
1637 4 : Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1638 4 : DAG.getConstant(~HighBitMask, dl, ShTy));
1639 :
1640 4 : switch (N->getOpcode()) {
1641 0 : default: llvm_unreachable("Unknown shift");
1642 0 : case ISD::SHL:
1643 0 : Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1644 0 : Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1645 0 : return true;
1646 2 : case ISD::SRL:
1647 2 : Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1648 4 : Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1649 2 : return true;
1650 0 : case ISD::SRA:
1651 0 : Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1652 0 : DAG.getConstant(NVTBits - 1, dl, ShTy));
1653 0 : Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1654 0 : return true;
1655 : }
1656 : }
1657 :
1658 : // If we know that all of the high bits of the shift amount are zero, then we
1659 : // can do this as a couple of simple shifts.
1660 56 : if (HighBitMask.isSubsetOf(Known.Zero)) {
1661 : // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1662 : // shift if x is zero. We can use XOR here because x is known to be smaller
1663 : // than 32.
1664 13 : SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1665 13 : DAG.getConstant(NVTBits - 1, dl, ShTy));
1666 :
1667 : unsigned Op1, Op2;
1668 26 : switch (N->getOpcode()) {
1669 0 : default: llvm_unreachable("Unknown shift");
1670 : case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1671 7 : case ISD::SRL:
1672 7 : case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1673 : }
1674 :
1675 : // When shifting right the arithmetic for Lo and Hi is swapped.
1676 13 : if (N->getOpcode() != ISD::SHL)
1677 : std::swap(InL, InH);
1678 :
1679 : // Use a little trick to get the bits that move from Lo to Hi. First
1680 : // shift by one bit.
1681 26 : SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1682 : // Then compute the remaining shift with amount-1.
1683 26 : SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1684 :
1685 39 : Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1686 26 : Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1687 :
1688 13 : if (N->getOpcode() != ISD::SHL)
1689 : std::swap(Hi, Lo);
1690 : return true;
1691 : }
1692 :
1693 : return false;
1694 : }
1695 :
1696 : /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1697 : /// of any size.
1698 104 : bool DAGTypeLegalizer::
1699 : ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1700 104 : SDValue Amt = N->getOperand(1);
1701 208 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1702 104 : EVT ShTy = Amt.getValueType();
1703 104 : unsigned NVTBits = NVT.getSizeInBits();
1704 : assert(isPowerOf2_32(NVTBits) &&
1705 : "Expanded integer type size not a power of two!");
1706 : SDLoc dl(N);
1707 :
1708 : // Get the incoming operand to be shifted.
1709 104 : SDValue InL, InH;
1710 208 : GetExpandedInteger(N->getOperand(0), InL, InH);
1711 :
1712 104 : SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1713 208 : SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1714 208 : SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1715 104 : SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1716 104 : Amt, NVBitsNode, ISD::SETULT);
1717 104 : SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1718 : Amt, DAG.getConstant(0, dl, ShTy),
1719 104 : ISD::SETEQ);
1720 :
1721 104 : SDValue LoS, HiS, LoL, HiL;
1722 208 : switch (N->getOpcode()) {
1723 0 : default: llvm_unreachable("Unknown shift");
1724 46 : case ISD::SHL:
1725 : // Short: ShAmt < NVTBits
1726 92 : LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1727 46 : HiS = DAG.getNode(ISD::OR, dl, NVT,
1728 46 : DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1729 46 : DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1730 :
1731 : // Long: ShAmt >= NVTBits
1732 46 : LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1733 92 : HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1734 :
1735 46 : Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1736 92 : Hi = DAG.getSelect(dl, NVT, isZero, InH,
1737 46 : DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1738 46 : return true;
1739 35 : case ISD::SRL:
1740 : // Short: ShAmt < NVTBits
1741 70 : HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1742 35 : LoS = DAG.getNode(ISD::OR, dl, NVT,
1743 35 : DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1744 : // FIXME: If Amt is zero, the following shift generates an undefined result
1745 : // on some architectures.
1746 35 : DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1747 :
1748 : // Long: ShAmt >= NVTBits
1749 35 : HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1750 70 : LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1751 :
1752 70 : Lo = DAG.getSelect(dl, NVT, isZero, InL,
1753 35 : DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1754 35 : Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1755 35 : return true;
1756 23 : case ISD::SRA:
1757 : // Short: ShAmt < NVTBits
1758 46 : HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1759 23 : LoS = DAG.getNode(ISD::OR, dl, NVT,
1760 23 : DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1761 23 : DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1762 :
1763 : // Long: ShAmt >= NVTBits
1764 23 : HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1765 23 : DAG.getConstant(NVTBits - 1, dl, ShTy));
1766 46 : LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1767 :
1768 46 : Lo = DAG.getSelect(dl, NVT, isZero, InL,
1769 23 : DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1770 23 : Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1771 23 : return true;
1772 : }
1773 : }
1774 :
1775 9 : static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1776 :
1777 9 : switch (Op) {
1778 0 : default: llvm_unreachable("invalid min/max opcode");
1779 2 : case ISD::SMAX:
1780 2 : return std::make_pair(ISD::SETGT, ISD::UMAX);
1781 3 : case ISD::UMAX:
1782 3 : return std::make_pair(ISD::SETUGT, ISD::UMAX);
1783 2 : case ISD::SMIN:
1784 2 : return std::make_pair(ISD::SETLT, ISD::UMIN);
1785 2 : case ISD::UMIN:
1786 2 : return std::make_pair(ISD::SETULT, ISD::UMIN);
1787 : }
1788 : }
1789 :
1790 9 : void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1791 : SDValue &Lo, SDValue &Hi) {
1792 : SDLoc DL(N);
1793 : ISD::NodeType LoOpc;
1794 : ISD::CondCode CondC;
1795 18 : std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1796 :
1797 : // Expand the subcomponents.
1798 9 : SDValue LHSL, LHSH, RHSL, RHSH;
1799 18 : GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1800 18 : GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1801 :
1802 : // Value types
1803 9 : EVT NVT = LHSL.getValueType();
1804 9 : EVT CCT = getSetCCResultType(NVT);
1805 :
1806 : // Hi part is always the same op
1807 36 : Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1808 :
1809 : // We need to know whether to select Lo part that corresponds to 'winning'
1810 : // Hi part or if Hi parts are equal.
1811 9 : SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1812 9 : SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1813 :
1814 : // Lo part corresponding to the 'winning' Hi part
1815 9 : SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1816 :
1817 : // Recursed Lo part if Hi parts are equal, this uses unsigned version
1818 27 : SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1819 :
1820 9 : Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1821 9 : }
1822 :
1823 110111 : void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1824 : SDValue &Lo, SDValue &Hi) {
1825 : SDLoc dl(N);
1826 : // Expand the subcomponents.
1827 110111 : SDValue LHSL, LHSH, RHSL, RHSH;
1828 220222 : GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1829 220222 : GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1830 :
1831 110111 : EVT NVT = LHSL.getValueType();
1832 110111 : SDValue LoOps[2] = { LHSL, RHSL };
1833 220222 : SDValue HiOps[3] = { LHSH, RHSH };
1834 :
1835 110111 : bool HasOpCarry = TLI.isOperationLegalOrCustom(
1836 110111 : N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
1837 110111 : TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1838 110111 : if (HasOpCarry) {
1839 108934 : SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1840 108934 : if (N->getOpcode() == ISD::ADD) {
1841 217354 : Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1842 108677 : HiOps[2] = Lo.getValue(1);
1843 217354 : Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1844 : } else {
1845 514 : Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1846 257 : HiOps[2] = Lo.getValue(1);
1847 514 : Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1848 : }
1849 : return;
1850 : }
1851 :
1852 : // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1853 : // them. TODO: Teach operation legalization how to expand unsupported
1854 : // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1855 : // a carry of type MVT::Glue, but there doesn't seem to be any way to
1856 : // generate a value of this type in the expanded code sequence.
1857 : bool hasCarry =
1858 1929 : TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1859 : ISD::ADDC : ISD::SUBC,
1860 1177 : TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1861 :
1862 1177 : if (hasCarry) {
1863 384 : SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1864 192 : if (N->getOpcode() == ISD::ADD) {
1865 280 : Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1866 140 : HiOps[2] = Lo.getValue(1);
1867 280 : Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1868 : } else {
1869 104 : Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1870 52 : HiOps[2] = Lo.getValue(1);
1871 104 : Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1872 : }
1873 : return;
1874 : }
1875 :
1876 : bool hasOVF =
1877 1685 : TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1878 : ISD::UADDO : ISD::USUBO,
1879 985 : TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1880 985 : TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
1881 :
1882 985 : if (hasOVF) {
1883 690 : EVT OvfVT = getSetCCResultType(NVT);
1884 690 : SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1885 : int RevOpc;
1886 690 : if (N->getOpcode() == ISD::ADD) {
1887 : RevOpc = ISD::SUB;
1888 126 : Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1889 126 : Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1890 : } else {
1891 : RevOpc = ISD::ADD;
1892 1254 : Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1893 1254 : Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1894 : }
1895 690 : SDValue OVF = Lo.getValue(1);
1896 :
1897 690 : switch (BoolType) {
1898 0 : case TargetLoweringBase::UndefinedBooleanContent:
1899 0 : OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1900 : LLVM_FALLTHROUGH;
1901 0 : case TargetLoweringBase::ZeroOrOneBooleanContent:
1902 0 : OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1903 0 : Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1904 0 : break;
1905 690 : case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
1906 690 : OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1907 1380 : Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1908 : }
1909 : return;
1910 : }
1911 :
1912 295 : if (N->getOpcode() == ISD::ADD) {
1913 444 : Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1914 444 : Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1915 222 : SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1916 222 : ISD::SETULT);
1917 :
1918 222 : if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
1919 221 : SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
1920 442 : Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
1921 : return;
1922 : }
1923 :
1924 1 : SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1925 1 : DAG.getConstant(1, dl, NVT),
1926 1 : DAG.getConstant(0, dl, NVT));
1927 1 : SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1928 1 : ISD::SETULT);
1929 1 : SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1930 1 : DAG.getConstant(1, dl, NVT), Carry1);
1931 2 : Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1932 : } else {
1933 146 : Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1934 146 : Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1935 : SDValue Cmp =
1936 73 : DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1937 146 : LoOps[0], LoOps[1], ISD::SETULT);
1938 :
1939 73 : SDValue Borrow;
1940 73 : if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
1941 73 : Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
1942 : else
1943 0 : Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
1944 0 : DAG.getConstant(0, dl, NVT));
1945 :
1946 146 : Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1947 : }
1948 : }
1949 :
1950 19 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1951 : SDValue &Lo, SDValue &Hi) {
1952 : // Expand the subcomponents.
1953 19 : SDValue LHSL, LHSH, RHSL, RHSH;
1954 : SDLoc dl(N);
1955 38 : GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1956 38 : GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1957 57 : SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1958 19 : SDValue LoOps[2] = { LHSL, RHSL };
1959 38 : SDValue HiOps[3] = { LHSH, RHSH };
1960 :
1961 19 : if (N->getOpcode() == ISD::ADDC) {
1962 26 : Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1963 13 : HiOps[2] = Lo.getValue(1);
1964 26 : Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1965 : } else {
1966 12 : Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1967 6 : HiOps[2] = Lo.getValue(1);
1968 12 : Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1969 : }
1970 :
1971 : // Legalized the flag result - switch anything that used the old flag to
1972 : // use the new one.
1973 38 : ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1974 19 : }
1975 :
1976 19 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1977 : SDValue &Lo, SDValue &Hi) {
1978 : // Expand the subcomponents.
1979 19 : SDValue LHSL, LHSH, RHSL, RHSH;
1980 : SDLoc dl(N);
1981 38 : GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1982 38 : GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1983 57 : SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1984 19 : SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1985 38 : SDValue HiOps[3] = { LHSH, RHSH };
1986 :
1987 57 : Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1988 19 : HiOps[2] = Lo.getValue(1);
1989 57 : Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1990 :
1991 : // Legalized the flag result - switch anything that used the old flag to
1992 : // use the new one.
1993 38 : ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1994 19 : }
1995 :
1996 609 : void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1997 : SDValue &Lo, SDValue &Hi) {
1998 609 : SDValue LHS = N->getOperand(0);
1999 609 : SDValue RHS = N->getOperand(1);
2000 : SDLoc dl(N);
2001 :
2002 609 : SDValue Ovf;
2003 :
2004 1218 : bool HasOpCarry = TLI.isOperationLegalOrCustom(
2005 609 : N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2006 609 : TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2007 :
2008 609 : if (HasOpCarry) {
2009 : // Expand the subcomponents.
2010 593 : SDValue LHSL, LHSH, RHSL, RHSH;
2011 593 : GetExpandedInteger(LHS, LHSL, LHSH);
2012 593 : GetExpandedInteger(RHS, RHSL, RHSH);
2013 1779 : SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2014 593 : SDValue LoOps[2] = { LHSL, RHSL };
2015 1186 : SDValue HiOps[3] = { LHSH, RHSH };
2016 :
2017 1186 : unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2018 1186 : Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2019 593 : HiOps[2] = Lo.getValue(1);
2020 1186 : Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2021 :
2022 593 : Ovf = Hi.getValue(1);
2023 : } else {
2024 : // Expand the result by simply replacing it with the equivalent
2025 : // non-overflow-checking operation.
2026 16 : auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2027 32 : SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2028 16 : SplitInteger(Sum, Lo, Hi);
2029 :
2030 : // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2031 : // overflows iff a - b > a.
2032 16 : auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2033 32 : Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2034 : }
2035 :
2036 : // Legalized the flag result - switch anything that used the old flag to
2037 : // use the new one.
2038 609 : ReplaceValueWith(SDValue(N, 1), Ovf);
2039 609 : }
2040 :
2041 798 : void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2042 : SDValue &Lo, SDValue &Hi) {
2043 : // Expand the subcomponents.
2044 798 : SDValue LHSL, LHSH, RHSL, RHSH;
2045 : SDLoc dl(N);
2046 1596 : GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2047 1596 : GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2048 2394 : SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2049 798 : SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2050 798 : SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2051 :
2052 2394 : Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2053 798 : HiOps[2] = Lo.getValue(1);
2054 2394 : Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2055 :
2056 : // Legalized the flag result - switch anything that used the old flag to
2057 : // use the new one.
2058 1596 : ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2059 798 : }
2060 :
2061 4088 : void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2062 : SDValue &Lo, SDValue &Hi) {
2063 8176 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2064 : SDLoc dl(N);
2065 4088 : SDValue Op = N->getOperand(0);
2066 4088 : if (Op.getValueType().bitsLE(NVT)) {
2067 : // The low part is any extension of the input (which degenerates to a copy).
2068 7640 : Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2069 3820 : Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2070 : } else {
2071 : // For example, extension of an i48 to an i64. The operand type necessarily
2072 : // promotes to the result type, so will end up being expanded too.
2073 : assert(getTypeAction(Op.getValueType()) ==
2074 : TargetLowering::TypePromoteInteger &&
2075 : "Only know how to promote this result!");
2076 268 : SDValue Res = GetPromotedInteger(Op);
2077 : assert(Res.getValueType() == N->getValueType(0) &&
2078 : "Operand over promoted?");
2079 : // Split the promoted operand. This will simplify when it is expanded.
2080 268 : SplitInteger(Res, Lo, Hi);
2081 : }
2082 4088 : }
2083 :
2084 317 : void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2085 : SDValue &Lo, SDValue &Hi) {
2086 : SDLoc dl(N);
2087 634 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
2088 317 : EVT NVT = Lo.getValueType();
2089 317 : EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2090 317 : unsigned NVTBits = NVT.getSizeInBits();
2091 317 : unsigned EVTBits = EVT.getSizeInBits();
2092 :
2093 317 : if (NVTBits < EVTBits) {
2094 254 : Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2095 254 : DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2096 254 : EVTBits - NVTBits)));
2097 : } else {
2098 126 : Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2099 : // The high part replicates the sign bit of Lo, make it explicit.
2100 63 : Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2101 63 : DAG.getConstant(NVTBits - 1, dl,
2102 126 : TLI.getPointerTy(DAG.getDataLayout())));
2103 : }
2104 317 : }
2105 :
2106 315 : void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2107 : SDValue &Lo, SDValue &Hi) {
2108 : SDLoc dl(N);
2109 630 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
2110 315 : EVT NVT = Lo.getValueType();
2111 315 : EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2112 315 : unsigned NVTBits = NVT.getSizeInBits();
2113 315 : unsigned EVTBits = EVT.getSizeInBits();
2114 :
2115 315 : if (NVTBits < EVTBits) {
2116 252 : Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2117 252 : DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2118 252 : EVTBits - NVTBits)));
2119 : } else {
2120 126 : Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2121 : // The high part must be zero, make it explicit.
2122 63 : Hi = DAG.getConstant(0, dl, NVT);
2123 : }
2124 315 : }
2125 :
2126 32 : void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2127 : SDValue &Lo, SDValue &Hi) {
2128 : SDLoc dl(N);
2129 64 : GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2130 96 : Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2131 96 : Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2132 32 : }
2133 :
2134 75 : void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2135 : SDValue &Lo, SDValue &Hi) {
2136 : SDLoc dl(N);
2137 150 : GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2138 225 : Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2139 225 : Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2140 75 : }
2141 :
2142 100839 : void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2143 : SDValue &Lo, SDValue &Hi) {
2144 201678 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2145 100839 : unsigned NBitWidth = NVT.getSizeInBits();
2146 : auto Constant = cast<ConstantSDNode>(N);
2147 100839 : const APInt &Cst = Constant->getAPIntValue();
2148 100839 : bool IsTarget = Constant->isTargetOpcode();
2149 100839 : bool IsOpaque = Constant->isOpaque();
2150 : SDLoc dl(N);
2151 100839 : Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2152 208188 : Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2153 100839 : IsOpaque);
2154 100839 : }
2155 :
2156 54 : void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2157 : SDValue &Lo, SDValue &Hi) {
2158 : SDLoc dl(N);
2159 : // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2160 108 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
2161 54 : EVT NVT = Lo.getValueType();
2162 :
2163 54 : SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2164 54 : DAG.getConstant(0, dl, NVT), ISD::SETNE);
2165 :
2166 162 : SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2167 108 : SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2168 :
2169 54 : Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2170 : DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2171 54 : DAG.getConstant(NVT.getSizeInBits(), dl,
2172 54 : NVT)));
2173 54 : Hi = DAG.getConstant(0, dl, NVT);
2174 54 : }
2175 :
2176 9 : void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2177 : SDValue &Lo, SDValue &Hi) {
2178 : SDLoc dl(N);
2179 : // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2180 18 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
2181 9 : EVT NVT = Lo.getValueType();
2182 9 : Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2183 9 : DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2184 9 : Hi = DAG.getConstant(0, dl, NVT);
2185 9 : }
2186 :
2187 14 : void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2188 : SDValue &Lo, SDValue &Hi) {
2189 : SDLoc dl(N);
2190 : // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2191 28 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
2192 14 : EVT NVT = Lo.getValueType();
2193 :
2194 14 : SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2195 14 : DAG.getConstant(0, dl, NVT), ISD::SETNE);
2196 :
2197 28 : SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2198 42 : SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2199 :
2200 14 : Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2201 : DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2202 14 : DAG.getConstant(NVT.getSizeInBits(), dl,
2203 14 : NVT)));
2204 14 : Hi = DAG.getConstant(0, dl, NVT);
2205 14 : }
2206 :
2207 1 : void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2208 : SDValue &Hi) {
2209 : SDLoc dl(N);
2210 2 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2211 1 : unsigned NBitWidth = NVT.getSizeInBits();
2212 :
2213 1 : EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2214 1 : Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2215 : // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2216 1 : Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2217 1 : DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2218 1 : }
2219 :
2220 96 : void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2221 : SDValue &Hi) {
2222 : SDLoc dl(N);
2223 96 : EVT VT = N->getValueType(0);
2224 :
2225 96 : SDValue Op = N->getOperand(0);
2226 192 : if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2227 6 : Op = GetPromotedFloat(Op);
2228 :
2229 192 : RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
2230 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2231 192 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2232 : Lo, Hi);
2233 96 : }
2234 :
2235 88 : void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2236 : SDValue &Hi) {
2237 : SDLoc dl(N);
2238 88 : EVT VT = N->getValueType(0);
2239 :
2240 88 : SDValue Op = N->getOperand(0);
2241 176 : if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2242 4 : Op = GetPromotedFloat(Op);
2243 :
2244 176 : RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
2245 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2246 176 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2247 : Lo, Hi);
2248 88 : }
2249 :
2250 113656 : void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2251 : SDValue &Lo, SDValue &Hi) {
2252 : if (ISD::isNormalLoad(N)) {
2253 112547 : ExpandRes_NormalLoad(N, Lo, Hi);
2254 112547 : return;
2255 : }
2256 :
2257 : assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2258 :
2259 1109 : EVT VT = N->getValueType(0);
2260 1109 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2261 1109 : SDValue Ch = N->getChain();
2262 1109 : SDValue Ptr = N->getBasePtr();
2263 : ISD::LoadExtType ExtType = N->getExtensionType();
2264 1109 : unsigned Alignment = N->getAlignment();
2265 1109 : MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2266 : AAMDNodes AAInfo = N->getAAInfo();
2267 : SDLoc dl(N);
2268 :
2269 : assert(NVT.isByteSized() && "Expanded type not byte sized!");
2270 :
2271 1109 : if (N->getMemoryVT().bitsLE(NVT)) {
2272 545 : EVT MemVT = N->getMemoryVT();
2273 :
2274 1635 : Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2275 1090 : Alignment, MMOFlags, AAInfo);
2276 :
2277 : // Remember the chain.
2278 545 : Ch = Lo.getValue(1);
2279 :
2280 545 : if (ExtType == ISD::SEXTLOAD) {
2281 : // The high part is obtained by SRA'ing all but one of the bits of the
2282 : // lo part.
2283 106 : unsigned LoSize = Lo.getValueSizeInBits();
2284 106 : Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2285 106 : DAG.getConstant(LoSize - 1, dl,
2286 212 : TLI.getPointerTy(DAG.getDataLayout())));
2287 439 : } else if (ExtType == ISD::ZEXTLOAD) {
2288 : // The high part is just a zero.
2289 313 : Hi = DAG.getConstant(0, dl, NVT);
2290 : } else {
2291 : assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2292 : // The high part is undefined.
2293 126 : Hi = DAG.getUNDEF(NVT);
2294 : }
2295 564 : } else if (DAG.getDataLayout().isLittleEndian()) {
2296 : // Little-endian - low bits are at low addresses.
2297 1668 : Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2298 1112 : AAInfo);
2299 :
2300 : unsigned ExcessBits =
2301 556 : N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2302 556 : EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2303 :
2304 : // Increment the pointer to the other half.
2305 556 : unsigned IncrementSize = NVT.getSizeInBits()/8;
2306 556 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2307 556 : DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2308 556 : Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2309 556 : N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2310 1112 : MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2311 :
2312 : // Build a factor node to remember that this load is independent of the
2313 : // other one.
2314 556 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2315 1112 : Hi.getValue(1));
2316 : } else {
2317 : // Big-endian - high bits are at low addresses. Favor aligned loads at
2318 : // the cost of some bit-fiddling.
2319 8 : EVT MemVT = N->getMemoryVT();
2320 : unsigned EBytes = MemVT.getStoreSize();
2321 8 : unsigned IncrementSize = NVT.getSizeInBits()/8;
2322 8 : unsigned ExcessBits = (EBytes - IncrementSize)*8;
2323 :
2324 : // Load both the high bits and maybe some of the low bits.
2325 8 : Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2326 8 : EVT::getIntegerVT(*DAG.getContext(),
2327 : MemVT.getSizeInBits() - ExcessBits),
2328 8 : Alignment, MMOFlags, AAInfo);
2329 :
2330 : // Increment the pointer to the other half.
2331 8 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2332 8 : DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2333 : // Load the rest of the low bits.
2334 8 : Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2335 8 : N->getPointerInfo().getWithOffset(IncrementSize),
2336 8 : EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2337 8 : MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2338 :
2339 : // Build a factor node to remember that this load is independent of the
2340 : // other one.
2341 8 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2342 16 : Hi.getValue(1));
2343 :
2344 8 : if (ExcessBits < NVT.getSizeInBits()) {
2345 : // Transfer low bits from the bottom of Hi to the top of Lo.
2346 7 : Lo = DAG.getNode(
2347 : ISD::OR, dl, NVT, Lo,
2348 : DAG.getNode(ISD::SHL, dl, NVT, Hi,
2349 : DAG.getConstant(ExcessBits, dl,
2350 14 : TLI.getPointerTy(DAG.getDataLayout()))));
2351 : // Move high bits to the right position in Hi.
2352 7 : Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2353 : Hi,
2354 7 : DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2355 21 : TLI.getPointerTy(DAG.getDataLayout())));
2356 : }
2357 : }
2358 :
2359 : // Legalize the chain result - switch anything that used the old chain to
2360 : // use the new one.
2361 1109 : ReplaceValueWith(SDValue(N, 1), Ch);
2362 : }
2363 :
2364 20529 : void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2365 : SDValue &Lo, SDValue &Hi) {
2366 : SDLoc dl(N);
2367 20529 : SDValue LL, LH, RL, RH;
2368 41058 : GetExpandedInteger(N->getOperand(0), LL, LH);
2369 41058 : GetExpandedInteger(N->getOperand(1), RL, RH);
2370 82116 : Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2371 82116 : Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2372 20529 : }
2373 :
2374 1854 : void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2375 : SDValue &Lo, SDValue &Hi) {
2376 1854 : EVT VT = N->getValueType(0);
2377 1854 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2378 : SDLoc dl(N);
2379 :
2380 1854 : SDValue LL, LH, RL, RH;
2381 3708 : GetExpandedInteger(N->getOperand(0), LL, LH);
2382 3708 : GetExpandedInteger(N->getOperand(1), RL, RH);
2383 :
2384 1854 : if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2385 : TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
2386 : LL, LH, RL, RH))
2387 : return;
2388 :
2389 : // If nothing else, we can make a libcall.
2390 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2391 : if (VT == MVT::i16)
2392 : LC = RTLIB::MUL_I16;
2393 : else if (VT == MVT::i32)
2394 : LC = RTLIB::MUL_I32;
2395 : else if (VT == MVT::i64)
2396 : LC = RTLIB::MUL_I64;
2397 : else if (VT == MVT::i128)
2398 : LC = RTLIB::MUL_I128;
2399 :
2400 534 : if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2401 : // We'll expand the multiplication by brute force because we have no other
2402 : // options. This is a trivially-generalized version of the code from
2403 : // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2404 : // 4.3.1).
2405 287 : unsigned Bits = NVT.getSizeInBits();
2406 287 : unsigned HalfBits = Bits >> 1;
2407 287 : SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2408 287 : NVT);
2409 574 : SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2410 574 : SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2411 :
2412 574 : SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2413 574 : SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2414 :
2415 287 : EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2416 287 : if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2417 : // The type from TLI is too small to fit the shift amount we want.
2418 : // Override it with i32. The shift will have to be legalized.
2419 2 : ShiftAmtTy = MVT::i32;
2420 : }
2421 287 : SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2422 574 : SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2423 574 : SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2424 574 : SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2425 :
2426 287 : SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2427 287 : DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2428 574 : SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2429 574 : SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2430 :
2431 287 : SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2432 287 : DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2433 574 : SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2434 :
2435 287 : SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2436 287 : DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2437 287 : DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2438 287 : Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2439 287 : DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2440 :
2441 287 : Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2442 : DAG.getNode(ISD::ADD, dl, NVT,
2443 287 : DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2444 287 : DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2445 : return;
2446 : }
2447 :
2448 74 : SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2449 74 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2450 : Lo, Hi);
2451 : }
2452 :
2453 2 : void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2454 : SDValue &Hi) {
2455 : SDLoc DL(N);
2456 4 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2457 4 : SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2458 6 : SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2459 2 : Lo = R.getValue(0);
2460 2 : Hi = R.getValue(1);
2461 2 : ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2462 2 : }
2463 :
2464 1 : void DAGTypeLegalizer::ExpandIntRes_SADDSAT(SDNode *N, SDValue &Lo,
2465 : SDValue &Hi) {
2466 1 : SDValue Result = TLI.getExpandedSignedSaturationAddition(N, DAG);
2467 1 : SplitInteger(Result, Lo, Hi);
2468 1 : }
2469 :
2470 21 : void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2471 : SDValue &Lo, SDValue &Hi) {
2472 21 : SDValue LHS = Node->getOperand(0);
2473 21 : SDValue RHS = Node->getOperand(1);
2474 : SDLoc dl(Node);
2475 :
2476 : // Expand the result by simply replacing it with the equivalent
2477 : // non-overflow-checking operation.
2478 21 : SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2479 : ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2480 26 : LHS, RHS);
2481 21 : SplitInteger(Sum, Lo, Hi);
2482 :
2483 : // Compute the overflow.
2484 : //
2485 : // LHSSign -> LHS >= 0
2486 : // RHSSign -> RHS >= 0
2487 : // SumSign -> Sum >= 0
2488 : //
2489 : // Add:
2490 : // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2491 : // Sub:
2492 : // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2493 : //
2494 21 : EVT OType = Node->getValueType(1);
2495 42 : SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2496 :
2497 21 : SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2498 21 : SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2499 21 : SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2500 21 : Node->getOpcode() == ISD::SADDO ?
2501 26 : ISD::SETEQ : ISD::SETNE);
2502 :
2503 21 : SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2504 21 : SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2505 :
2506 42 : SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2507 :
2508 : // Use the calculated overflow everywhere.
2509 21 : ReplaceValueWith(SDValue(Node, 1), Cmp);
2510 21 : }
2511 :
2512 81 : void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2513 : SDValue &Lo, SDValue &Hi) {
2514 162 : EVT VT = N->getValueType(0);
2515 : SDLoc dl(N);
2516 81 : SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2517 :
2518 81 : if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
2519 12 : SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2520 6 : SplitInteger(Res.getValue(0), Lo, Hi);
2521 : return;
2522 : }
2523 :
2524 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2525 : if (VT == MVT::i16)
2526 : LC = RTLIB::SDIV_I16;
2527 : else if (VT == MVT::i32)
2528 : LC = RTLIB::SDIV_I32;
2529 : else if (VT == MVT::i64)
2530 : LC = RTLIB::SDIV_I64;
2531 : else if (VT == MVT::i128)
2532 : LC = RTLIB::SDIV_I128;
2533 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2534 :
2535 75 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2536 : }
2537 :
2538 86172 : void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2539 : SDValue &Lo, SDValue &Hi) {
2540 172344 : EVT VT = N->getValueType(0);
2541 : SDLoc dl(N);
2542 :
2543 : // If we can emit an efficient shift operation, do so now. Check to see if
2544 : // the RHS is a constant.
2545 86172 : if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2546 170086 : return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2547 :
2548 : // If we can determine that the high bit of the shift is zero or one, even if
2549 : // the low bits are variable, emit this shift in an optimized form.
2550 1129 : if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2551 : return;
2552 :
2553 : // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2554 : unsigned PartsOpc;
2555 2228 : if (N->getOpcode() == ISD::SHL) {
2556 : PartsOpc = ISD::SHL_PARTS;
2557 589 : } else if (N->getOpcode() == ISD::SRL) {
2558 : PartsOpc = ISD::SRL_PARTS;
2559 : } else {
2560 : assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2561 : PartsOpc = ISD::SRA_PARTS;
2562 : }
2563 :
2564 : // Next check to see if the target supports this SHL_PARTS operation or if it
2565 : // will custom expand it.
2566 1114 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2567 1114 : TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2568 1193 : if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2569 : Action == TargetLowering::Custom) {
2570 : // Expand the subcomponents.
2571 926 : SDValue LHSL, LHSH;
2572 1852 : GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2573 926 : EVT VT = LHSL.getValueType();
2574 :
2575 : // If the shift amount operand is coming from a vector legalization it may
2576 : // have an illegal type. Fix that first by casting the operand, otherwise
2577 : // the new SHL_PARTS operation would need further legalization.
2578 926 : SDValue ShiftOp = N->getOperand(1);
2579 926 : EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2580 : assert(ShiftTy.getScalarSizeInBits() >=
2581 : Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2582 : "ShiftAmountTy is too small to cover the range of this type!");
2583 926 : if (ShiftOp.getValueType() != ShiftTy)
2584 61 : ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2585 :
2586 926 : SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2587 1852 : Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2588 926 : Hi = Lo.getValue(1);
2589 : return;
2590 : }
2591 :
2592 : // Otherwise, emit a libcall.
2593 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2594 : bool isSigned;
2595 376 : if (N->getOpcode() == ISD::SHL) {
2596 : isSigned = false; /*sign irrelevant*/
2597 : if (VT == MVT::i16)
2598 : LC = RTLIB::SHL_I16;
2599 : else if (VT == MVT::i32)
2600 : LC = RTLIB::SHL_I32;
2601 : else if (VT == MVT::i64)
2602 : LC = RTLIB::SHL_I64;
2603 : else if (VT == MVT::i128)
2604 : LC = RTLIB::SHL_I128;
2605 96 : } else if (N->getOpcode() == ISD::SRL) {
2606 : isSigned = false;
2607 : if (VT == MVT::i16)
2608 : LC = RTLIB::SRL_I16;
2609 : else if (VT == MVT::i32)
2610 : LC = RTLIB::SRL_I32;
2611 : else if (VT == MVT::i64)
2612 : LC = RTLIB::SRL_I64;
2613 : else if (VT == MVT::i128)
2614 : LC = RTLIB::SRL_I128;
2615 : } else {
2616 : assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2617 : isSigned = true;
2618 : if (VT == MVT::i16)
2619 : LC = RTLIB::SRA_I16;
2620 : else if (VT == MVT::i32)
2621 : LC = RTLIB::SRA_I32;
2622 : else if (VT == MVT::i64)
2623 : LC = RTLIB::SRA_I64;
2624 : else if (VT == MVT::i128)
2625 : LC = RTLIB::SRA_I128;
2626 : }
2627 :
2628 298 : if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2629 84 : SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2630 168 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2631 : return;
2632 : }
2633 :
2634 104 : if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2635 0 : llvm_unreachable("Unsupported shift!");
2636 : }
2637 :
2638 1695 : void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2639 : SDValue &Lo, SDValue &Hi) {
2640 3390 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2641 : SDLoc dl(N);
2642 1695 : SDValue Op = N->getOperand(0);
2643 3390 : if (Op.getValueType().bitsLE(NVT)) {
2644 : // The low part is sign extension of the input (degenerates to a copy).
2645 3388 : Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2646 : // The high part is obtained by SRA'ing all but one of the bits of low part.
2647 1694 : unsigned LoSize = NVT.getSizeInBits();
2648 1694 : Hi = DAG.getNode(
2649 : ISD::SRA, dl, NVT, Lo,
2650 3388 : DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2651 : } else {
2652 : // For example, extension of an i48 to an i64. The operand type necessarily
2653 : // promotes to the result type, so will end up being expanded too.
2654 : assert(getTypeAction(Op.getValueType()) ==
2655 : TargetLowering::TypePromoteInteger &&
2656 : "Only know how to promote this result!");
2657 1 : SDValue Res = GetPromotedInteger(Op);
2658 : assert(Res.getValueType() == N->getValueType(0) &&
2659 : "Operand over promoted?");
2660 : // Split the promoted operand. This will simplify when it is expanded.
2661 1 : SplitInteger(Res, Lo, Hi);
2662 1 : unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2663 1 : Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2664 1 : DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2665 1 : ExcessBits)));
2666 : }
2667 1695 : }
2668 :
2669 234 : void DAGTypeLegalizer::
2670 : ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2671 : SDLoc dl(N);
2672 468 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
2673 234 : EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2674 :
2675 468 : if (EVT.bitsLE(Lo.getValueType())) {
2676 : // sext_inreg the low part if needed.
2677 109 : Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2678 109 : N->getOperand(1));
2679 :
2680 : // The high part gets the sign extension from the lo-part. This handles
2681 : // things like sextinreg V:i64 from i8.
2682 109 : Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2683 109 : DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2684 218 : TLI.getPointerTy(DAG.getDataLayout())));
2685 : } else {
2686 : // For example, extension of an i48 to an i64. Leave the low part alone,
2687 : // sext_inreg the high part.
2688 125 : unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2689 125 : Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2690 125 : DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2691 125 : ExcessBits)));
2692 : }
2693 234 : }
2694 :
2695 78 : void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2696 : SDValue &Lo, SDValue &Hi) {
2697 156 : EVT VT = N->getValueType(0);
2698 : SDLoc dl(N);
2699 78 : SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2700 :
2701 78 : if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
2702 20 : SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2703 10 : SplitInteger(Res.getValue(1), Lo, Hi);
2704 : return;
2705 : }
2706 :
2707 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2708 : if (VT == MVT::i16)
2709 : LC = RTLIB::SREM_I16;
2710 : else if (VT == MVT::i32)
2711 : LC = RTLIB::SREM_I32;
2712 : else if (VT == MVT::i64)
2713 : LC = RTLIB::SREM_I64;
2714 : else if (VT == MVT::i128)
2715 : LC = RTLIB::SREM_I128;
2716 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2717 :
2718 68 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2719 : }
2720 :
2721 18320 : void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2722 : SDValue &Lo, SDValue &Hi) {
2723 36640 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2724 : SDLoc dl(N);
2725 54960 : Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2726 18320 : Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2727 18320 : N->getOperand(0),
2728 18320 : DAG.getConstant(NVT.getSizeInBits(), dl,
2729 36640 : TLI.getPointerTy(DAG.getDataLayout())));
2730 36640 : Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2731 18320 : }
2732 :
2733 40 : void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2734 : SDValue &Lo, SDValue &Hi) {
2735 80 : EVT VT = N->getValueType(0);
2736 : SDLoc dl(N);
2737 :
2738 40 : if (N->getOpcode() == ISD::UMULO) {
2739 : // This section expands the operation into the following sequence of
2740 : // instructions. `iNh` here refers to a type which has half the bit width of
2741 : // the type the original operation operated on.
2742 : //
2743 : // %0 = %LHS.HI != 0 && %RHS.HI != 0
2744 : // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
2745 : // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
2746 : // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
2747 : // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
2748 : // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
2749 : //
2750 : // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
2751 30 : SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2752 30 : SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
2753 30 : SplitInteger(LHS, LHSLow, LHSHigh);
2754 30 : SplitInteger(RHS, RHSLow, RHSHigh);
2755 30 : EVT HalfVT = LHSLow.getValueType()
2756 30 : , BitVT = N->getValueType(1);
2757 30 : SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
2758 30 : SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
2759 :
2760 30 : SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
2761 30 : SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
2762 30 : DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
2763 30 : DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
2764 :
2765 30 : SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
2766 60 : Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
2767 30 : SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
2768 30 : One.getValue(0));
2769 :
2770 30 : SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
2771 60 : Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
2772 30 : SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
2773 30 : Two.getValue(0));
2774 :
2775 : // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
2776 : // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
2777 : // operation recursively legalized?).
2778 : //
2779 : // Many backends understand this pattern and will convert into LOHI
2780 : // themselves, if applicable.
2781 30 : SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
2782 30 : DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
2783 30 : DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
2784 60 : SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
2785 30 : SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
2786 60 : Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
2787 30 : SplitInteger(Five, Lo, Hi);
2788 30 : ReplaceValueWith(SDValue(N, 1), Overflow);
2789 : return;
2790 : }
2791 :
2792 10 : Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2793 10 : EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2794 10 : Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2795 :
2796 : // Replace this with a libcall that will check overflow.
2797 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2798 : if (VT == MVT::i32)
2799 : LC = RTLIB::MULO_I32;
2800 : else if (VT == MVT::i64)
2801 : LC = RTLIB::MULO_I64;
2802 : else if (VT == MVT::i128)
2803 : LC = RTLIB::MULO_I128;
2804 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2805 :
2806 10 : SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2807 : // Temporary for the overflow value, default it to zero.
2808 : SDValue Chain =
2809 10 : DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2810 10 : MachinePointerInfo());
2811 :
2812 : TargetLowering::ArgListTy Args;
2813 : TargetLowering::ArgListEntry Entry;
2814 30 : for (const SDValue &Op : N->op_values()) {
2815 20 : EVT ArgVT = Op.getValueType();
2816 20 : Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2817 20 : Entry.Node = Op;
2818 20 : Entry.Ty = ArgTy;
2819 20 : Entry.IsSExt = true;
2820 20 : Entry.IsZExt = false;
2821 20 : Args.push_back(Entry);
2822 : }
2823 :
2824 : // Also pass the address of the overflow check.
2825 10 : Entry.Node = Temp;
2826 10 : Entry.Ty = PtrTy->getPointerTo();
2827 10 : Entry.IsSExt = true;
2828 10 : Entry.IsZExt = false;
2829 10 : Args.push_back(Entry);
2830 :
2831 20 : SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2832 :
2833 10 : TargetLowering::CallLoweringInfo CLI(DAG);
2834 : CLI.setDebugLoc(dl)
2835 10 : .setChain(Chain)
2836 20 : .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2837 : .setSExtResult();
2838 :
2839 10 : std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2840 :
2841 10 : SplitInteger(CallInfo.first, Lo, Hi);
2842 : SDValue Temp2 =
2843 20 : DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2844 10 : SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2845 : DAG.getConstant(0, dl, PtrVT),
2846 10 : ISD::SETNE);
2847 : // Use the overflow from the libcall everywhere.
2848 10 : ReplaceValueWith(SDValue(N, 1), Ofl);
2849 : }
2850 :
2851 74 : void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2852 : SDValue &Lo, SDValue &Hi) {
2853 148 : EVT VT = N->getValueType(0);
2854 : SDLoc dl(N);
2855 74 : SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2856 :
2857 74 : if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
2858 26 : SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2859 13 : SplitInteger(Res.getValue(0), Lo, Hi);
2860 : return;
2861 : }
2862 :
2863 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2864 : if (VT == MVT::i16)
2865 : LC = RTLIB::UDIV_I16;
2866 : else if (VT == MVT::i32)
2867 : LC = RTLIB::UDIV_I32;
2868 : else if (VT == MVT::i64)
2869 : LC = RTLIB::UDIV_I64;
2870 : else if (VT == MVT::i128)
2871 : LC = RTLIB::UDIV_I128;
2872 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2873 :
2874 61 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2875 : }
2876 :
2877 78 : void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2878 : SDValue &Lo, SDValue &Hi) {
2879 156 : EVT VT = N->getValueType(0);
2880 : SDLoc dl(N);
2881 78 : SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2882 :
2883 78 : if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
2884 24 : SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2885 12 : SplitInteger(Res.getValue(1), Lo, Hi);
2886 : return;
2887 : }
2888 :
2889 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2890 : if (VT == MVT::i16)
2891 : LC = RTLIB::UREM_I16;
2892 : else if (VT == MVT::i32)
2893 : LC = RTLIB::UREM_I32;
2894 : else if (VT == MVT::i64)
2895 : LC = RTLIB::UREM_I64;
2896 : else if (VT == MVT::i128)
2897 : LC = RTLIB::UREM_I128;
2898 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2899 :
2900 66 : SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2901 : }
2902 :
2903 5893 : void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2904 : SDValue &Lo, SDValue &Hi) {
2905 11786 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2906 : SDLoc dl(N);
2907 5893 : SDValue Op = N->getOperand(0);
2908 11786 : if (Op.getValueType().bitsLE(NVT)) {
2909 : // The low part is zero extension of the input (degenerates to a copy).
2910 11786 : Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2911 5893 : Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2912 : } else {
2913 : // For example, extension of an i48 to an i64. The operand type necessarily
2914 : // promotes to the result type, so will end up being expanded too.
2915 : assert(getTypeAction(Op.getValueType()) ==
2916 : TargetLowering::TypePromoteInteger &&
2917 : "Only know how to promote this result!");
2918 0 : SDValue Res = GetPromotedInteger(Op);
2919 : assert(Res.getValueType() == N->getValueType(0) &&
2920 : "Operand over promoted?");
2921 : // Split the promoted operand. This will simplify when it is expanded.
2922 0 : SplitInteger(Res, Lo, Hi);
2923 0 : unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2924 0 : Hi = DAG.getZeroExtendInReg(Hi, dl,
2925 0 : EVT::getIntegerVT(*DAG.getContext(),
2926 0 : ExcessBits));
2927 : }
2928 5893 : }
2929 :
2930 13 : void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2931 : SDValue &Lo, SDValue &Hi) {
2932 : SDLoc dl(N);
2933 13 : EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2934 26 : SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2935 13 : SDValue Zero = DAG.getConstant(0, dl, VT);
2936 13 : SDValue Swap = DAG.getAtomicCmpSwap(
2937 : ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2938 : cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2939 26 : N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2940 :
2941 13 : ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2942 13 : ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2943 13 : }
2944 :
2945 : //===----------------------------------------------------------------------===//
2946 : // Integer Operand Expansion
2947 : //===----------------------------------------------------------------------===//
2948 :
2949 : /// ExpandIntegerOperand - This method is called when the specified operand of
2950 : /// the specified node is found to need expansion. At this point, all of the
2951 : /// result types of the node are known to be legal, but other operands of the
2952 : /// node may need promotion or expansion as well as the specified one.
2953 294486 : bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2954 : LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
2955 : dbgs() << "\n");
2956 294486 : SDValue Res = SDValue();
2957 :
2958 883458 : if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2959 : return false;
2960 :
2961 587590 : switch (N->getOpcode()) {
2962 0 : default:
2963 : #ifndef NDEBUG
2964 : dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2965 : N->dump(&DAG); dbgs() << "\n";
2966 : #endif
2967 0 : llvm_unreachable("Do not know how to expand this operator's operand!");
2968 :
2969 8605 : case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2970 0 : case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2971 23951 : case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2972 19582 : case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2973 32 : case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2974 64 : case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2975 1754 : case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2976 1072 : case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2977 26 : case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
2978 45 : case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2979 114143 : case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2980 121879 : case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2981 42 : case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2982 :
2983 2586 : case ISD::SHL:
2984 : case ISD::SRA:
2985 : case ISD::SRL:
2986 : case ISD::ROTL:
2987 2586 : case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2988 1 : case ISD::RETURNADDR:
2989 1 : case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2990 :
2991 13 : case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2992 : }
2993 :
2994 : // If the result is null, the sub-method took care of registering results etc.
2995 293795 : if (!Res.getNode()) return false;
2996 :
2997 : // If the result is N, the sub-method updated N in place. Tell the legalizer
2998 : // core about this.
2999 293795 : if (Res.getNode() == N)
3000 : return true;
3001 :
3002 : assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3003 : "Invalid operand expansion");
3004 :
3005 288782 : ReplaceValueWith(SDValue(N, 0), Res);
3006 288782 : return false;
3007 : }
3008 :
3009 : /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3010 : /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3011 2826 : void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3012 : SDValue &NewRHS,
3013 : ISD::CondCode &CCCode,
3014 : const SDLoc &dl) {
3015 2826 : SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3016 2826 : GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3017 2826 : GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3018 :
3019 2826 : if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3020 460 : if (RHSLo == RHSHi) {
3021 : if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3022 572 : if (RHSCST->isAllOnesValue()) {
3023 : // Equality comparison to -1.
3024 4 : NewLHS = DAG.getNode(ISD::AND, dl,
3025 8 : LHSLo.getValueType(), LHSLo, LHSHi);
3026 4 : NewRHS = RHSLo;
3027 955 : return;
3028 : }
3029 : }
3030 : }
3031 :
3032 1368 : NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3033 1368 : NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3034 1368 : NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3035 912 : NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3036 456 : return;
3037 : }
3038 :
3039 : // If this is a comparison of the sign bit, just look at the top part.
3040 : // X > -1, x < 0
3041 : if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3042 585 : if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3043 220 : (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3044 232 : NewLHS = LHSHi;
3045 232 : NewRHS = RHSHi;
3046 232 : return;
3047 : }
3048 :
3049 : // FIXME: This generated code sucks.
3050 : ISD::CondCode LowCC;
3051 2134 : switch (CCCode) {
3052 0 : default: llvm_unreachable("Unknown integer setcc!");
3053 : case ISD::SETLT:
3054 : case ISD::SETULT: LowCC = ISD::SETULT; break;
3055 94 : case ISD::SETGT:
3056 94 : case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3057 36 : case ISD::SETLE:
3058 36 : case ISD::SETULE: LowCC = ISD::SETULE; break;
3059 1812 : case ISD::SETGE:
3060 1812 : case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3061 : }
3062 :
3063 : // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3064 : // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3065 : // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3066 :
3067 : // NOTE: on targets without efficient SELECT of bools, we can always use
3068 : // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3069 : TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3070 2134 : nullptr);
3071 : SDValue LoCmp, HiCmp;
3072 2134 : if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3073 2108 : TLI.isTypeLegal(RHSLo.getValueType()))
3074 2108 : LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3075 2108 : RHSLo, LowCC, false, DagCombineInfo, dl);
3076 2134 : if (!LoCmp.getNode())
3077 2048 : LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3078 4096 : RHSLo, LowCC);
3079 2134 : if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3080 2108 : TLI.isTypeLegal(RHSHi.getValueType()))
3081 2108 : HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3082 2108 : RHSHi, CCCode, false, DagCombineInfo, dl);
3083 2134 : if (!HiCmp.getNode())
3084 2014 : HiCmp =
3085 2014 : DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3086 2014 : LHSHi, RHSHi, DAG.getCondCode(CCCode));
3087 :
3088 : ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3089 : ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3090 :
3091 2054 : bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3092 2435 : CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3093 :
3094 2139 : if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3095 286 : (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3096 12 : (LoCmpC && LoCmpC->isNullValue())))) {
3097 : // For LE / GE, if high part is known false, ignore the low part.
3098 : // For LT / GT: if low part is known false, return the high part.
3099 : // if high part is known true, ignore the low part.
3100 8 : NewLHS = HiCmp;
3101 8 : NewRHS = SDValue();
3102 8 : return;
3103 : }
3104 :
3105 2126 : if (LHSHi == RHSHi) {
3106 : // Comparing the low bits is enough.
3107 8 : NewLHS = LoCmp;
3108 8 : NewRHS = SDValue();
3109 8 : return;
3110 : }
3111 :
3112 : // Lower with SETCCCARRY if the target supports it.
3113 2118 : EVT HiVT = LHSHi.getValueType();
3114 2118 : EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3115 2118 : bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3116 :
3117 : // FIXME: Make all targets support this, then remove the other lowering.
3118 : if (HasSETCCCARRY) {
3119 : // SETCCCARRY can detect < and >= directly. For > and <=, flip
3120 : // operands and condition code.
3121 : bool FlipOperands = false;
3122 247 : switch (CCCode) {
3123 40 : case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3124 21 : case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3125 14 : case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3126 13 : case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3127 : default: break;
3128 : }
3129 : if (FlipOperands) {
3130 : std::swap(LHSLo, RHSLo);
3131 : std::swap(LHSHi, RHSHi);
3132 : }
3133 : // Perform a wide subtraction, feeding the carry from the low part into
3134 : // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3135 : // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3136 : // zero or positive iff LHS >= RHS.
3137 247 : EVT LoVT = LHSLo.getValueType();
3138 247 : SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3139 247 : SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3140 247 : SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3141 : LHSHi, RHSHi, LowCmp.getValue(1),
3142 247 : DAG.getCondCode(CCCode));
3143 247 : NewLHS = Res;
3144 247 : NewRHS = SDValue();
3145 : return;
3146 : }
3147 :
3148 1871 : NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3149 1871 : false, DagCombineInfo, dl);
3150 1871 : if (!NewLHS.getNode())
3151 1842 : NewLHS =
3152 1842 : DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3153 3742 : NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3154 1871 : NewRHS = SDValue();
3155 : }
3156 :
3157 0 : SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3158 0 : SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3159 0 : ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3160 0 : IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3161 :
3162 : // If ExpandSetCCOperands returned a scalar, we need to compare the result
3163 : // against zero to select between true and false values.
3164 0 : if (!NewRHS.getNode()) {
3165 0 : NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3166 0 : CCCode = ISD::SETNE;
3167 : }
3168 :
3169 : // Update N to have the operands specified.
3170 0 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3171 : DAG.getCondCode(CCCode), NewLHS, NewRHS,
3172 0 : N->getOperand(4)), 0);
3173 : }
3174 :
3175 1754 : SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3176 1754 : SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3177 1754 : ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3178 1754 : IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3179 :
3180 : // If ExpandSetCCOperands returned a scalar, we need to compare the result
3181 : // against zero to select between true and false values.
3182 1754 : if (!NewRHS.getNode()) {
3183 1734 : NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3184 1734 : CCCode = ISD::SETNE;
3185 : }
3186 :
3187 : // Update N to have the operands specified.
3188 1754 : return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3189 1754 : N->getOperand(2), N->getOperand(3),
3190 1754 : DAG.getCondCode(CCCode)), 0);
3191 : }
3192 :
3193 1072 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3194 1072 : SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3195 1072 : ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3196 1072 : IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3197 :
3198 : // If ExpandSetCCOperands returned a scalar, use it.
3199 1072 : if (!NewRHS.getNode()) {
3200 : assert(NewLHS.getValueType() == N->getValueType(0) &&
3201 : "Unexpected setcc expansion!");
3202 400 : return NewLHS;
3203 : }
3204 :
3205 : // Otherwise, update N to have the operands specified.
3206 672 : return SDValue(
3207 1344 : DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3208 : }
3209 :
3210 26 : SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3211 26 : SDValue LHS = N->getOperand(0);
3212 26 : SDValue RHS = N->getOperand(1);
3213 26 : SDValue Carry = N->getOperand(2);
3214 26 : SDValue Cond = N->getOperand(3);
3215 : SDLoc dl = SDLoc(N);
3216 :
3217 26 : SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3218 26 : GetExpandedInteger(LHS, LHSLo, LHSHi);
3219 26 : GetExpandedInteger(RHS, RHSLo, RHSHi);
3220 :
3221 : // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3222 78 : SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3223 26 : SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3224 26 : return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3225 52 : LowCmp.getValue(1), Cond);
3226 : }
3227 :
3228 2586 : SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3229 : // The value being shifted is legal, but the shift amount is too big.
3230 : // It follows that either the result of the shift is undefined, or the
3231 : // upper half of the shift amount is zero. Just use the lower half.
3232 2586 : SDValue Lo, Hi;
3233 5172 : GetExpandedInteger(N->getOperand(1), Lo, Hi);
3234 5172 : return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3235 : }
3236 :
3237 1 : SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3238 : // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3239 : // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3240 : // constant to valid type.
3241 1 : SDValue Lo, Hi;
3242 2 : GetExpandedInteger(N->getOperand(0), Lo, Hi);
3243 1 : return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3244 : }
3245 :
3246 45 : SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3247 45 : SDValue Op = N->getOperand(0);
3248 45 : EVT DstVT = N->getValueType(0);
3249 90 : RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
3250 : assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3251 : "Don't know how to expand this SINT_TO_FP!");
3252 45 : return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3253 : }
3254 :
3255 114143 : SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3256 : if (ISD::isNormalStore(N))
3257 112712 : return ExpandOp_NormalStore(N, OpNo);
3258 :
3259 : assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3260 : assert(OpNo == 1 && "Can only expand the stored value so far");
3261 :
3262 1431 : EVT VT = N->getOperand(1).getValueType();
3263 1431 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3264 1431 : SDValue Ch = N->getChain();
3265 1431 : SDValue Ptr = N->getBasePtr();
3266 1431 : unsigned Alignment = N->getAlignment();
3267 1431 : MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3268 : AAMDNodes AAInfo = N->getAAInfo();
3269 : SDLoc dl(N);
3270 1431 : SDValue Lo, Hi;
3271 :
3272 : assert(NVT.isByteSized() && "Expanded type not byte sized!");
3273 :
3274 1431 : if (N->getMemoryVT().bitsLE(NVT)) {
3275 352 : GetExpandedInteger(N->getValue(), Lo, Hi);
3276 704 : return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3277 704 : N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3278 : }
3279 :
3280 1079 : if (DAG.getDataLayout().isLittleEndian()) {
3281 : // Little-endian - low bits are at low addresses.
3282 1069 : GetExpandedInteger(N->getValue(), Lo, Hi);
3283 :
3284 3207 : Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3285 2138 : AAInfo);
3286 :
3287 : unsigned ExcessBits =
3288 1069 : N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3289 1069 : EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3290 :
3291 : // Increment the pointer to the other half.
3292 1069 : unsigned IncrementSize = NVT.getSizeInBits()/8;
3293 1069 : Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3294 2138 : Hi = DAG.getTruncStore(
3295 1069 : Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3296 3207 : MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3297 2138 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3298 : }
3299 :
3300 : // Big-endian - high bits are at low addresses. Favor aligned stores at
3301 : // the cost of some bit-fiddling.
3302 10 : GetExpandedInteger(N->getValue(), Lo, Hi);
3303 :
3304 10 : EVT ExtVT = N->getMemoryVT();
3305 : unsigned EBytes = ExtVT.getStoreSize();
3306 10 : unsigned IncrementSize = NVT.getSizeInBits()/8;
3307 10 : unsigned ExcessBits = (EBytes - IncrementSize)*8;
3308 10 : EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3309 10 : ExtVT.getSizeInBits() - ExcessBits);
3310 :
3311 10 : if (ExcessBits < NVT.getSizeInBits()) {
3312 : // Transfer high bits from the top of Lo to the bottom of Hi.
3313 8 : Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3314 8 : DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3315 16 : TLI.getPointerTy(DAG.getDataLayout())));
3316 8 : Hi = DAG.getNode(
3317 : ISD::OR, dl, NVT, Hi,
3318 : DAG.getNode(ISD::SRL, dl, NVT, Lo,
3319 : DAG.getConstant(ExcessBits, dl,
3320 16 : TLI.getPointerTy(DAG.getDataLayout()))));
3321 : }
3322 :
3323 : // Store both the high bits and maybe some of the low bits.
3324 30 : Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3325 20 : MMOFlags, AAInfo);
3326 :
3327 : // Increment the pointer to the other half.
3328 10 : Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3329 : // Store the lowest ExcessBits bits in the second half.
3330 20 : Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3331 10 : N->getPointerInfo().getWithOffset(IncrementSize),
3332 10 : EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3333 20 : MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3334 20 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3335 : }
3336 :
3337 121879 : SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3338 121879 : SDValue InL, InH;
3339 243758 : GetExpandedInteger(N->getOperand(0), InL, InH);
3340 : // Just truncate the low part of the source.
3341 243832 : return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3342 : }
3343 :
3344 42 : SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3345 42 : SDValue Op = N->getOperand(0);
3346 42 : EVT SrcVT = Op.getValueType();
3347 84 : EVT DstVT = N->getValueType(0);
3348 : SDLoc dl(N);
3349 :
3350 : // The following optimization is valid only if every value in SrcVT (when
3351 : // treated as signed) is representable in DstVT. Check that the mantissa
3352 : // size of DstVT is >= than the number of bits in SrcVT -1.
3353 42 : const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3354 42 : if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3355 0 : TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){
3356 : // Do a signed conversion then adjust the result.
3357 0 : SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3358 0 : SignedConv = TLI.LowerOperation(SignedConv, DAG);
3359 :
3360 : // The result of the signed conversion needs adjusting if the 'sign bit' of
3361 : // the incoming integer was set. To handle this, we dynamically test to see
3362 : // if it is set, and, if so, add a fudge factor.
3363 :
3364 : const uint64_t F32TwoE32 = 0x4F800000ULL;
3365 : const uint64_t F32TwoE64 = 0x5F800000ULL;
3366 : const uint64_t F32TwoE128 = 0x7F800000ULL;
3367 :
3368 : APInt FF(32, 0);
3369 : if (SrcVT == MVT::i32)
3370 0 : FF = APInt(32, F32TwoE32);
3371 : else if (SrcVT == MVT::i64)
3372 0 : FF = APInt(32, F32TwoE64);
3373 : else if (SrcVT == MVT::i128)
3374 0 : FF = APInt(32, F32TwoE128);
3375 : else
3376 0 : llvm_unreachable("Unsupported UINT_TO_FP!");
3377 :
3378 : // Check whether the sign bit is set.
3379 0 : SDValue Lo, Hi;
3380 0 : GetExpandedInteger(Op, Lo, Hi);
3381 0 : SDValue SignSet = DAG.getSetCC(dl,
3382 : getSetCCResultType(Hi.getValueType()),
3383 : Hi,
3384 : DAG.getConstant(0, dl, Hi.getValueType()),
3385 0 : ISD::SETLT);
3386 :
3387 : // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3388 : SDValue FudgePtr =
3389 0 : DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3390 0 : TLI.getPointerTy(DAG.getDataLayout()));
3391 :
3392 : // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3393 0 : SDValue Zero = DAG.getIntPtrConstant(0, dl);
3394 0 : SDValue Four = DAG.getIntPtrConstant(4, dl);
3395 0 : if (DAG.getDataLayout().isBigEndian())
3396 : std::swap(Zero, Four);
3397 0 : SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3398 0 : Zero, Four);
3399 0 : unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3400 0 : FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3401 0 : FudgePtr, Offset);
3402 0 : Alignment = std::min(Alignment, 4u);
3403 :
3404 : // Load the value out, extending it from f32 to the destination float type.
3405 : // FIXME: Avoid the extend by constructing the right constant pool?
3406 0 : SDValue Fudge = DAG.getExtLoad(
3407 0 : ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3408 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
3409 0 : Alignment);
3410 0 : return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3411 : }
3412 :
3413 : // Otherwise, use a libcall.
3414 42 : RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3415 : assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3416 : "Don't know how to expand this UINT_TO_FP!");
3417 84 : return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3418 : }
3419 :
3420 13 : SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3421 : SDLoc dl(N);
3422 13 : SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3423 : cast<AtomicSDNode>(N)->getMemoryVT(),
3424 : N->getOperand(0),
3425 13 : N->getOperand(1), N->getOperand(2),
3426 26 : cast<AtomicSDNode>(N)->getMemOperand());
3427 13 : return Swap.getValue(1);
3428 : }
3429 :
3430 :
3431 2842 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3432 2842 : SDValue InOp0 = N->getOperand(0);
3433 2842 : EVT InVT = InOp0.getValueType();
3434 :
3435 2842 : EVT OutVT = N->getValueType(0);
3436 2842 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3437 : assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3438 : unsigned OutNumElems = OutVT.getVectorNumElements();
3439 2842 : EVT NOutVTElem = NOutVT.getVectorElementType();
3440 :
3441 : SDLoc dl(N);
3442 2842 : SDValue BaseIdx = N->getOperand(1);
3443 :
3444 : SmallVector<SDValue, 8> Ops;
3445 2842 : Ops.reserve(OutNumElems);
3446 11606 : for (unsigned i = 0; i != OutNumElems; ++i) {
3447 :
3448 : // Extract the element from the original vector.
3449 8764 : SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3450 8764 : BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3451 8764 : SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3452 17528 : InVT.getVectorElementType(), N->getOperand(0), Index);
3453 :
3454 17528 : SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3455 : // Insert the converted element to the new vector.
3456 8764 : Ops.push_back(Op);
3457 : }
3458 :
3459 5684 : return DAG.getBuildVector(NOutVT, dl, Ops);
3460 : }
3461 :
3462 :
3463 108 : SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3464 : ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3465 216 : EVT VT = N->getValueType(0);
3466 : SDLoc dl(N);
3467 :
3468 216 : ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3469 :
3470 216 : SDValue V0 = GetPromotedInteger(N->getOperand(0));
3471 216 : SDValue V1 = GetPromotedInteger(N->getOperand(1));
3472 108 : EVT OutVT = V0.getValueType();
3473 :
3474 108 : return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3475 : }
3476 :
3477 :
3478 1249 : SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3479 1249 : EVT OutVT = N->getValueType(0);
3480 1249 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3481 : assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3482 1249 : unsigned NumElems = N->getNumOperands();
3483 1249 : EVT NOutVTElem = NOutVT.getVectorElementType();
3484 :
3485 : SDLoc dl(N);
3486 :
3487 : SmallVector<SDValue, 8> Ops;
3488 1249 : Ops.reserve(NumElems);
3489 6727 : for (unsigned i = 0; i != NumElems; ++i) {
3490 5478 : SDValue Op;
3491 : // BUILD_VECTOR integer operand types are allowed to be larger than the
3492 : // result's element type. This may still be true after the promotion. For
3493 : // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3494 : // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3495 16434 : if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3496 10692 : Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3497 : else
3498 132 : Op = N->getOperand(i);
3499 5478 : Ops.push_back(Op);
3500 : }
3501 :
3502 2498 : return DAG.getBuildVector(NOutVT, dl, Ops);
3503 : }
3504 :
3505 27 : SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3506 :
3507 : SDLoc dl(N);
3508 :
3509 : assert(!N->getOperand(0).getValueType().isVector() &&
3510 : "Input must be a scalar");
3511 :
3512 27 : EVT OutVT = N->getValueType(0);
3513 27 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3514 : assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3515 27 : EVT NOutVTElem = NOutVT.getVectorElementType();
3516 :
3517 81 : SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3518 :
3519 54 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3520 : }
3521 :
3522 189 : SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3523 : SDLoc dl(N);
3524 :
3525 189 : EVT OutVT = N->getValueType(0);
3526 189 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3527 : assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3528 :
3529 189 : EVT OutElemTy = NOutVT.getVectorElementType();
3530 :
3531 567 : unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3532 : unsigned NumOutElem = NOutVT.getVectorNumElements();
3533 189 : unsigned NumOperands = N->getNumOperands();
3534 : assert(NumElem * NumOperands == NumOutElem &&
3535 : "Unexpected number of elements");
3536 :
3537 : // Take the elements from the first vector.
3538 189 : SmallVector<SDValue, 8> Ops(NumOutElem);
3539 637 : for (unsigned i = 0; i < NumOperands; ++i) {
3540 896 : SDValue Op = N->getOperand(i);
3541 448 : if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3542 448 : Op = GetPromotedInteger(Op);
3543 448 : EVT SclrTy = Op.getValueType().getVectorElementType();
3544 : assert(NumElem == Op.getValueType().getVectorNumElements() &&
3545 : "Unexpected number of elements");
3546 :
3547 1592 : for (unsigned j = 0; j < NumElem; ++j) {
3548 1144 : SDValue Ext = DAG.getNode(
3549 : ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3550 1144 : DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3551 1144 : Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3552 : }
3553 : }
3554 :
3555 378 : return DAG.getBuildVector(NOutVT, dl, Ops);
3556 : }
3557 :
3558 3 : SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3559 3 : EVT VT = N->getValueType(0);
3560 3 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3561 : assert(NVT.isVector() && "This type must be promoted to a vector type");
3562 :
3563 : SDLoc dl(N);
3564 :
3565 : // For operands whose TypeAction is to promote, extend the promoted node
3566 : // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3567 : // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3568 : // type..
3569 6 : if (getTypeAction(N->getOperand(0).getValueType())
3570 : == TargetLowering::TypePromoteInteger) {
3571 3 : SDValue Promoted;
3572 :
3573 6 : switch(N->getOpcode()) {
3574 1 : case ISD::SIGN_EXTEND_VECTOR_INREG:
3575 2 : Promoted = SExtPromotedInteger(N->getOperand(0));
3576 1 : break;
3577 1 : case ISD::ZERO_EXTEND_VECTOR_INREG:
3578 2 : Promoted = ZExtPromotedInteger(N->getOperand(0));
3579 1 : break;
3580 1 : case ISD::ANY_EXTEND_VECTOR_INREG:
3581 2 : Promoted = GetPromotedInteger(N->getOperand(0));
3582 1 : break;
3583 0 : default:
3584 0 : llvm_unreachable("Node has unexpected Opcode");
3585 : }
3586 9 : return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3587 : }
3588 :
3589 : // Directly extend to the appropriate transform-to type.
3590 0 : return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3591 : }
3592 :
3593 213 : SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3594 213 : EVT OutVT = N->getValueType(0);
3595 213 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3596 : assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3597 :
3598 213 : EVT NOutVTElem = NOutVT.getVectorElementType();
3599 :
3600 : SDLoc dl(N);
3601 426 : SDValue V0 = GetPromotedInteger(N->getOperand(0));
3602 :
3603 213 : SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3604 426 : NOutVTElem, N->getOperand(1));
3605 213 : return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3606 426 : V0, ConvElem, N->getOperand(2));
3607 : }
3608 :
3609 6375 : SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3610 : SDLoc dl(N);
3611 12750 : SDValue V0 = GetPromotedInteger(N->getOperand(0));
3612 6375 : SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3613 6375 : TLI.getVectorIdxTy(DAG.getDataLayout()));
3614 6375 : SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3615 12750 : V0->getValueType(0).getScalarType(), V0, V1);
3616 :
3617 : // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3618 : // element types. If this is the case then we need to expand the outgoing
3619 : // value and not truncate it.
3620 12750 : return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3621 : }
3622 :
3623 0 : SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3624 : SDLoc dl(N);
3625 0 : SDValue V0 = GetPromotedInteger(N->getOperand(0));
3626 0 : MVT InVT = V0.getValueType().getSimpleVT();
3627 : MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3628 0 : N->getValueType(0).getVectorNumElements());
3629 0 : SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3630 0 : return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3631 : }
3632 :
3633 3055 : SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3634 : SDLoc dl(N);
3635 3055 : unsigned NumElems = N->getNumOperands();
3636 :
3637 6110 : EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3638 :
3639 : SmallVector<SDValue, 8> NewOps;
3640 3055 : NewOps.reserve(NumElems);
3641 :
3642 : // For each incoming vector
3643 11055 : for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3644 16000 : SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3645 16000 : EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3646 24000 : unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3647 :
3648 39724 : for (unsigned i=0; i<NumElem; ++i) {
3649 : // Extract element from incoming vector
3650 31724 : SDValue Ex = DAG.getNode(
3651 : ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3652 31724 : DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3653 63448 : SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3654 31724 : NewOps.push_back(Tr);
3655 : }
3656 : }
3657 :
3658 9165 : return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3659 : }
|
