LLVM 19.0.0git
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type. For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type. For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//
19
20#include "LegalizeTypes.h"
28#include <algorithm>
29using namespace llvm;
30
31#define DEBUG_TYPE "legalize-types"
32
33//===----------------------------------------------------------------------===//
34// Integer Result Promotion
35//===----------------------------------------------------------------------===//
36
37/// PromoteIntegerResult - This method is called when a result of a node is
38/// found to be in need of promotion to a larger type. At this point, the node
39/// may also have invalid operands or may have other results that need
40/// expansion, we just know that (at least) one result needs promotion.
41void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
43 SDValue Res = SDValue();
44
45 // See if the target wants to custom expand this node.
46 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
47 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
48 return;
49 }
50
51 switch (N->getOpcode()) {
52 default:
53#ifndef NDEBUG
54 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
55 N->dump(&DAG); dbgs() << "\n";
56#endif
57 report_fatal_error("Do not know how to promote this operator!");
58 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
59 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
60 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
61 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
62 case ISD::VP_BITREVERSE:
63 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
64 case ISD::VP_BSWAP:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
66 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
67 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
69 case ISD::VP_CTLZ:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
72 case ISD::PARITY:
73 case ISD::VP_CTPOP:
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
76 case ISD::VP_CTTZ:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
80 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
81 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
82 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
83 break;
84 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
85 break;
86 case ISD::SELECT:
87 case ISD::VSELECT:
88 case ISD::VP_SELECT:
89 case ISD::VP_MERGE:
90 Res = PromoteIntRes_Select(N);
91 break;
92 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
95 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
96 case ISD::SMIN:
97 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
98 case ISD::UMIN:
99 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
100
101 case ISD::SHL:
102 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
104 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
105 case ISD::SRA:
106 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break;
107 case ISD::SRL:
108 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break;
109 case ISD::VP_TRUNCATE:
110 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
111 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
112 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
113 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
114
116 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
118 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
120 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
122 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
124 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
127 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
128 return;
130 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
132 Res = PromoteIntRes_BUILD_VECTOR(N);
133 break;
136 Res = PromoteIntRes_ScalarOp(N);
137 break;
138 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
140 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
141
145 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
146
147 case ISD::SIGN_EXTEND:
148 case ISD::VP_SIGN_EXTEND:
149 case ISD::ZERO_EXTEND:
150 case ISD::VP_ZERO_EXTEND:
151 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
152
153 case ISD::VP_FP_TO_SINT:
154 case ISD::VP_FP_TO_UINT:
157 case ISD::FP_TO_SINT:
158 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
159
162 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
163
164 case ISD::FP_TO_BF16:
165 case ISD::FP_TO_FP16:
166 Res = PromoteIntRes_FP_TO_FP16_BF16(N);
167 break;
170 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
171 break;
172 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
173
174 case ISD::AND:
175 case ISD::OR:
176 case ISD::XOR:
177 case ISD::ADD:
178 case ISD::SUB:
179 case ISD::MUL:
180 case ISD::VP_AND:
181 case ISD::VP_OR:
182 case ISD::VP_XOR:
183 case ISD::VP_ADD:
184 case ISD::VP_SUB:
185 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
186
187 case ISD::VP_SMIN:
188 case ISD::VP_SMAX:
189 case ISD::SDIV:
190 case ISD::SREM:
191 case ISD::VP_SDIV:
192 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
193
194 case ISD::VP_UMIN:
195 case ISD::VP_UMAX:
196 case ISD::UDIV:
197 case ISD::UREM:
198 case ISD::VP_UDIV:
199 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
200
201 case ISD::SADDO:
202 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
203 case ISD::UADDO:
204 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
205 case ISD::SMULO:
206 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
207
208 case ISD::ADDE:
209 case ISD::SUBE:
210 case ISD::UADDO_CARRY:
211 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
212
213 case ISD::SADDO_CARRY:
214 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
215
216 case ISD::SADDSAT:
217 case ISD::UADDSAT:
218 case ISD::SSUBSAT:
219 case ISD::USUBSAT:
220 case ISD::SSHLSAT:
221 case ISD::USHLSAT:
222 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(N);
223 break;
224 case ISD::VP_SADDSAT:
225 case ISD::VP_UADDSAT:
226 case ISD::VP_SSUBSAT:
227 case ISD::VP_USUBSAT:
228 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(N);
229 break;
230
231 case ISD::SMULFIX:
232 case ISD::SMULFIXSAT:
233 case ISD::UMULFIX:
234 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
235
236 case ISD::SDIVFIX:
237 case ISD::SDIVFIXSAT:
238 case ISD::UDIVFIX:
239 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
240
241 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
242
243 case ISD::ATOMIC_LOAD:
244 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
245
257 case ISD::ATOMIC_SWAP:
258 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
259
262 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
263 break;
264
274 Res = PromoteIntRes_VECREDUCE(N);
275 break;
276
277 case ISD::VP_REDUCE_ADD:
278 case ISD::VP_REDUCE_MUL:
279 case ISD::VP_REDUCE_AND:
280 case ISD::VP_REDUCE_OR:
281 case ISD::VP_REDUCE_XOR:
282 case ISD::VP_REDUCE_SMAX:
283 case ISD::VP_REDUCE_SMIN:
284 case ISD::VP_REDUCE_UMAX:
285 case ISD::VP_REDUCE_UMIN:
286 Res = PromoteIntRes_VP_REDUCE(N);
287 break;
288
289 case ISD::FREEZE:
290 Res = PromoteIntRes_FREEZE(N);
291 break;
292
293 case ISD::ROTL:
294 case ISD::ROTR:
295 Res = PromoteIntRes_Rotate(N);
296 break;
297
298 case ISD::FSHL:
299 case ISD::FSHR:
300 Res = PromoteIntRes_FunnelShift(N);
301 break;
302
303 case ISD::VP_FSHL:
304 case ISD::VP_FSHR:
305 Res = PromoteIntRes_VPFunnelShift(N);
306 break;
307
308 case ISD::IS_FPCLASS:
309 Res = PromoteIntRes_IS_FPCLASS(N);
310 break;
311 case ISD::FFREXP:
312 Res = PromoteIntRes_FFREXP(N);
313 break;
314
315 case ISD::LRINT:
316 case ISD::LLRINT:
317 Res = PromoteIntRes_XRINT(N);
318 break;
319 }
320
321 // If the result is null then the sub-method took care of registering it.
322 if (Res.getNode())
323 SetPromotedInteger(SDValue(N, ResNo), Res);
324}
325
326SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
327 unsigned ResNo) {
328 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
329 return GetPromotedInteger(Op);
330}
331
332SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
333 // Sign-extend the new bits, and continue the assertion.
334 SDValue Op = SExtPromotedInteger(N->getOperand(0));
335 return DAG.getNode(ISD::AssertSext, SDLoc(N),
336 Op.getValueType(), Op, N->getOperand(1));
337}
338
339SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
340 // Zero the new bits, and continue the assertion.
341 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
342 return DAG.getNode(ISD::AssertZext, SDLoc(N),
343 Op.getValueType(), Op, N->getOperand(1));
344}
345
346SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
347 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
348 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
349 N->getMemoryVT(), ResVT,
350 N->getChain(), N->getBasePtr(),
351 N->getMemOperand());
352 if (N->getOpcode() == ISD::ATOMIC_LOAD) {
353 ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
354 if (ETy == ISD::NON_EXTLOAD) {
355 switch (TLI.getExtendForAtomicOps()) {
356 case ISD::SIGN_EXTEND:
357 ETy = ISD::SEXTLOAD;
358 break;
359 case ISD::ZERO_EXTEND:
360 ETy = ISD::ZEXTLOAD;
361 break;
362 case ISD::ANY_EXTEND:
363 ETy = ISD::EXTLOAD;
364 break;
365 default:
366 llvm_unreachable("Invalid atomic op extension");
367 }
368 }
369 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
370 }
371
372 // Legalize the chain result - switch anything that used the old chain to
373 // use the new one.
374 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
375 return Res;
376}
377
378SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
379 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
380 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
381 N->getMemoryVT(),
382 N->getChain(), N->getBasePtr(),
383 Op2, N->getMemOperand());
384 // Legalize the chain result - switch anything that used the old chain to
385 // use the new one.
386 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
387 return Res;
388}
389
390SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
391 unsigned ResNo) {
392 if (ResNo == 1) {
394 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
395 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
396
397 // Only use the result of getSetCCResultType if it is legal,
398 // otherwise just use the promoted result type (NVT).
399 if (!TLI.isTypeLegal(SVT))
400 SVT = NVT;
401
402 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
403 SDValue Res = DAG.getAtomicCmpSwap(
404 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
405 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
406 N->getMemOperand());
407 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
408 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
409 return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT);
410 }
411
412 // Op2 is used for the comparison and thus must be extended according to the
413 // target's atomic operations. Op3 is merely stored and so can be left alone.
414 SDValue Op2 = N->getOperand(2);
415 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
416 switch (TLI.getExtendForAtomicCmpSwapArg()) {
417 case ISD::SIGN_EXTEND:
418 Op2 = SExtPromotedInteger(Op2);
419 break;
420 case ISD::ZERO_EXTEND:
421 Op2 = ZExtPromotedInteger(Op2);
422 break;
423 case ISD::ANY_EXTEND:
424 Op2 = GetPromotedInteger(Op2);
425 break;
426 default:
427 llvm_unreachable("Invalid atomic op extension");
428 }
429
430 SDVTList VTs =
431 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
432 SDValue Res = DAG.getAtomicCmpSwap(
433 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
434 N->getBasePtr(), Op2, Op3, N->getMemOperand());
435 // Update the use to N with the newly created Res.
436 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
437 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
438 return Res;
439}
440
441SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
442 SDValue InOp = N->getOperand(0);
443 EVT InVT = InOp.getValueType();
444 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
445 EVT OutVT = N->getValueType(0);
446 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
447 SDLoc dl(N);
448
449 switch (getTypeAction(InVT)) {
451 break;
453 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
454 // The input promotes to the same size. Convert the promoted value.
455 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
456 break;
458 // Promote the integer operand by hand.
459 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
461 // Promote the integer operand by hand.
462 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
464 // Convert the promoted float by hand.
465 if (!NOutVT.isVector())
466 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
467 break;
468 }
471 break;
473 // Convert the element to an integer and promote it by hand.
474 if (!NOutVT.isVector())
475 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
476 BitConvertToInteger(GetScalarizedVector(InOp)));
477 break;
479 report_fatal_error("Scalarization of scalable vectors is not supported.");
481 if (!NOutVT.isVector()) {
482 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
483 // pieces of the input into integers and reassemble in the final type.
484 SDValue Lo, Hi;
485 GetSplitVector(N->getOperand(0), Lo, Hi);
486 Lo = BitConvertToInteger(Lo);
487 Hi = BitConvertToInteger(Hi);
488
489 if (DAG.getDataLayout().isBigEndian())
490 std::swap(Lo, Hi);
491
492 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
494 NOutVT.getSizeInBits()),
495 JoinIntegers(Lo, Hi));
496 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
497 }
498 break;
499 }
501 // The input is widened to the same size. Convert to the widened value.
502 // Make sure that the outgoing value is not a vector, because this would
503 // make us bitcast between two vectors which are legalized in different ways.
504 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
505 SDValue Res =
506 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
507
508 // For big endian targets we need to shift the casted value or the
509 // interesting bits will end up at the wrong place.
510 if (DAG.getDataLayout().isBigEndian()) {
511 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
512 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
513 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
514 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
515 }
516 return Res;
517 }
518 // If the output type is also a vector and widening it to the same size
519 // as the widened input type would be a legal type, we can widen the bitcast
520 // and handle the promotion after.
521 if (NOutVT.isVector()) {
522 TypeSize WidenInSize = NInVT.getSizeInBits();
523 TypeSize OutSize = OutVT.getSizeInBits();
524 if (WidenInSize.hasKnownScalarFactor(OutSize)) {
525 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
526 EVT WideOutVT =
528 OutVT.getVectorElementCount() * Scale);
529 if (isTypeLegal(WideOutVT)) {
530 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
531 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
532 DAG.getVectorIdxConstant(0, dl));
533 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
534 }
535 }
536 }
537 }
538
539 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
540 CreateStackStoreLoad(InOp, OutVT));
541}
542
543SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
544 SDValue V = GetPromotedInteger(N->getOperand(0));
545 return DAG.getNode(ISD::FREEZE, SDLoc(N),
546 V.getValueType(), V);
547}
548
549SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
550 SDValue Op = GetPromotedInteger(N->getOperand(0));
551 EVT OVT = N->getValueType(0);
552 EVT NVT = Op.getValueType();
553 SDLoc dl(N);
554
555 // If the larger BSWAP isn't supported by the target, try to expand now.
556 // If we expand later we'll end up with more operations since we lost the
557 // original type. We only do this for scalars since we have a shuffle
558 // based lowering for vectors in LegalizeVectorOps.
559 if (!OVT.isVector() &&
561 if (SDValue Res = TLI.expandBSWAP(N, DAG))
562 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
563 }
564
565 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
566 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
567 if (N->getOpcode() == ISD::BSWAP)
568 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
569 ShAmt);
570 SDValue Mask = N->getOperand(1);
571 SDValue EVL = N->getOperand(2);
572 return DAG.getNode(ISD::VP_LSHR, dl, NVT,
573 DAG.getNode(ISD::VP_BSWAP, dl, NVT, Op, Mask, EVL), ShAmt,
574 Mask, EVL);
575}
576
577SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
578 SDValue Op = GetPromotedInteger(N->getOperand(0));
579 EVT OVT = N->getValueType(0);
580 EVT NVT = Op.getValueType();
581 SDLoc dl(N);
582
583 // If the larger BITREVERSE isn't supported by the target, try to expand now.
584 // If we expand later we'll end up with more operations since we lost the
585 // original type. We only do this for scalars since we have a shuffle
586 // based lowering for vectors in LegalizeVectorOps.
587 if (!OVT.isVector() && OVT.isSimple() &&
589 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
590 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
591 }
592
593 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
594 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
595 if (N->getOpcode() == ISD::BITREVERSE)
596 return DAG.getNode(ISD::SRL, dl, NVT,
597 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt);
598 SDValue Mask = N->getOperand(1);
599 SDValue EVL = N->getOperand(2);
600 return DAG.getNode(ISD::VP_LSHR, dl, NVT,
601 DAG.getNode(ISD::VP_BITREVERSE, dl, NVT, Op, Mask, EVL),
602 ShAmt, Mask, EVL);
603}
604
605SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
606 // The pair element type may be legal, or may not promote to the same type as
607 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
608 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
610 N->getValueType(0)), JoinIntegers(N->getOperand(0),
611 N->getOperand(1)));
612}
613
614SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
615 EVT VT = N->getValueType(0);
616 // FIXME there is no actual debug info here
617 SDLoc dl(N);
618 // Zero extend things like i1, sign extend everything else. It shouldn't
619 // matter in theory which one we pick, but this tends to give better code?
620 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
621 SDValue Result = DAG.getNode(Opc, dl,
622 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
623 SDValue(N, 0));
624 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
625 return Result;
626}
627
628SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
629 EVT OVT = N->getValueType(0);
630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
631 SDLoc dl(N);
632
633 // If the larger CTLZ isn't supported by the target, try to expand now.
634 // If we expand later we'll end up with more operations since we lost the
635 // original type.
636 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
639 if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
640 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
641 return Result;
642 }
643 }
644
645 // Zero extend to the promoted type and do the count there.
646 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
647
648 // Subtract off the extra leading bits in the bigger type.
649 SDValue ExtractLeadingBits = DAG.getConstant(
650 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
651 if (!N->isVPOpcode())
652 return DAG.getNode(ISD::SUB, dl, NVT,
653 DAG.getNode(N->getOpcode(), dl, NVT, Op),
654 ExtractLeadingBits);
655 SDValue Mask = N->getOperand(1);
656 SDValue EVL = N->getOperand(2);
657 return DAG.getNode(ISD::VP_SUB, dl, NVT,
658 DAG.getNode(N->getOpcode(), dl, NVT, Op, Mask, EVL),
659 ExtractLeadingBits, Mask, EVL);
660}
661
662SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
663 EVT OVT = N->getValueType(0);
664 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
665
666 // If the larger CTPOP isn't supported by the target, try to expand now.
667 // If we expand later we'll end up with more operations since we lost the
668 // original type.
669 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
670 // TargetLowering.
671 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
673 if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
674 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
675 return Result;
676 }
677 }
678
679 // Zero extend to the promoted type and do the count or parity there.
680 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
681 if (!N->isVPOpcode())
682 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
683 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op,
684 N->getOperand(1), N->getOperand(2));
685}
686
687SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
688 SDValue Op = GetPromotedInteger(N->getOperand(0));
689 EVT OVT = N->getValueType(0);
690 EVT NVT = Op.getValueType();
691 SDLoc dl(N);
692
693 // If the larger CTTZ isn't supported by the target, try to expand now.
694 // If we expand later we'll end up with more operations since we lost the
695 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
696 // larger type.
697 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
700 !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
701 !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
702 if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
703 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
704 return Result;
705 }
706 }
707
708 if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) {
709 // The count is the same in the promoted type except if the original
710 // value was zero. This can be handled by setting the bit just off
711 // the top of the original type.
712 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
713 OVT.getScalarSizeInBits());
714 if (N->getOpcode() == ISD::CTTZ)
715 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
716 else
717 Op =
718 DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT),
719 N->getOperand(1), N->getOperand(2));
720 }
721 if (!N->isVPOpcode())
722 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
723 return DAG.getNode(N->getOpcode(), dl, NVT, Op, N->getOperand(1),
724 N->getOperand(2));
725}
726
727SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
728 SDLoc dl(N);
729 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
730
731 SDValue Op0 = N->getOperand(0);
732 SDValue Op1 = N->getOperand(1);
733
734 // If the input also needs to be promoted, do that first so we can get a
735 // get a good idea for the output type.
736 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
738 SDValue In = GetPromotedInteger(Op0);
739
740 // If the new type is larger than NVT, use it. We probably won't need to
741 // promote it again.
742 EVT SVT = In.getValueType().getScalarType();
743 if (SVT.bitsGE(NVT)) {
744 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
745 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
746 }
747 }
748
749 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
750}
751
752SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
754 unsigned NewOpc = N->getOpcode();
755 SDLoc dl(N);
756
757 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
758 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
759 // and SINT conversions are Custom, there is no way to tell which is
760 // preferable. We choose SINT because that's the right thing on PPC.)
761 if (N->getOpcode() == ISD::FP_TO_UINT &&
764 NewOpc = ISD::FP_TO_SINT;
765
766 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
769 NewOpc = ISD::STRICT_FP_TO_SINT;
770
771 if (N->getOpcode() == ISD::VP_FP_TO_UINT &&
772 !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) &&
773 TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT))
774 NewOpc = ISD::VP_FP_TO_SINT;
775
776 SDValue Res;
777 if (N->isStrictFPOpcode()) {
778 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
779 {N->getOperand(0), N->getOperand(1)});
780 // Legalize the chain result - switch anything that used the old chain to
781 // use the new one.
782 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
783 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
784 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1),
785 N->getOperand(2)});
786 } else {
787 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
788 }
789
790 // Assert that the converted value fits in the original type. If it doesn't
791 // (eg: because the value being converted is too big), then the result of the
792 // original operation was undefined anyway, so the assert is still correct.
793 //
794 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
795 // before legalization: fp-to-uint16, 65534. -> 0xfffe
796 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
797 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
798 N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
799 N->getOpcode() == ISD::VP_FP_TO_UINT)
802 dl, NVT, Res,
803 DAG.getValueType(N->getValueType(0).getScalarType()));
804}
805
806SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
807 // Promote the result type, while keeping the original width in Op1.
808 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
809 SDLoc dl(N);
810 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
811 N->getOperand(1));
812}
813
814SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
815 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
816 SDLoc dl(N);
817
818 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
819}
820
821SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
823 SDLoc dl(N);
824
825 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
826 N->getOperand(0), N->getOperand(1));
827 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
828 return Res;
829}
830
831SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
833 SDLoc dl(N);
834 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
835}
836
837SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
838 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
839 SDLoc dl(N);
840
841 SDValue Res =
842 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
843
844 // Legalize the chain result - switch anything that used the old chain to
845 // use the new one.
846 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
847 return Res;
848}
849
850SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
852 SDLoc dl(N);
853
854 if (getTypeAction(N->getOperand(0).getValueType())
856 SDValue Res = GetPromotedInteger(N->getOperand(0));
857 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
858
859 // If the result and operand types are the same after promotion, simplify
860 // to an in-register extension. Unless this is a VP_*_EXTEND.
861 if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
862 // The high bits are not guaranteed to be anything. Insert an extend.
863 if (N->getOpcode() == ISD::SIGN_EXTEND)
864 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
865 DAG.getValueType(N->getOperand(0).getValueType()));
866 if (N->getOpcode() == ISD::ZERO_EXTEND)
867 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
868 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
869 return Res;
870 }
871 }
872
873 // Otherwise, just extend the original operand all the way to the larger type.
874 if (N->getNumOperands() != 1) {
875 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
876 assert(N->isVPOpcode() && "Expected VP opcode");
877 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
878 N->getOperand(1), N->getOperand(2));
879 }
880 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
881}
882
883SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
884 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
885 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
886 ISD::LoadExtType ExtType =
887 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
888 SDLoc dl(N);
889 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
890 N->getMemoryVT(), N->getMemOperand());
891
892 // Legalize the chain result - switch anything that used the old chain to
893 // use the new one.
894 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
895 return Res;
896}
897
898SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
899 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
900 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
901
902 ISD::LoadExtType ExtType = N->getExtensionType();
903 if (ExtType == ISD::NON_EXTLOAD)
904 ExtType = ISD::EXTLOAD;
905
906 SDLoc dl(N);
907 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
908 N->getOffset(), N->getMask(), ExtPassThru,
909 N->getMemoryVT(), N->getMemOperand(),
910 N->getAddressingMode(), ExtType,
911 N->isExpandingLoad());
912 // Legalize the chain result - switch anything that used the old chain to
913 // use the new one.
914 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
915 return Res;
916}
917
918SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
920 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
921 assert(NVT == ExtPassThru.getValueType() &&
922 "Gather result type and the passThru argument type should be the same");
923
924 ISD::LoadExtType ExtType = N->getExtensionType();
925 if (ExtType == ISD::NON_EXTLOAD)
926 ExtType = ISD::EXTLOAD;
927
928 SDLoc dl(N);
929 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
930 N->getIndex(), N->getScale() };
931 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
932 N->getMemoryVT(), dl, Ops,
933 N->getMemOperand(), N->getIndexType(),
934 ExtType);
935 // Legalize the chain result - switch anything that used the old chain to
936 // use the new one.
937 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
938 return Res;
939}
940
941/// Promote the overflow flag of an overflowing arithmetic node.
942SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
943 // Change the return type of the boolean result while obeying
944 // getSetCCResultType.
945 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
946 EVT VT = N->getValueType(0);
947 EVT SVT = getSetCCResultType(VT);
948 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
949 unsigned NumOps = N->getNumOperands();
950 assert(NumOps <= 3 && "Too many operands");
951 if (NumOps == 3)
952 Ops[2] = N->getOperand(2);
953
954 SDLoc dl(N);
955 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
956 ArrayRef(Ops, NumOps));
957
958 // Modified the sum result - switch anything that used the old sum to use
959 // the new one.
960 ReplaceValueWith(SDValue(N, 0), Res);
961
962 // Convert to the expected type.
963 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
964}
965
966template <class MatchContextClass>
967SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
968 // If the promoted type is legal, we can convert this to:
969 // 1. ANY_EXTEND iN to iM
970 // 2. SHL by M-N
971 // 3. [US][ADD|SUB|SHL]SAT
972 // 4. L/ASHR by M-N
973 // Else it is more efficient to convert this to a min and a max
974 // operation in the higher precision arithmetic.
975 SDLoc dl(N);
976 SDValue Op1 = N->getOperand(0);
977 SDValue Op2 = N->getOperand(1);
978 MatchContextClass matcher(DAG, TLI, N);
979 unsigned OldBits = Op1.getScalarValueSizeInBits();
980
981 unsigned Opcode = matcher.getRootBaseOpcode();
982 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
983
984 // FIXME: We need vp-aware PromotedInteger functions.
985 SDValue Op1Promoted, Op2Promoted;
986 if (IsShift) {
987 Op1Promoted = GetPromotedInteger(Op1);
988 Op2Promoted = ZExtPromotedInteger(Op2);
989 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
990 Op1Promoted = ZExtPromotedInteger(Op1);
991 Op2Promoted = ZExtPromotedInteger(Op2);
992 } else {
993 Op1Promoted = SExtPromotedInteger(Op1);
994 Op2Promoted = SExtPromotedInteger(Op2);
995 }
996 EVT PromotedType = Op1Promoted.getValueType();
997 unsigned NewBits = PromotedType.getScalarSizeInBits();
998
999 if (Opcode == ISD::UADDSAT) {
1000 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
1001 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1002 SDValue Add =
1003 matcher.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1004 return matcher.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
1005 }
1006
1007 // USUBSAT can always be promoted as long as we have zero-extended the args.
1008 if (Opcode == ISD::USUBSAT)
1009 return matcher.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1010 Op2Promoted);
1011
1012 // Shift cannot use a min/max expansion, we can't detect overflow if all of
1013 // the bits have been shifted out.
1014 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1015 unsigned ShiftOp;
1016 switch (Opcode) {
1017 case ISD::SADDSAT:
1018 case ISD::SSUBSAT:
1019 case ISD::SSHLSAT:
1020 ShiftOp = ISD::SRA;
1021 break;
1022 case ISD::USHLSAT:
1023 ShiftOp = ISD::SRL;
1024 break;
1025 default:
1026 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
1027 "addition, subtraction or left shift");
1028 }
1029
1030 unsigned SHLAmount = NewBits - OldBits;
1031 SDValue ShiftAmount =
1032 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
1033 Op1Promoted =
1034 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
1035 if (!IsShift)
1036 Op2Promoted =
1037 matcher.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1038
1039 SDValue Result =
1040 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1041 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1042 }
1043
1044 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
1045 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
1046 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
1047 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
1048 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1049 SDValue Result =
1050 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1051 Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
1052 Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
1053 return Result;
1054}
1055
1056SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
1057 // Can just promote the operands then continue with operation.
1058 SDLoc dl(N);
1059 SDValue Op1Promoted, Op2Promoted;
1060 bool Signed =
1061 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
1062 bool Saturating =
1063 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
1064 if (Signed) {
1065 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1066 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1067 } else {
1068 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1069 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1070 }
1071 EVT OldType = N->getOperand(0).getValueType();
1072 EVT PromotedType = Op1Promoted.getValueType();
1073 unsigned DiffSize =
1074 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
1075
1076 if (Saturating) {
1077 // Promoting the operand and result values changes the saturation width,
1078 // which is extends the values that we clamp to on saturation. This could be
1079 // resolved by shifting one of the operands the same amount, which would
1080 // also shift the result we compare against, then shifting back.
1081 Op1Promoted =
1082 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1083 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1084 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1085 Op2Promoted, N->getOperand(2));
1086 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
1087 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
1088 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1089 }
1090 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1091 N->getOperand(2));
1092}
1093
1095 unsigned SatW, bool Signed,
1096 const TargetLowering &TLI,
1097 SelectionDAG &DAG) {
1098 EVT VT = V.getValueType();
1099 unsigned VTW = VT.getScalarSizeInBits();
1100
1101 if (!Signed) {
1102 // Saturate to the unsigned maximum by getting the minimum of V and the
1103 // maximum.
1104 return DAG.getNode(ISD::UMIN, dl, VT, V,
1105 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
1106 dl, VT));
1107 }
1108
1109 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
1110 // signed minimum of it and V.
1111 V = DAG.getNode(ISD::SMIN, dl, VT, V,
1112 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
1113 dl, VT));
1114 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
1115 // signed maximum of it and V.
1116 V = DAG.getNode(ISD::SMAX, dl, VT, V,
1117 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
1118 dl, VT));
1119 return V;
1120}
1121
1123 unsigned Scale, const TargetLowering &TLI,
1124 SelectionDAG &DAG, unsigned SatW = 0) {
1125 EVT VT = LHS.getValueType();
1126 unsigned VTSize = VT.getScalarSizeInBits();
1127 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1128 N->getOpcode() == ISD::SDIVFIXSAT;
1129 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1130 N->getOpcode() == ISD::UDIVFIXSAT;
1131
1132 SDLoc dl(N);
1133 // Widen the types by a factor of two. This is guaranteed to expand, since it
1134 // will always have enough high bits in the LHS to shift into.
1135 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
1136 if (VT.isVector())
1137 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
1139 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT);
1140 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT);
1141 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
1142 DAG);
1143 assert(Res && "Expanding DIVFIX with wide type failed?");
1144 if (Saturating) {
1145 // If the caller has told us to saturate at something less, use that width
1146 // instead of the type before doubling. However, it cannot be more than
1147 // what we just widened!
1148 assert(SatW <= VTSize &&
1149 "Tried to saturate to more than the original type?");
1150 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1151 TLI, DAG);
1152 }
1153 return DAG.getZExtOrTrunc(Res, dl, VT);
1154}
1155
1156SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1157 SDLoc dl(N);
1158 SDValue Op1Promoted, Op2Promoted;
1159 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1160 N->getOpcode() == ISD::SDIVFIXSAT;
1161 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1162 N->getOpcode() == ISD::UDIVFIXSAT;
1163 if (Signed) {
1164 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1165 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1166 } else {
1167 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1168 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1169 }
1170 EVT PromotedType = Op1Promoted.getValueType();
1171 unsigned Scale = N->getConstantOperandVal(2);
1172
1173 // If the type is already legal and the operation is legal in that type, we
1174 // should not early expand.
1175 if (TLI.isTypeLegal(PromotedType)) {
1177 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1178 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1179 unsigned Diff = PromotedType.getScalarSizeInBits() -
1180 N->getValueType(0).getScalarSizeInBits();
1181 if (Saturating)
1182 Op1Promoted =
1183 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1184 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1185 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1186 Op2Promoted, N->getOperand(2));
1187 if (Saturating)
1188 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1189 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1190 return Res;
1191 }
1192 }
1193
1194 // See if we can perform the division in this type without expanding.
1195 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1196 Op2Promoted, Scale, DAG)) {
1197 if (Saturating)
1198 Res = SaturateWidenedDIVFIX(Res, dl,
1199 N->getValueType(0).getScalarSizeInBits(),
1200 Signed, TLI, DAG);
1201 return Res;
1202 }
1203 // If we cannot, expand it to twice the type width. If we are saturating, give
1204 // it the original width as a saturating width so we don't need to emit
1205 // two saturations.
1206 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1207 N->getValueType(0).getScalarSizeInBits());
1208}
1209
1210SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1211 if (ResNo == 1)
1212 return PromoteIntRes_Overflow(N);
1213
1214 // The operation overflowed iff the result in the larger type is not the
1215 // sign extension of its truncation to the original type.
1216 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1217 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1218 EVT OVT = N->getOperand(0).getValueType();
1219 EVT NVT = LHS.getValueType();
1220 SDLoc dl(N);
1221
1222 // Do the arithmetic in the larger type.
1223 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1224 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1225
1226 // Calculate the overflow flag: sign extend the arithmetic result from
1227 // the original type.
1228 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1229 DAG.getValueType(OVT));
1230 // Overflowed if and only if this is not equal to Res.
1231 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1232
1233 // Use the calculated overflow everywhere.
1234 ReplaceValueWith(SDValue(N, 1), Ofl);
1235
1236 return Res;
1237}
1238
1239SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1240 SDValue Mask = N->getOperand(0);
1241
1242 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1243 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1244
1245 unsigned Opcode = N->getOpcode();
1246 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1247 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
1248 N->getOperand(3));
1249 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
1250}
1251
1252SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1253 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1254 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1255 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1256 LHS.getValueType(), N->getOperand(0),
1257 N->getOperand(1), LHS, RHS, N->getOperand(4));
1258}
1259
1260SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1261 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1262 EVT InVT = N->getOperand(OpNo).getValueType();
1263 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1264
1265 EVT SVT = getSetCCResultType(InVT);
1266
1267 // If we got back a type that needs to be promoted, this likely means the
1268 // the input type also needs to be promoted. So get the promoted type for
1269 // the input and try the query again.
1270 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1271 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1272 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1273 SVT = getSetCCResultType(InVT);
1274 } else {
1275 // Input type isn't promoted, just use the default promoted type.
1276 SVT = NVT;
1277 }
1278 }
1279
1280 SDLoc dl(N);
1281 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1282 "Vector compare must return a vector result!");
1283
1284 // Get the SETCC result using the canonical SETCC type.
1285 SDValue SetCC;
1286 if (N->isStrictFPOpcode()) {
1287 SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
1288 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1289 N->getOperand(2), N->getOperand(3)};
1290 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
1291 // Legalize the chain result - switch anything that used the old chain to
1292 // use the new one.
1293 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1294 } else
1295 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1296 N->getOperand(1), N->getOperand(2), N->getFlags());
1297
1298 // Convert to the expected type.
1299 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1300}
1301
1302SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1303 SDLoc DL(N);
1304 SDValue Arg = N->getOperand(0);
1305 SDValue Test = N->getOperand(1);
1306 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1307 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
1308}
1309
1310SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
1311 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1312 EVT VT = N->getValueType(0);
1313
1314 SDLoc dl(N);
1315 SDValue Res =
1316 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0));
1317
1318 ReplaceValueWith(SDValue(N, 0), Res);
1319 return Res.getValue(1);
1320}
1321
1322SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1323 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1324 SDValue RHS = N->getOperand(1);
1325 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1326 RHS = ZExtPromotedInteger(RHS);
1327 if (N->getOpcode() != ISD::VP_SHL)
1328 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1329 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1330 N->getOperand(2), N->getOperand(3));
1331}
1332
1333SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1334 SDValue Op = GetPromotedInteger(N->getOperand(0));
1336 Op.getValueType(), Op, N->getOperand(1));
1337}
1338
1339SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1340 // The input may have strange things in the top bits of the registers, but
1341 // these operations don't care. They may have weird bits going out, but
1342 // that too is okay if they are integer operations.
1343 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1344 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1345 if (N->getNumOperands() == 2)
1346 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1347 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1348 assert(N->isVPOpcode() && "Expected VP opcode");
1349 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1350 N->getOperand(2), N->getOperand(3));
1351}
1352
1353SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1354 // Sign extend the input.
1355 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1356 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1357 if (N->getNumOperands() == 2)
1358 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1359 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1360 assert(N->isVPOpcode() && "Expected VP opcode");
1361 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1362 N->getOperand(2), N->getOperand(3));
1363}
1364
1365SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1366 // Zero extend the input.
1367 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1368 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1369 if (N->getNumOperands() == 2)
1370 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1371 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1372 assert(N->isVPOpcode() && "Expected VP opcode");
1373 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1374 N->getOperand(2), N->getOperand(3));
1375}
1376
1377SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1378 SDValue LHS = N->getOperand(0);
1379 SDValue RHS = N->getOperand(1);
1380
1381 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1382 // whatever is best for the target and the promoted operands.
1383 SExtOrZExtPromotedOperands(LHS, RHS);
1384
1385 return DAG.getNode(N->getOpcode(), SDLoc(N),
1386 LHS.getValueType(), LHS, RHS);
1387}
1388
1389SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1390 // The input value must be properly sign extended.
1391 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1392 SDValue RHS = N->getOperand(1);
1393 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1394 RHS = ZExtPromotedInteger(RHS);
1395 if (N->getOpcode() != ISD::VP_ASHR)
1396 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1397 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1398 N->getOperand(2), N->getOperand(3));
1399}
1400
1401SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1402 // The input value must be properly zero extended.
1403 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1404 SDValue RHS = N->getOperand(1);
1405 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1406 RHS = ZExtPromotedInteger(RHS);
1407 if (N->getOpcode() != ISD::VP_LSHR)
1408 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1409 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1410 N->getOperand(2), N->getOperand(3));
1411}
1412
1413SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1414 // Lower the rotate to shifts and ORs which can be promoted.
1415 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
1416 ReplaceValueWith(SDValue(N, 0), Res);
1417 return SDValue();
1418}
1419
1420SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1421 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1422 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1423 SDValue Amt = N->getOperand(2);
1424 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1425 Amt = ZExtPromotedInteger(Amt);
1426 EVT AmtVT = Amt.getValueType();
1427
1428 SDLoc DL(N);
1429 EVT OldVT = N->getOperand(0).getValueType();
1430 EVT VT = Lo.getValueType();
1431 unsigned Opcode = N->getOpcode();
1432 bool IsFSHR = Opcode == ISD::FSHR;
1433 unsigned OldBits = OldVT.getScalarSizeInBits();
1434 unsigned NewBits = VT.getScalarSizeInBits();
1435
1436 // Amount has to be interpreted modulo the old bit width.
1437 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1438 DAG.getConstant(OldBits, DL, AmtVT));
1439
1440 // If the promoted type is twice the size (or more), then we use the
1441 // traditional funnel 'double' shift codegen. This isn't necessary if the
1442 // shift amount is constant.
1443 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1444 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1445 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1446 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1447 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1448 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1449 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1450 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1451 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1452 if (!IsFSHR)
1453 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1454 return Res;
1455 }
1456
1457 // Shift Lo up to occupy the upper bits of the promoted type.
1458 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1459 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1460
1461 // Increase Amount to shift the result into the lower bits of the promoted
1462 // type.
1463 if (IsFSHR)
1464 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset);
1465
1466 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1467}
1468
1469// A vp version of PromoteIntRes_FunnelShift.
1470SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) {
1471 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1472 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1473 SDValue Amt = N->getOperand(2);
1474 SDValue Mask = N->getOperand(3);
1475 SDValue EVL = N->getOperand(4);
1476 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1477 Amt = ZExtPromotedInteger(Amt);
1478 EVT AmtVT = Amt.getValueType();
1479
1480 SDLoc DL(N);
1481 EVT OldVT = N->getOperand(0).getValueType();
1482 EVT VT = Lo.getValueType();
1483 unsigned Opcode = N->getOpcode();
1484 bool IsFSHR = Opcode == ISD::VP_FSHR;
1485 unsigned OldBits = OldVT.getScalarSizeInBits();
1486 unsigned NewBits = VT.getScalarSizeInBits();
1487
1488 // Amount has to be interpreted modulo the old bit width.
1489 Amt = DAG.getNode(ISD::VP_UREM, DL, AmtVT, Amt,
1490 DAG.getConstant(OldBits, DL, AmtVT), Mask, EVL);
1491
1492 // If the promoted type is twice the size (or more), then we use the
1493 // traditional funnel 'double' shift codegen. This isn't necessary if the
1494 // shift amount is constant.
1495 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1496 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1497 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1498 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1499 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1500 Hi = DAG.getNode(ISD::VP_SHL, DL, VT, Hi, HiShift, Mask, EVL);
1501 // FIXME: Replace it by vp operations.
1502 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1503 SDValue Res = DAG.getNode(ISD::VP_OR, DL, VT, Hi, Lo, Mask, EVL);
1504 Res = DAG.getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL, DL, VT, Res, Amt,
1505 Mask, EVL);
1506 if (!IsFSHR)
1507 Res = DAG.getNode(ISD::VP_LSHR, DL, VT, Res, HiShift, Mask, EVL);
1508 return Res;
1509 }
1510
1511 // Shift Lo up to occupy the upper bits of the promoted type.
1512 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1513 Lo = DAG.getNode(ISD::VP_SHL, DL, VT, Lo, ShiftOffset, Mask, EVL);
1514
1515 // Increase Amount to shift the result into the lower bits of the promoted
1516 // type.
1517 if (IsFSHR)
1518 Amt = DAG.getNode(ISD::VP_ADD, DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1519
1520 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt, Mask, EVL);
1521}
1522
1523SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1524 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1525 SDValue Res;
1526 SDValue InOp = N->getOperand(0);
1527 SDLoc dl(N);
1528
1529 switch (getTypeAction(InOp.getValueType())) {
1530 default: llvm_unreachable("Unknown type action!");
1533 Res = InOp;
1534 break;
1536 Res = GetPromotedInteger(InOp);
1537 break;
1539 EVT InVT = InOp.getValueType();
1540 assert(InVT.isVector() && "Cannot split scalar types");
1541 ElementCount NumElts = InVT.getVectorElementCount();
1542 assert(NumElts == NVT.getVectorElementCount() &&
1543 "Dst and Src must have the same number of elements");
1545 "Promoted vector type must be a power of two");
1546
1547 SDValue EOp1, EOp2;
1548 GetSplitVector(InOp, EOp1, EOp2);
1549
1550 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1551 NumElts.divideCoefficientBy(2));
1552 if (N->getOpcode() == ISD::TRUNCATE) {
1553 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1554 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1555 } else {
1556 assert(N->getOpcode() == ISD::VP_TRUNCATE &&
1557 "Expected VP_TRUNCATE opcode");
1558 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1559 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
1560 std::tie(EVLLo, EVLHi) =
1561 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
1562 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1563 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1564 }
1565 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1566 }
1567 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
1568 // targets.
1570 SDValue WideInOp = GetWidenedVector(InOp);
1571
1572 // Truncate widened InOp.
1573 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1574 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1575 N->getValueType(0).getScalarType(), NumElem);
1576 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1577
1578 // Zero extend so that the elements are of same type as those of NVT
1580 NumElem);
1581 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1582
1583 // Extract the low NVT subvector.
1584 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1585 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1586 }
1587 }
1588
1589 // Truncate to NVT instead of VT
1590 if (N->getOpcode() == ISD::VP_TRUNCATE)
1591 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1),
1592 N->getOperand(2));
1593 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1594}
1595
1596SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1597 if (ResNo == 1)
1598 return PromoteIntRes_Overflow(N);
1599
1600 // The operation overflowed iff the result in the larger type is not the
1601 // zero extension of its truncation to the original type.
1602 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1603 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1604 EVT OVT = N->getOperand(0).getValueType();
1605 EVT NVT = LHS.getValueType();
1606 SDLoc dl(N);
1607
1608 // Do the arithmetic in the larger type.
1609 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1610 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1611
1612 // Calculate the overflow flag: zero extend the arithmetic result from
1613 // the original type.
1614 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1615 // Overflowed if and only if this is not equal to Res.
1616 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1617
1618 // Use the calculated overflow everywhere.
1619 ReplaceValueWith(SDValue(N, 1), Ofl);
1620
1621 return Res;
1622}
1623
1624// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
1625// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1626// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
1627SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
1628 unsigned ResNo) {
1629 if (ResNo == 1)
1630 return PromoteIntRes_Overflow(N);
1631
1632 // We need to sign-extend the operands so the carry value computed by the
1633 // wide operation will be equivalent to the carry value computed by the
1634 // narrow operation.
1635 // An UADDO_CARRY can generate carry only if any of the operands has its
1636 // most significant bit set. Sign extension propagates the most significant
1637 // bit into the higher bits which means the extra bit that the narrow
1638 // addition would need (i.e. the carry) will be propagated through the higher
1639 // bits of the wide addition.
1640 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
1641 // be preserved by sign extension.
1642 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1643 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1644
1645 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1646
1647 // Do the arithmetic in the wide type.
1648 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1649 LHS, RHS, N->getOperand(2));
1650
1651 // Update the users of the original carry/borrow value.
1652 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1653
1654 return SDValue(Res.getNode(), 0);
1655}
1656
1657SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1658 unsigned ResNo) {
1659 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1660 return PromoteIntRes_Overflow(N);
1661}
1662
1663SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1664 EVT OVT = N->getValueType(0);
1665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1666
1667 // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1668 // If we expand later we'll end up sign extending more than just the sra input
1669 // in sra+xor+sub expansion.
1670 if (!OVT.isVector() &&
1672 !TLI.isOperationLegal(ISD::SMAX, NVT)) {
1673 if (SDValue Res = TLI.expandABS(N, DAG))
1674 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
1675 }
1676
1677 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1678 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1679}
1680
1681SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1682 // Promote the overflow bit trivially.
1683 if (ResNo == 1)
1684 return PromoteIntRes_Overflow(N);
1685
1686 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1687 SDLoc DL(N);
1688 EVT SmallVT = LHS.getValueType();
1689
1690 // To determine if the result overflowed in a larger type, we extend the
1691 // input to the larger type, do the multiply (checking if it overflows),
1692 // then also check the high bits of the result to see if overflow happened
1693 // there.
1694 if (N->getOpcode() == ISD::SMULO) {
1695 LHS = SExtPromotedInteger(LHS);
1696 RHS = SExtPromotedInteger(RHS);
1697 } else {
1698 LHS = ZExtPromotedInteger(LHS);
1699 RHS = ZExtPromotedInteger(RHS);
1700 }
1701 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1702 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1703
1704 // Overflow occurred if it occurred in the larger type, or if the high part
1705 // of the result does not zero/sign-extend the low part. Check this second
1706 // possibility first.
1707 SDValue Overflow;
1708 if (N->getOpcode() == ISD::UMULO) {
1709 // Unsigned overflow occurred if the high part is non-zero.
1710 unsigned Shift = SmallVT.getScalarSizeInBits();
1711 SDValue Hi =
1712 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1713 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
1714 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1715 DAG.getConstant(0, DL, Hi.getValueType()),
1716 ISD::SETNE);
1717 } else {
1718 // Signed overflow occurred if the high part does not sign extend the low.
1719 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1720 Mul, DAG.getValueType(SmallVT));
1721 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1722 }
1723
1724 // The only other way for overflow to occur is if the multiplication in the
1725 // larger type itself overflowed.
1726 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1727 SDValue(Mul.getNode(), 1));
1728
1729 // Use the calculated overflow everywhere.
1730 ReplaceValueWith(SDValue(N, 1), Overflow);
1731 return Mul;
1732}
1733
1734SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1735 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1736 N->getValueType(0)));
1737}
1738
1739SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1740 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1741
1742 const APInt &MulImm = N->getConstantOperandAPInt(0);
1743 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
1744}
1745
1746SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1747 SDValue Chain = N->getOperand(0); // Get the chain.
1748 SDValue Ptr = N->getOperand(1); // Get the pointer.
1749 EVT VT = N->getValueType(0);
1750 SDLoc dl(N);
1751
1752 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1753 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1754 // The argument is passed as NumRegs registers of type RegVT.
1755
1756 SmallVector<SDValue, 8> Parts(NumRegs);
1757 for (unsigned i = 0; i < NumRegs; ++i) {
1758 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1759 N->getConstantOperandVal(3));
1760 Chain = Parts[i].getValue(1);
1761 }
1762
1763 // Handle endianness of the load.
1764 if (DAG.getDataLayout().isBigEndian())
1765 std::reverse(Parts.begin(), Parts.end());
1766
1767 // Assemble the parts in the promoted type.
1768 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1769 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1770 for (unsigned i = 1; i < NumRegs; ++i) {
1771 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1772 // Shift it to the right position and "or" it in.
1773 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1774 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1775 TLI.getPointerTy(DAG.getDataLayout())));
1776 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1777 }
1778
1779 // Modified the chain result - switch anything that used the old chain to
1780 // use the new one.
1781 ReplaceValueWith(SDValue(N, 1), Chain);
1782
1783 return Res;
1784}
1785
1786//===----------------------------------------------------------------------===//
1787// Integer Operand Promotion
1788//===----------------------------------------------------------------------===//
1789
1790/// PromoteIntegerOperand - This method is called when the specified operand of
1791/// the specified node is found to need promotion. At this point, all of the
1792/// result types of the node are known to be legal, but other operands of the
1793/// node may need promotion or expansion as well as the specified one.
1794bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1795 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
1796 SDValue Res = SDValue();
1797 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1798 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1799 return false;
1800 }
1801
1802 switch (N->getOpcode()) {
1803 default:
1804 #ifndef NDEBUG
1805 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1806 N->dump(&DAG); dbgs() << "\n";
1807 #endif
1808 report_fatal_error("Do not know how to promote this operator's operand!");
1809
1810 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1811 case ISD::ATOMIC_STORE:
1812 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1813 break;
1814 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1815 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1816 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1817 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1818 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1819 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1820 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1822 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
1823 break;
1824 case ISD::SPLAT_VECTOR:
1826 Res = PromoteIntOp_ScalarOp(N);
1827 break;
1828 case ISD::VSELECT:
1829 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1830 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1831 case ISD::VP_SETCC:
1832 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1833 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1834 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break;
1835 case ISD::VP_SINT_TO_FP:
1836 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1837 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1838 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1839 OpNo); break;
1840 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1841 OpNo); break;
1842 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1843 OpNo); break;
1844 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1845 OpNo); break;
1846 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1847 OpNo); break;
1848 case ISD::VP_TRUNCATE:
1849 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1850 case ISD::BF16_TO_FP:
1851 case ISD::FP16_TO_FP:
1852 case ISD::VP_UINT_TO_FP:
1853 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1855 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1856 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1857 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break;
1858 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1859 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1860
1861 case ISD::SHL:
1862 case ISD::SRA:
1863 case ISD::SRL:
1864 case ISD::ROTL:
1865 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1866
1867 case ISD::FSHL:
1868 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
1869
1870 case ISD::SADDO_CARRY:
1871 case ISD::SSUBO_CARRY:
1872 case ISD::UADDO_CARRY:
1873 case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break;
1874
1875 case ISD::FRAMEADDR:
1876 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1877
1878 case ISD::SMULFIX:
1879 case ISD::SMULFIXSAT:
1880 case ISD::UMULFIX:
1881 case ISD::UMULFIXSAT:
1882 case ISD::SDIVFIX:
1883 case ISD::SDIVFIXSAT:
1884 case ISD::UDIVFIX:
1885 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1886 case ISD::FPOWI:
1887 case ISD::STRICT_FPOWI:
1888 case ISD::FLDEXP:
1889 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
1890 case ISD::VECREDUCE_ADD:
1891 case ISD::VECREDUCE_MUL:
1892 case ISD::VECREDUCE_AND:
1893 case ISD::VECREDUCE_OR:
1894 case ISD::VECREDUCE_XOR:
1898 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1899 case ISD::VP_REDUCE_ADD:
1900 case ISD::VP_REDUCE_MUL:
1901 case ISD::VP_REDUCE_AND:
1902 case ISD::VP_REDUCE_OR:
1903 case ISD::VP_REDUCE_XOR:
1904 case ISD::VP_REDUCE_SMAX:
1905 case ISD::VP_REDUCE_SMIN:
1906 case ISD::VP_REDUCE_UMAX:
1907 case ISD::VP_REDUCE_UMIN:
1908 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1909 break;
1910
1911 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1912 case ISD::STACKMAP:
1913 Res = PromoteIntOp_STACKMAP(N, OpNo);
1914 break;
1915 case ISD::PATCHPOINT:
1916 Res = PromoteIntOp_PATCHPOINT(N, OpNo);
1917 break;
1918 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1919 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1920 Res = PromoteIntOp_VP_STRIDED(N, OpNo);
1921 break;
1922 case ISD::EXPERIMENTAL_VP_SPLICE:
1923 Res = PromoteIntOp_VP_SPLICE(N, OpNo);
1924 break;
1925 }
1926
1927 // If the result is null, the sub-method took care of registering results etc.
1928 if (!Res.getNode()) return false;
1929
1930 // If the result is N, the sub-method updated N in place. Tell the legalizer
1931 // core about this.
1932 if (Res.getNode() == N)
1933 return true;
1934
1935 const bool IsStrictFp = N->isStrictFPOpcode();
1936 assert(Res.getValueType() == N->getValueType(0) &&
1937 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1938 "Invalid operand expansion");
1939 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1940 Res.dump());
1941
1942 ReplaceValueWith(SDValue(N, 0), Res);
1943 if (IsStrictFp)
1944 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1945
1946 return false;
1947}
1948
1949// These operands can be either sign extended or zero extended as long as we
1950// treat them the same. If an extension is free, choose that. Otherwise, follow
1951// target preference.
1952void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) {
1953 SDValue OpL = GetPromotedInteger(LHS);
1954 SDValue OpR = GetPromotedInteger(RHS);
1955
1956 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
1957 // The target would prefer to promote the comparison operand with sign
1958 // extension. Honor that unless the promoted values are already zero
1959 // extended.
1960 unsigned OpLEffectiveBits =
1962 unsigned OpREffectiveBits =
1964 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1965 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1966 LHS = OpL;
1967 RHS = OpR;
1968 return;
1969 }
1970
1971 // The promoted values aren't zero extended, use a sext_inreg.
1972 LHS = SExtPromotedInteger(LHS);
1973 RHS = SExtPromotedInteger(RHS);
1974 return;
1975 }
1976
1977 // Prefer to promote the comparison operand with zero extension.
1978
1979 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1980 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
1981 // that we might not be able to remove.
1982 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
1983 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
1984 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1985 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1986 LHS = OpL;
1987 RHS = OpR;
1988 return;
1989 }
1990
1991 // Otherwise, use zext_inreg.
1992 LHS = ZExtPromotedInteger(LHS);
1993 RHS = ZExtPromotedInteger(RHS);
1994}
1995
1996/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1997/// shared among BR_CC, SELECT_CC, and SETCC handlers.
1998void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
1999 ISD::CondCode CCCode) {
2000 // We have to insert explicit sign or zero extends. Note that we could
2001 // insert sign extends for ALL conditions. For those operations where either
2002 // zero or sign extension would be valid, we ask the target which extension
2003 // it would prefer.
2004
2005 // Signed comparisons always require sign extension.
2006 if (ISD::isSignedIntSetCC(CCCode)) {
2007 LHS = SExtPromotedInteger(LHS);
2008 RHS = SExtPromotedInteger(RHS);
2009 return;
2010 }
2011
2013 "Unknown integer comparison!");
2014
2015 SExtOrZExtPromotedOperands(LHS, RHS);
2016}
2017
2018SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
2019 SDValue Op = GetPromotedInteger(N->getOperand(0));
2020 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
2021}
2022
2023SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
2024 SDValue Op1 = GetPromotedInteger(N->getOperand(1));
2025 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
2026 N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
2027}
2028
2029SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
2030 // This should only occur in unusual situations like bitcasting to an
2031 // x86_fp80, so just turn it into a store+load
2032 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
2033}
2034
2035SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
2036 assert(OpNo == 2 && "Don't know how to promote this operand!");
2037
2038 SDValue LHS = N->getOperand(2);
2039 SDValue RHS = N->getOperand(3);
2040 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
2041
2042 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
2043 // legal types.
2044 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2045 N->getOperand(1), LHS, RHS, N->getOperand(4)),
2046 0);
2047}
2048
2049SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
2050 assert(OpNo == 1 && "only know how to promote condition");
2051
2052 // Promote all the way up to the canonical SetCC type.
2053 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
2054
2055 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
2056 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
2057 N->getOperand(2)), 0);
2058}
2059
2060SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
2061 // Since the result type is legal, the operands must promote to it.
2062 EVT OVT = N->getOperand(0).getValueType();
2063 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
2064 SDValue Hi = GetPromotedInteger(N->getOperand(1));
2065 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
2066 SDLoc dl(N);
2067
2068 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
2069 DAG.getConstant(OVT.getSizeInBits(), dl,
2070 TLI.getPointerTy(DAG.getDataLayout())));
2071 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
2072}
2073
2074SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
2075 // The vector type is legal but the element type is not. This implies
2076 // that the vector is a power-of-two in length and that the element
2077 // type does not have a strange size (eg: it is not i1).
2078 EVT VecVT = N->getValueType(0);
2079 unsigned NumElts = VecVT.getVectorNumElements();
2080 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2081 "Legal vector of one illegal element?");
2082
2083 // Promote the inserted value. The type does not need to match the
2084 // vector element type. Check that any extra bits introduced will be
2085 // truncated away.
2086 assert(N->getOperand(0).getValueSizeInBits() >=
2087 N->getValueType(0).getScalarSizeInBits() &&
2088 "Type of inserted value narrower than vector element type!");
2089
2091 for (unsigned i = 0; i < NumElts; ++i)
2092 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
2093
2094 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2095}
2096
2097SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
2098 unsigned OpNo) {
2099 if (OpNo == 1) {
2100 // Promote the inserted value. This is valid because the type does not
2101 // have to match the vector element type.
2102
2103 // Check that any extra bits introduced will be truncated away.
2104 assert(N->getOperand(1).getValueSizeInBits() >=
2105 N->getValueType(0).getScalarSizeInBits() &&
2106 "Type of inserted value narrower than vector element type!");
2107 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2108 GetPromotedInteger(N->getOperand(1)),
2109 N->getOperand(2)),
2110 0);
2111 }
2112
2113 assert(OpNo == 2 && "Different operand and result vector types?");
2114
2115 // Promote the index.
2116 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
2117 TLI.getVectorIdxTy(DAG.getDataLayout()));
2118 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2119 N->getOperand(1), Idx), 0);
2120}
2121
2122SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2123 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
2124 // so just promote the operand in place.
2125 return SDValue(DAG.UpdateNodeOperands(N,
2126 GetPromotedInteger(N->getOperand(0))), 0);
2127}
2128
2129SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
2130 assert(OpNo == 0 && "Only know how to promote the condition!");
2131 SDValue Cond = N->getOperand(0);
2132 EVT OpTy = N->getOperand(1).getValueType();
2133
2134 if (N->getOpcode() == ISD::VSELECT)
2135 if (SDValue Res = WidenVSELECTMask(N))
2136 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2137 Res, N->getOperand(1), N->getOperand(2));
2138
2139 // Promote all the way up to the canonical SetCC type.
2140 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
2141 Cond = PromoteTargetBoolean(Cond, OpVT);
2142
2143 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
2144 N->getOperand(2)), 0);
2145}
2146
2147SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2148 assert(OpNo == 0 && "Don't know how to promote this operand!");
2149
2150 SDValue LHS = N->getOperand(0);
2151 SDValue RHS = N->getOperand(1);
2152 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
2153
2154 // The CC (#4) and the possible return values (#2 and #3) have legal types.
2155 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2156 N->getOperand(3), N->getOperand(4)), 0);
2157}
2158
2159SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
2160 assert(OpNo == 0 && "Don't know how to promote this operand!");
2161
2162 SDValue LHS = N->getOperand(0);
2163 SDValue RHS = N->getOperand(1);
2164 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
2165
2166 // The CC (#2) is always legal.
2167 if (N->getOpcode() == ISD::SETCC)
2168 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
2169
2170 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2171
2172 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2173 N->getOperand(3), N->getOperand(4)),
2174 0);
2175}
2176
2177SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
2178 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2179 ZExtPromotedInteger(N->getOperand(1))), 0);
2180}
2181
2182SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
2183 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2184 ZExtPromotedInteger(N->getOperand(2))), 0);
2185}
2186
2187SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
2188 SDValue Op = GetPromotedInteger(N->getOperand(0));
2189 SDLoc dl(N);
2190 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2191 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
2192 Op, DAG.getValueType(N->getOperand(0).getValueType()));
2193}
2194
2195SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) {
2196 SDLoc dl(N);
2197 EVT VT = N->getValueType(0);
2198 SDValue Op = GetPromotedInteger(N->getOperand(0));
2199 // FIXME: There is no VP_ANY_EXTEND yet.
2200 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2201 N->getOperand(2));
2202 unsigned Diff =
2203 VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits();
2204 SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl);
2205 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts.
2206 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1),
2207 N->getOperand(2));
2208 return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1),
2209 N->getOperand(2));
2210}
2211
2212SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2213 if (N->getOpcode() == ISD::VP_SINT_TO_FP)
2214 return SDValue(DAG.UpdateNodeOperands(N,
2215 SExtPromotedInteger(N->getOperand(0)),
2216 N->getOperand(1), N->getOperand(2)),
2217 0);
2218 return SDValue(DAG.UpdateNodeOperands(N,
2219 SExtPromotedInteger(N->getOperand(0))), 0);
2220}
2221
2222SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2223 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2224 SExtPromotedInteger(N->getOperand(1))), 0);
2225}
2226
2227SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2228 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2229 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2230 SDLoc dl(N);
2231
2232 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
2233
2234 // Truncate the value and store the result.
2235 return DAG.getTruncStore(Ch, dl, Val, Ptr,
2236 N->getMemoryVT(), N->getMemOperand());
2237}
2238
2239SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2240 unsigned OpNo) {
2241 SDValue DataOp = N->getValue();
2242 SDValue Mask = N->getMask();
2243
2244 if (OpNo == 4) {
2245 // The Mask. Update in place.
2246 EVT DataVT = DataOp.getValueType();
2247 Mask = PromoteTargetBoolean(Mask, DataVT);
2248 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2249 NewOps[4] = Mask;
2250 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2251 }
2252
2253 assert(OpNo == 1 && "Unexpected operand for promotion");
2254 DataOp = GetPromotedInteger(DataOp);
2255
2256 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2257 N->getOffset(), Mask, N->getMemoryVT(),
2258 N->getMemOperand(), N->getAddressingMode(),
2259 /*IsTruncating*/ true, N->isCompressingStore());
2260}
2261
2262SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2263 unsigned OpNo) {
2264 assert(OpNo == 3 && "Only know how to promote the mask!");
2265 EVT DataVT = N->getValueType(0);
2266 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2267 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2268 NewOps[OpNo] = Mask;
2269 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2270 if (Res == N)
2271 return SDValue(Res, 0);
2272
2273 // Update triggered CSE, do our own replacement since caller can't.
2274 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2275 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2276 return SDValue();
2277}
2278
2279SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2280 unsigned OpNo) {
2281 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2282
2283 if (OpNo == 2) {
2284 // The Mask
2285 EVT DataVT = N->getValueType(0);
2286 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2287 } else if (OpNo == 4) {
2288 // The Index
2289 if (N->isIndexSigned())
2290 // Need to sign extend the index since the bits will likely be used.
2291 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2292 else
2293 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2294 } else
2295 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2296
2297 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2298 if (Res == N)
2299 return SDValue(Res, 0);
2300
2301 // Update triggered CSE, do our own replacement since caller can't.
2302 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2303 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2304 return SDValue();
2305}
2306
2307SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2308 unsigned OpNo) {
2309 bool TruncateStore = N->isTruncatingStore();
2310 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2311
2312 if (OpNo == 2) {
2313 // The Mask
2314 EVT DataVT = N->getValue().getValueType();
2315 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2316 } else if (OpNo == 4) {
2317 // The Index
2318 if (N->isIndexSigned())
2319 // Need to sign extend the index since the bits will likely be used.
2320 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2321 else
2322 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2323 } else {
2324 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2325 TruncateStore = true;
2326 }
2327
2328 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2329 SDLoc(N), NewOps, N->getMemOperand(),
2330 N->getIndexType(), TruncateStore);
2331}
2332
2333SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2334 SDValue Op = GetPromotedInteger(N->getOperand(0));
2335 if (N->getOpcode() == ISD::VP_TRUNCATE)
2336 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op,
2337 N->getOperand(1), N->getOperand(2));
2338 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2339}
2340
2341SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2342 if (N->getOpcode() == ISD::VP_UINT_TO_FP)
2343 return SDValue(DAG.UpdateNodeOperands(N,
2344 ZExtPromotedInteger(N->getOperand(0)),
2345 N->getOperand(1), N->getOperand(2)),
2346 0);
2347 return SDValue(DAG.UpdateNodeOperands(N,
2348 ZExtPromotedInteger(N->getOperand(0))), 0);
2349}
2350
2351SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2352 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2353 ZExtPromotedInteger(N->getOperand(1))), 0);
2354}
2355
2356SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2357 SDLoc dl(N);
2358 SDValue Op = GetPromotedInteger(N->getOperand(0));
2359 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2360 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2361}
2362
2363SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) {
2364 SDLoc dl(N);
2365 EVT VT = N->getValueType(0);
2366 SDValue Op = GetPromotedInteger(N->getOperand(0));
2367 // FIXME: There is no VP_ANY_EXTEND yet.
2368 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2369 N->getOperand(2));
2371 N->getOperand(0).getScalarValueSizeInBits());
2372 return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT),
2373 N->getOperand(1), N->getOperand(2));
2374}
2375
2376SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) {
2377 assert(OpNo == 2 && "Don't know how to promote this operand!");
2378
2379 SDValue LHS = N->getOperand(0);
2380 SDValue RHS = N->getOperand(1);
2381 SDValue Carry = N->getOperand(2);
2382 SDLoc DL(N);
2383
2384 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2385
2386 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2387}
2388
2389SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2390 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2391 return SDValue(
2392 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2393}
2394
2395SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2396 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2397 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2398 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2399}
2400
2401SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
2402 bool IsStrict = N->isStrictFPOpcode();
2403 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2404
2405 bool IsPowI =
2406 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
2407
2408 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result
2409 // and floating point operand is already type legalized).
2410 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
2411 : RTLIB::getLDEXP(N->getValueType(0));
2412
2413 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2414 SDValue Op = SExtPromotedInteger(N->getOperand(1));
2415 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2416 }
2417
2418 // We can't just promote the exponent type in FPOWI, since we want to lower
2419 // the node to a libcall and we if we promote to a type larger than
2420 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2421 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2422 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2423
2424 unsigned OpOffset = IsStrict ? 1 : 0;
2425 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2426 assert(DAG.getLibInfo().getIntSize() ==
2427 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2428 "POWI exponent should match with sizeof(int) when doing the libcall.");
2430 CallOptions.setSExt(true);
2431 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2432 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2433 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
2434 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2435 if (IsStrict)
2436 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2437 return SDValue();
2438}
2439
2441 switch (N->getOpcode()) {
2442 default:
2443 llvm_unreachable("Expected integer vector reduction");
2444 case ISD::VECREDUCE_ADD:
2445 case ISD::VECREDUCE_MUL:
2446 case ISD::VECREDUCE_AND:
2447 case ISD::VECREDUCE_OR:
2448 case ISD::VECREDUCE_XOR:
2449 case ISD::VP_REDUCE_ADD:
2450 case ISD::VP_REDUCE_MUL:
2451 case ISD::VP_REDUCE_AND:
2452 case ISD::VP_REDUCE_OR:
2453 case ISD::VP_REDUCE_XOR:
2454 return ISD::ANY_EXTEND;
2457 case ISD::VP_REDUCE_SMAX:
2458 case ISD::VP_REDUCE_SMIN:
2459 return ISD::SIGN_EXTEND;
2462 case ISD::VP_REDUCE_UMAX:
2463 case ISD::VP_REDUCE_UMIN:
2464 return ISD::ZERO_EXTEND;
2465 }
2466}
2467
2468SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2469 switch (getExtendForIntVecReduction(N)) {
2470 default:
2471 llvm_unreachable("Impossible extension kind for integer reduction");
2472 case ISD::ANY_EXTEND:
2473 return GetPromotedInteger(V);
2474 case ISD::SIGN_EXTEND:
2475 return SExtPromotedInteger(V);
2476 case ISD::ZERO_EXTEND:
2477 return ZExtPromotedInteger(V);
2478 }
2479}
2480
2481SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2482 SDLoc dl(N);
2483 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2484
2485 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
2486 EVT InVT = Op.getValueType();
2487 EVT EltVT = InVT.getVectorElementType();
2488 EVT ResVT = N->getValueType(0);
2489 unsigned Opcode = N->getOpcode();
2490
2491 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2492 // vecreduce_xor is not legal
2493 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2496 Opcode = ISD::VECREDUCE_ADD;
2497
2498 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2499 // vecreduce_or is not legal
2500 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2503 Opcode = ISD::VECREDUCE_UMAX;
2504 // Can't use promoteTargetBoolean here because we still need
2505 // to either sign_ext or zero_ext in the undefined case.
2506 switch (TLI.getBooleanContents(InVT)) {
2509 Op = ZExtPromotedInteger(N->getOperand(0));
2510 break;
2512 Op = SExtPromotedInteger(N->getOperand(0));
2513 break;
2514 }
2515 }
2516
2517 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2518 // vecreduce_and is not legal
2519 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2522 Opcode = ISD::VECREDUCE_UMIN;
2523 // Can't use promoteTargetBoolean here because we still need
2524 // to either sign_ext or zero_ext in the undefined case.
2525 switch (TLI.getBooleanContents(InVT)) {
2528 Op = ZExtPromotedInteger(N->getOperand(0));
2529 break;
2531 Op = SExtPromotedInteger(N->getOperand(0));
2532 break;
2533 }
2534 }
2535
2536 if (ResVT.bitsGE(EltVT))
2537 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
2538
2539 // Result size must be >= element size. If this is not the case after
2540 // promotion, also promote the result type and then truncate.
2541 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
2542 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
2543}
2544
2545SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2546 SDLoc DL(N);
2547 SDValue Op = N->getOperand(OpNo);
2548 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2549
2550 if (OpNo == 2) { // Mask
2551 // Update in place.
2552 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2553 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2554 }
2555
2556 assert(OpNo == 1 && "Unexpected operand for promotion");
2557
2558 Op = PromoteIntOpVectorReduction(N, Op);
2559
2560 NewOps[OpNo] = Op;
2561
2562 EVT VT = N->getValueType(0);
2563 EVT EltVT = Op.getValueType().getScalarType();
2564
2565 if (VT.bitsGE(EltVT))
2566 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2567
2568 // Result size must be >= element/start-value size. If this is not the case
2569 // after promotion, also promote both the start value and result type and
2570 // then truncate.
2571 NewOps[0] =
2572 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2573 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2574 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2575}
2576
2577SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2578 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2579 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2580}
2581
2582SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
2583 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
2584 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2585 SDValue Operand = N->getOperand(OpNo);
2586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2587 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2588 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2589}
2590
2591SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
2592 assert(OpNo >= 7);
2593 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2594 SDValue Operand = N->getOperand(OpNo);
2595 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2596 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2597 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2598}
2599
2600SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
2601 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2602 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2603
2604 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
2605 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2606
2607 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2608}
2609
2610SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) {
2611 SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end());
2612
2613 if (OpNo == 2) { // Offset operand
2614 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2615 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2616 }
2617
2618 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion");
2619
2620 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2621 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2622}
2623
2624//===----------------------------------------------------------------------===//
2625// Integer Result Expansion
2626//===----------------------------------------------------------------------===//
2627
2628/// ExpandIntegerResult - This method is called when the specified result of the
2629/// specified node is found to need expansion. At this point, the node may also
2630/// have invalid operands or may have other results that need promotion, we just
2631/// know that (at least) one result needs expansion.
2632void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2633 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG));
2634 SDValue Lo, Hi;
2635 Lo = Hi = SDValue();
2636
2637 // See if the target wants to custom expand this node.
2638 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2639 return;
2640
2641 switch (N->getOpcode()) {
2642 default:
2643#ifndef NDEBUG
2644 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2645 N->dump(&DAG); dbgs() << "\n";
2646#endif
2647 report_fatal_error("Do not know how to expand the result of this "
2648 "operator!");
2649
2650 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
2651 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2652 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
2653 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2654 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2655 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2656
2657 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2658 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2659 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2660 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2661 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2662
2663 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2664 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2665 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2666 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2667 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2668 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2669 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2670 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2672 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2673 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2675 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2676 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break;
2678 case ISD::FP_TO_SINT:
2680 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break;
2682 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2683 case ISD::STRICT_LROUND:
2684 case ISD::STRICT_LRINT:
2685 case ISD::LROUND:
2686 case ISD::LRINT:
2688 case ISD::STRICT_LLRINT:
2689 case ISD::LLROUND:
2690 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break;
2691 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2692 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2694 case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break;
2695 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2696 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2697 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2698 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2699 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2700 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2701 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2702 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2703 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2704
2716 case ISD::ATOMIC_SWAP:
2717 case ISD::ATOMIC_CMP_SWAP: {
2718 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2719 SplitInteger(Tmp.first, Lo, Hi);
2720 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2721 break;
2722 }
2724 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2725 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2726 SDValue Tmp = DAG.getAtomicCmpSwap(
2728 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2729 AN->getMemOperand());
2730
2731 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2732 // success simply by comparing the loaded value against the ingoing
2733 // comparison.
2734 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2735 N->getOperand(2), ISD::SETEQ);
2736
2737 SplitInteger(Tmp, Lo, Hi);
2738 ReplaceValueWith(SDValue(N, 1), Success);
2739 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2740 break;
2741 }
2742
2743 case ISD::AND:
2744 case ISD::OR:
2745 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2746
2747 case ISD::UMAX:
2748 case ISD::SMAX:
2749 case ISD::UMIN:
2750 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2751
2752 case ISD::ADD:
2753 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2754
2755 case ISD::ADDC:
2756 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2757
2758 case ISD::ADDE:
2759 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2760
2761 case ISD::UADDO_CARRY:
2762 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
2763
2764 case ISD::SADDO_CARRY:
2765 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2766
2767 case ISD::SHL:
2768 case ISD::SRA:
2769 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2770
2771 case ISD::SADDO:
2772 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2773 case ISD::UADDO:
2774 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2775 case ISD::UMULO:
2776 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2777
2778 case ISD::SADDSAT:
2779 case ISD::UADDSAT:
2780 case ISD::SSUBSAT:
2781 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2782
2783 case ISD::SSHLSAT:
2784 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2785
2786 case ISD::SMULFIX:
2787 case ISD::SMULFIXSAT:
2788 case ISD::UMULFIX:
2789 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2790
2791 case ISD::SDIVFIX:
2792 case ISD::SDIVFIXSAT:
2793 case ISD::UDIVFIX:
2794 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2795
2796 case ISD::VECREDUCE_ADD:
2797 case ISD::VECREDUCE_MUL:
2798 case ISD::VECREDUCE_AND:
2799 case ISD::VECREDUCE_OR:
2800 case ISD::VECREDUCE_XOR:
2804 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2805
2806 case ISD::ROTL:
2807 case ISD::ROTR:
2808 ExpandIntRes_Rotate(N, Lo, Hi);
2809 break;
2810
2811 case ISD::FSHL:
2812 case ISD::FSHR:
2813 ExpandIntRes_FunnelShift(N, Lo, Hi);
2814 break;
2815
2816 case ISD::VSCALE:
2817 ExpandIntRes_VSCALE(N, Lo, Hi);
2818 break;
2819 }
2820
2821 // If Lo/Hi is null, the sub-method took care of registering results etc.
2822 if (Lo.getNode())
2823 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2824}
2825
2826/// Lower an atomic node to the appropriate builtin call.
2827std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2828 unsigned Opc = Node->getOpcode();
2829 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2830 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2831 // Lower to outline atomic libcall if outline atomics enabled,
2832 // or to sync libcall otherwise
2833 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2834 EVT RetVT = Node->getValueType(0);
2837 if (TLI.getLibcallName(LC)) {
2838 Ops.append(Node->op_begin() + 2, Node->op_end());
2839 Ops.push_back(Node->getOperand(1));
2840 } else {
2841 LC = RTLIB::getSYNC(Opc, VT);
2842 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2843 "Unexpected atomic op or value type!");
2844 Ops.append(Node->op_begin() + 1, Node->op_end());
2845 }
2846 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2847 Node->getOperand(0));
2848}
2849
2850/// N is a shift by a value that needs to be expanded,
2851/// and the shift amount is a constant 'Amt'. Expand the operation.
2852void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2853 SDValue &Lo, SDValue &Hi) {
2854 SDLoc DL(N);
2855 // Expand the incoming operand to be shifted, so that we have its parts
2856 SDValue InL, InH;
2857 GetExpandedInteger(N->getOperand(0), InL, InH);
2858
2859 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2860 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2861 if (!Amt) {
2862 Lo = InL;
2863 Hi = InH;
2864 return;
2865 }
2866
2867 EVT NVT = InL.getValueType();
2868 unsigned VTBits = N->getValueType(0).getSizeInBits();
2869 unsigned NVTBits = NVT.getSizeInBits();
2870
2871 if (N->getOpcode() == ISD::SHL) {
2872 if (Amt.uge(VTBits)) {
2873 Lo = Hi = DAG.getConstant(0, DL, NVT);
2874 } else if (Amt.ugt(NVTBits)) {
2875 Lo = DAG.getConstant(0, DL, NVT);
2876 Hi = DAG.getNode(ISD::SHL, DL, NVT, InL,
2877 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
2878 } else if (Amt == NVTBits) {
2879 Lo = DAG.getConstant(0, DL, NVT);
2880 Hi = InL;
2881 } else {
2882 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL,
2883 DAG.getShiftAmountConstant(Amt, NVT, DL));
2884 Hi = DAG.getNode(
2885 ISD::OR, DL, NVT,
2886 DAG.getNode(ISD::SHL, DL, NVT, InH,
2887 DAG.getShiftAmountConstant(Amt, NVT, DL)),
2888 DAG.getNode(ISD::SRL, DL, NVT, InL,
2889 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
2890 }
2891 return;
2892 }
2893
2894 if (N->getOpcode() == ISD::SRL) {
2895 if (Amt.uge(VTBits)) {
2896 Lo = Hi = DAG.getConstant(0, DL, NVT);
2897 } else if (Amt.ugt(NVTBits)) {
2898 Lo = DAG.getNode(ISD::SRL, DL, NVT, InH,
2899 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
2900 Hi = DAG.getConstant(0, DL, NVT);
2901 } else if (Amt == NVTBits) {
2902 Lo = InH;
2903 Hi = DAG.getConstant(0, DL, NVT);
2904 } else {
2905 Lo = DAG.getNode(
2906 ISD::OR, DL, NVT,
2907 DAG.getNode(ISD::SRL, DL, NVT, InL,
2908 DAG.getShiftAmountConstant(Amt, NVT, DL)),
2909 DAG.getNode(ISD::SHL, DL, NVT, InH,
2910 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
2911 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH,
2912 DAG.getShiftAmountConstant(Amt, NVT, DL));
2913 }
2914 return;
2915 }
2916
2917 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2918 if (Amt.uge(VTBits)) {
2919 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2920 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
2921 } else if (Amt.ugt(NVTBits)) {
2922 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2923 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
2924 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2925 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
2926 } else if (Amt == NVTBits) {
2927 Lo = InH;
2928 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2929 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
2930 } else {
2931 Lo = DAG.getNode(
2932 ISD::OR, DL, NVT,
2933 DAG.getNode(ISD::SRL, DL, NVT, InL,
2934 DAG.getShiftAmountConstant(Amt, NVT, DL)),
2935 DAG.getNode(ISD::SHL, DL, NVT, InH,
2936 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
2937 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2938 DAG.getShiftAmountConstant(Amt, NVT, DL));
2939 }
2940}
2941
2942/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2943/// this shift based on knowledge of the high bit of the shift amount. If we
2944/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2945/// shift amount.
2946bool DAGTypeLegalizer::
2947ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2948 unsigned Opc = N->getOpcode();
2949 SDValue In = N->getOperand(0);
2950 SDValue Amt = N->getOperand(1);
2951 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2952 EVT ShTy = Amt.getValueType();
2953 unsigned ShBits = ShTy.getScalarSizeInBits();
2954 unsigned NVTBits = NVT.getScalarSizeInBits();
2955 assert(isPowerOf2_32(NVTBits) &&
2956 "Expanded integer type size not a power of two!");
2957 SDLoc dl(N);
2958
2959 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2960 KnownBits Known = DAG.computeKnownBits(Amt);
2961
2962 // If we don't know anything about the high bits, exit.
2963 if (((Known.Zero | Known.One) & HighBitMask) == 0)
2964 return false;
2965
2966 // Get the incoming operand to be shifted.
2967 SDValue InL, InH;
2968 GetExpandedInteger(In, InL, InH);
2969
2970 // If we know that any of the high bits of the shift amount are one, then we
2971 // can do this as a couple of simple shifts.
2972 if (Known.One.intersects(HighBitMask)) {
2973 // Mask out the high bit, which we know is set.
2974 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2975 DAG.getConstant(~HighBitMask, dl, ShTy));
2976
2977 switch (Opc) {
2978 default: llvm_unreachable("Unknown shift");
2979 case ISD::SHL:
2980 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2981 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2982 return true;
2983 case ISD::SRL:
2984 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2985 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2986 return true;
2987 case ISD::SRA:
2988 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2989 DAG.getConstant(NVTBits - 1, dl, ShTy));
2990 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2991 return true;
2992 }
2993 }
2994
2995 // If we know that all of the high bits of the shift amount are zero, then we
2996 // can do this as a couple of simple shifts.
2997 if (HighBitMask.isSubsetOf(Known.Zero)) {
2998 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2999 // shift if x is zero. We can use XOR here because x is known to be smaller
3000 // than 32.
3001 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
3002 DAG.getConstant(NVTBits - 1, dl, ShTy));
3003
3004 unsigned Op1, Op2;
3005 switch (Opc) {
3006 default: llvm_unreachable("Unknown shift");
3007 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
3008 case ISD::SRL:
3009 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
3010 }
3011
3012 // When shifting right the arithmetic for Lo and Hi is swapped.
3013 if (Opc != ISD::SHL)
3014 std::swap(InL, InH);
3015
3016 // Use a little trick to get the bits that move from Lo to Hi. First
3017 // shift by one bit.
3018 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
3019 // Then compute the remaining shift with amount-1.
3020 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
3021
3022 Lo = DAG.getNode(Opc, dl, NVT, InL, Amt);
3023 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
3024
3025 if (Opc != ISD::SHL)
3026 std::swap(Hi, Lo);
3027 return true;
3028 }
3029
3030 return false;
3031}
3032
3033/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
3034/// of any size.
3035bool DAGTypeLegalizer::
3036ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3037 SDValue Amt = N->getOperand(1);
3038 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3039 EVT ShTy = Amt.getValueType();
3040 unsigned NVTBits = NVT.getSizeInBits();
3041 assert(isPowerOf2_32(NVTBits) &&
3042 "Expanded integer type size not a power of two!");
3043 SDLoc dl(N);
3044
3045 // Get the incoming operand to be shifted.
3046 SDValue InL, InH;
3047 GetExpandedInteger(N->getOperand(0), InL, InH);
3048
3049 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
3050 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
3051 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
3052 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3053 Amt, NVBitsNode, ISD::SETULT);
3054 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3055 Amt, DAG.getConstant(0, dl, ShTy),
3056 ISD::SETEQ);
3057
3058 SDValue LoS, HiS, LoL, HiL;
3059 switch (N->getOpcode()) {
3060 default: llvm_unreachable("Unknown shift");
3061 case ISD::SHL:
3062 // Short: ShAmt < NVTBits
3063 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
3064 HiS = DAG.getNode(ISD::OR, dl, NVT,
3065 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
3066 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
3067
3068 // Long: ShAmt >= NVTBits
3069 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
3070 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
3071
3072 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
3073 Hi = DAG.getSelect(dl, NVT, isZero, InH,
3074 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
3075 return true;
3076 case ISD::SRL:
3077 // Short: ShAmt < NVTBits
3078 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
3079 LoS = DAG.getNode(ISD::OR, dl, NVT,
3080 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3081 // FIXME: If Amt is zero, the following shift generates an undefined result
3082 // on some architectures.
3083 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3084
3085 // Long: ShAmt >= NVTBits
3086 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3087 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3088
3089 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3090 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3091 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3092 return true;
3093 case ISD::SRA:
3094 // Short: ShAmt < NVTBits
3095 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
3096 LoS = DAG.getNode(ISD::OR, dl, NVT,
3097 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3098 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3099
3100 // Long: ShAmt >= NVTBits
3101 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
3102 DAG.getConstant(NVTBits - 1, dl, ShTy));
3103 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3104
3105 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3106 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3107 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3108 return true;
3109 }
3110}
3111
3112static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
3113
3114 switch (Op) {
3115 default: llvm_unreachable("invalid min/max opcode");
3116 case ISD::SMAX:
3117 return std::make_pair(ISD::SETGT, ISD::UMAX);
3118 case ISD::UMAX:
3119 return std::make_pair(ISD::SETUGT, ISD::UMAX);
3120 case ISD::SMIN:
3121 return std::make_pair(ISD::SETLT, ISD::UMIN);
3122 case ISD::UMIN:
3123 return std::make_pair(ISD::SETULT, ISD::UMIN);
3124 }
3125}
3126
3127void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
3128 SDValue &Lo, SDValue &Hi) {
3129 SDLoc DL(N);
3130
3131 SDValue LHS = N->getOperand(0);
3132 SDValue RHS = N->getOperand(1);
3133
3134 // If the upper halves are all sign bits, then we can perform the MINMAX on
3135 // the lower half and sign-extend the result to the upper half.
3136 unsigned NumBits = N->getValueType(0).getScalarSizeInBits();
3137 unsigned NumHalfBits = NumBits / 2;
3138 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits &&
3139 DAG.ComputeNumSignBits(RHS) > NumHalfBits) {
3140 SDValue LHSL, LHSH, RHSL, RHSH;
3141 GetExpandedInteger(LHS, LHSL, LHSH);
3142 GetExpandedInteger(RHS, RHSL, RHSH);
3143 EVT NVT = LHSL.getValueType();
3144
3145 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL);
3146 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo,
3147 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL));
3148 return;
3149 }
3150
3151 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1.
3152 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL.
3153 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) ||
3154 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) {
3155 SDValue LHSL, LHSH, RHSL, RHSH;
3156 GetExpandedInteger(LHS, LHSL, LHSH);
3157 GetExpandedInteger(RHS, RHSL, RHSH);
3158 EVT NVT = LHSL.getValueType();
3159 EVT CCT = getSetCCResultType(NVT);
3160
3161 SDValue HiNeg =
3162 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
3163 if (N->getOpcode() == ISD::SMIN) {
3164 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT));
3165 } else {
3166 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
3167 }
3168 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3169 return;
3170 }
3171
3172 const APInt *RHSVal = nullptr;
3173 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3174 RHSVal = &RHSConst->getAPIntValue();
3175
3176 // The high half of MIN/MAX is always just the the MIN/MAX of the
3177 // high halves of the operands. Expand this way if it appears profitable.
3178 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) &&
3179 (RHSVal->countLeadingOnes() >= NumHalfBits ||
3180 RHSVal->countLeadingZeros() >= NumHalfBits)) {
3181 SDValue LHSL, LHSH, RHSL, RHSH;
3182 GetExpandedInteger(LHS, LHSL, LHSH);
3183 GetExpandedInteger(RHS, RHSL, RHSH);
3184 EVT NVT = LHSL.getValueType();
3185 EVT CCT = getSetCCResultType(NVT);
3186
3187 ISD::NodeType LoOpc;
3188 ISD::CondCode CondC;
3189 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
3190
3191 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3192 // We need to know whether to select Lo part that corresponds to 'winning'
3193 // Hi part or if Hi parts are equal.
3194 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
3195 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
3196
3197 // Lo part corresponding to the 'winning' Hi part
3198 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
3199
3200 // Recursed Lo part if Hi parts are equal, this uses unsigned version
3201 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
3202
3203 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
3204 return;
3205 }
3206
3207 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies
3208 // the compare.
3209 ISD::CondCode Pred;
3210 switch (N->getOpcode()) {
3211 default: llvm_unreachable("How did we get here?");
3212 case ISD::SMAX:
3213 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3214 Pred = ISD::SETGE;
3215 else
3216 Pred = ISD::SETGT;
3217 break;
3218 case ISD::SMIN:
3219 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3220 Pred = ISD::SETLE;
3221 else
3222 Pred = ISD::SETLT;
3223 break;
3224 case ISD::UMAX:
3225 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3226 Pred = ISD::SETUGE;
3227 else
3228 Pred = ISD::SETUGT;
3229 break;
3230 case ISD::UMIN:
3231 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3232 Pred = ISD::SETULE;
3233 else
3234 Pred = ISD::SETULT;
3235 break;
3236 }
3237 EVT VT = N->getValueType(0);
3238 EVT CCT = getSetCCResultType(VT);
3239 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred);
3240 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS);
3241 SplitInteger(Result, Lo, Hi);
3242}
3243
3244void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
3245 SDValue &Lo, SDValue &Hi) {
3246 SDLoc dl(N);
3247 // Expand the subcomponents.
3248 SDValue LHSL, LHSH, RHSL, RHSH;
3249 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3250 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3251
3252 EVT NVT = LHSL.getValueType();
3253 SDValue LoOps[2] = { LHSL, RHSL };
3254 SDValue HiOps[3] = { LHSH, RHSH };
3255
3256 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3257 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
3258 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3259 if (HasOpCarry) {
3260 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3261 if (N->getOpcode() == ISD::ADD) {
3262 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3263 HiOps[2] = Lo.getValue(1);
3264 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3265 ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2))
3266 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps);
3267 } else {
3268 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3269 HiOps[2] = Lo.getValue(1);
3270 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3271 ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2))
3272 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps);
3273 }
3274 return;
3275 }
3276
3277 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
3278 // them. TODO: Teach operation legalization how to expand unsupported
3279 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
3280 // a carry of type MVT::Glue, but there doesn't seem to be any way to
3281 // generate a value of this type in the expanded code sequence.
3282 bool hasCarry =
3283 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3285 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3286
3287 if (hasCarry) {
3288 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
3289 if (N->getOpcode() == ISD::ADD) {
3290 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3291 HiOps[2] = Lo.getValue(1);
3292 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3293 } else {
3294 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3295 HiOps[2] = Lo.getValue(1);
3296 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3297 }
3298 return;
3299 }
3300
3301 bool hasOVF =
3302 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3304 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3306
3307 if (hasOVF) {
3308 EVT OvfVT = getSetCCResultType(NVT);
3309 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
3310 int RevOpc;
3311 if (N->getOpcode() == ISD::ADD) {
3312 RevOpc = ISD::SUB;
3313 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3314 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3315 } else {
3316 RevOpc = ISD::ADD;
3317 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3318 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3319 }
3320 SDValue OVF = Lo.getValue(1);
3321
3322 switch (BoolType) {
3324 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
3325 [[fallthrough]];
3327 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
3328 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
3329 break;
3331 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
3332 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
3333 }
3334 return;
3335 }
3336
3337 if (N->getOpcode() == ISD::ADD) {
3338 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
3339 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3340 SDValue Cmp;
3341 // Special case: X+1 has a carry out if X+1==0. This may reduce the live
3342 // range of X. We assume comparing with 0 is cheap.
3343 if (isOneConstant(LoOps[1]))
3344 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3345 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3346 else if (isAllOnesConstant(LoOps[1])) {
3347 if (isAllOnesConstant(HiOps[1]))
3348 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3349 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3350 else
3351 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3352 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3353 } else
3354 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
3355 ISD::SETULT);
3356
3357 SDValue Carry;
3359 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3360 else
3361 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3362 DAG.getConstant(0, dl, NVT));
3363
3364 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1]))
3365 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry);
3366 else
3367 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
3368 } else {
3369 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
3370 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3371 SDValue Cmp =
3372 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3373 LoOps[0], LoOps[1], ISD::SETULT);
3374
3375 SDValue Borrow;
3377 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3378 else
3379 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3380 DAG.getConstant(0, dl, NVT));
3381
3382 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
3383 }
3384}
3385
3386void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
3387 SDValue &Lo, SDValue &Hi) {
3388 // Expand the subcomponents.
3389 SDValue LHSL, LHSH, RHSL, RHSH;
3390 SDLoc dl(N);
3391 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3392 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3393 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3394 SDValue LoOps[2] = { LHSL, RHSL };
3395 SDValue HiOps[3] = { LHSH, RHSH };
3396
3397 if (N->getOpcode() == ISD::ADDC) {
3398 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3399 HiOps[2] = Lo.getValue(1);
3400 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3401 } else {
3402 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3403 HiOps[2] = Lo.getValue(1);
3404 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3405 }
3406
3407 // Legalized the flag result - switch anything that used the old flag to
3408 // use the new one.
3409 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3410}
3411
3412void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
3413 SDValue &Lo, SDValue &Hi) {
3414 // Expand the subcomponents.
3415 SDValue LHSL, LHSH, RHSL, RHSH;
3416 SDLoc dl(N);
3417 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3418 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3419 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3420 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3421 SDValue HiOps[3] = { LHSH, RHSH };
3422
3423 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3424 HiOps[2] = Lo.getValue(1);
3425 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3426
3427 // Legalized the flag result - switch anything that used the old flag to
3428 // use the new one.
3429 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3430}
3431
3432void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
3433 SDValue &Lo, SDValue &Hi) {
3434 SDValue LHS = N->getOperand(0);
3435 SDValue RHS = N->getOperand(1);
3436 SDLoc dl(N);
3437
3438 SDValue Ovf;
3439
3440 unsigned CarryOp, NoCarryOp;
3442 switch(N->getOpcode()) {
3443 case ISD::UADDO:
3444 CarryOp = ISD::UADDO_CARRY;
3445 NoCarryOp = ISD::ADD;
3446 Cond = ISD::SETULT;
3447 break;
3448 case ISD::USUBO:
3449 CarryOp = ISD::USUBO_CARRY;
3450 NoCarryOp = ISD::SUB;
3451 Cond = ISD::SETUGT;
3452 break;
3453 default:
3454 llvm_unreachable("Node has unexpected Opcode");
3455 }
3456
3457 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3458 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3459
3460 if (HasCarryOp) {
3461 // Expand the subcomponents.
3462 SDValue LHSL, LHSH, RHSL, RHSH;
3463 GetExpandedInteger(LHS, LHSL, LHSH);
3464 GetExpandedInteger(RHS, RHSL, RHSH);
3465 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3466 SDValue LoOps[2] = { LHSL, RHSL };
3467 SDValue HiOps[3] = { LHSH, RHSH };
3468
3469 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3470 HiOps[2] = Lo.getValue(1);
3471 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
3472
3473 Ovf = Hi.getValue(1);
3474 } else {
3475 // Expand the result by simply replacing it with the equivalent
3476 // non-overflow-checking operation.
3477 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
3478 SplitInteger(Sum, Lo, Hi);
3479
3480 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) {
3481 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this
3482 // with (Lo | Hi) == 0.
3483 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi);
3484 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or,
3485 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ);
3486 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) {
3487 // Special case: uaddo X, -1 overflows if X == 0.
3488 Ovf =
3489 DAG.getSetCC(dl, N->getValueType(1), LHS,
3490 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE);
3491 } else {
3492 // Calculate the overflow: addition overflows iff a + b < a, and
3493 // subtraction overflows iff a - b > a.
3494 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
3495 }
3496 }
3497
3498 // Legalized the flag result - switch anything that used the old flag to
3499 // use the new one.
3500 ReplaceValueWith(SDValue(N, 1), Ovf);
3501}
3502
3503void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
3504 SDValue &Hi) {
3505 // Expand the subcomponents.
3506 SDValue LHSL, LHSH, RHSL, RHSH;
3507 SDLoc dl(N);
3508 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3509 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3510 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3511 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3512 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
3513
3514 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3515 HiOps[2] = Lo.getValue(1);
3516 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3517
3518 // Legalized the flag result - switch anything that used the old flag to
3519 // use the new one.
3520 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3521}
3522
3523void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
3524 SDValue &Lo, SDValue &Hi) {
3525 // Expand the subcomponents.
3526 SDValue LHSL, LHSH, RHSL, RHSH;
3527 SDLoc dl(N);
3528 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3529 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3530 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3531
3532 // We need to use an unsigned carry op for the lo part.
3533 unsigned CarryOp =
3535 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
3536 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3537
3538 // Legalized the flag result - switch anything that used the old flag to
3539 // use the new one.
3540 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3541}
3542
3543void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
3544 SDValue &Lo, SDValue &Hi) {
3545 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3546 SDLoc dl(N);
3547 SDValue Op = N->getOperand(0);
3548 if (Op.getValueType().bitsLE(NVT)) {
3549 // The low part is any extension of the input (which degenerates to a copy).
3550 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
3551 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
3552 } else {
3553 // For example, extension of an i48 to an i64. The operand type necessarily
3554 // promotes to the result type, so will end up being expanded too.
3555 assert(getTypeAction(Op.getValueType()) ==
3557 "Only know how to promote this result!");
3558 SDValue Res = GetPromotedInteger(Op);
3559 assert(Res.getValueType() == N->getValueType(0) &&
3560 "Operand over promoted?");
3561 // Split the promoted operand. This will simplify when it is expanded.
3562 SplitInteger(Res, Lo, Hi);
3563 }
3564}
3565
3566void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3567 SDValue &Lo, SDValue &Hi) {
3568 SDLoc dl(N);
3569 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3570 EVT NVT = Lo.getValueType();
3571 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3572 unsigned NVTBits = NVT.getSizeInBits();
3573 unsigned EVTBits = EVT.getSizeInBits();
3574
3575 if (NVTBits < EVTBits) {
3576 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3578 EVTBits - NVTBits)));
3579 } else {
3580 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3581 // The high part replicates the sign bit of Lo, make it explicit.
3582 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3583 DAG.getConstant(NVTBits - 1, dl,
3584 TLI.getPointerTy(DAG.getDataLayout())));
3585 }
3586}
3587
3588void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3589 SDValue &Lo, SDValue &Hi) {
3590 SDLoc dl(N);
3591 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3592 EVT NVT = Lo.getValueType();
3593 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3594 unsigned NVTBits = NVT.getSizeInBits();
3595 unsigned EVTBits = EVT.getSizeInBits();
3596
3597 if (NVTBits < EVTBits) {
3598 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3600 EVTBits - NVTBits)));
3601 } else {
3602 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3603 // The high part must be zero, make it explicit.
3604 Hi = DAG.getConstant(0, dl, NVT);
3605 }
3606}
3607
3608void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3609 SDValue &Lo, SDValue &Hi) {
3610 SDLoc dl(N);
3611 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3612 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3613 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3614}
3615
3616void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3617 SDValue &Lo, SDValue &Hi) {
3618 SDLoc dl(N);
3619 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3620 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3621 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3622}
3623
3624void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3625 SDValue &Hi) {
3626 SDLoc dl(N);
3627 // parity(HiLo) -> parity(Lo^Hi)
3628 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3629 EVT NVT = Lo.getValueType();
3630 Lo =
3631 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3632 Hi = DAG.getConstant(0, dl, NVT);
3633}
3634
3635void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3636 SDValue &Lo, SDValue &Hi) {
3637 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3638 unsigned NBitWidth = NVT.getSizeInBits();
3639 auto Constant = cast<ConstantSDNode>(N);
3640 const APInt &Cst = Constant->getAPIntValue();
3641 bool IsTarget = Constant->isTargetOpcode();
3642 bool IsOpaque = Constant->isOpaque();
3643 SDLoc dl(N);
3644 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3645 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3646 IsOpaque);
3647}
3648
3649void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3650 SDLoc dl(N);
3651
3652 SDValue N0 = N->getOperand(0);
3653 GetExpandedInteger(N0, Lo, Hi);
3654 EVT NVT = Lo.getValueType();
3655
3656 // If the upper half is all sign bits, then we can perform the ABS on the
3657 // lower half and zero-extend.
3658 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) {
3659 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo);
3660 Hi = DAG.getConstant(0, dl, NVT);
3661 return;
3662 }
3663
3664 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
3665 // we use in LegalizeDAG. The SUB part of the expansion is based on
3666 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
3667 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
3668 // expanded if needed. Shift expansion has a special case for filling with
3669 // sign bits so that we will only end up with one SRA.
3670 bool HasSubCarry = TLI.isOperationLegalOrCustom(
3672 if (HasSubCarry) {
3673 SDValue Sign = DAG.getNode(
3674 ISD::SRA, dl, NVT, Hi,
3675 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl));
3676 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3677 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3678 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3679 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
3680 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3681 return;
3682 }
3683
3684 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3685 EVT VT = N->getValueType(0);
3686 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3687 DAG.getConstant(0, dl, VT), N0);
3688 SDValue NegLo, NegHi;
3689 SplitInteger(Neg, NegLo, NegHi);
3690
3691 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3692 DAG.getConstant(0, dl, NVT), ISD::SETLT);
3693 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3694 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3695}
3696
3697void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3698 SDValue &Lo, SDValue &Hi) {
3699 SDLoc dl(N);
3700 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3701 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3702 EVT NVT = Lo.getValueType();
3703
3704 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3705 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3706
3707 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3708 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3709
3710 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3711 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3712 DAG.getConstant(NVT.getSizeInBits(), dl,
3713 NVT)));
3714 Hi = DAG.getConstant(0, dl, NVT);
3715}
3716
3717void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3718 SDValue &Lo, SDValue &Hi) {
3719 SDLoc dl(N);
3720 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3721 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3722 EVT NVT = Lo.getValueType();
3723 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3724 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3725 Hi = DAG.getConstant(0, dl, NVT);
3726}
3727
3728void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3729 SDValue &Lo, SDValue &Hi) {
3730 SDLoc dl(N);
3731 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3732 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3733 EVT NVT = Lo.getValueType();
3734
3735 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3736 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3737
3738 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3739 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3740
3741 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3742 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3743 DAG.getConstant(NVT.getSizeInBits(), dl,
3744 NVT)));
3745 Hi = DAG.getConstant(0, dl, NVT);
3746}
3747
3748void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo,
3749 SDValue &Hi) {
3750 SDLoc dl(N);
3751 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3752 unsigned NBitWidth = NVT.getSizeInBits();
3753
3754 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0));
3755 SDValue Chain = Lo.getValue(1);
3756 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING
3757 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3758 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
3759
3760 // Legalize the chain result - switch anything that used the old chain to
3761 // use the new one.
3762 ReplaceValueWith(SDValue(N, 1), Chain);
3763}
3764
3765// Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op.
3766static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT,
3767 SDLoc DL, SelectionDAG &DAG) {
3768 if (IsStrict) {
3769 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op});
3770 Chain = Op.getValue(1);
3771 return Op;
3772 }
3773 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Op);
3774}
3775
3776void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
3777 SDValue &Hi) {
3778 SDLoc dl(N);
3779 EVT VT = N->getValueType(0);
3780
3781 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3782 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3783 bool IsStrict = N->isStrictFPOpcode();
3784 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3785 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3786 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3787 Op = GetPromotedFloat(Op);
3788
3789 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3790 EVT OFPVT = Op.getValueType();
3791 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), OFPVT);
3792 Op = GetSoftPromotedHalf(Op);
3793 Op = DAG.getNode(OFPVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, dl,
3794 NFPVT, Op);
3795 Op = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, VT, Op);
3796 SplitInteger(Op, Lo, Hi);
3797 return;
3798 }
3799
3800 if (Op.getValueType() == MVT::bf16) {
3801 // Extend to f32 as there is no bf16 libcall.
3802 Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG);
3803 }
3804
3805 RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(Op.getValueType(), VT)
3806 : RTLIB::getFPTOUINT(Op.getValueType(), VT);
3807 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
3809 CallOptions.setSExt(true);
3810 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3811 CallOptions, dl, Chain);
3812 SplitInteger(Tmp.first, Lo, Hi);
3813
3814 if (IsStrict)
3815 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3816}
3817
3818void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3819 SDValue &Hi) {
3820 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3821 SplitInteger(Res, Lo, Hi);
3822}
3823
3824void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
3825 SDValue &Hi) {
3826 SDLoc dl(N);
3827 bool IsStrict = N->isStrictFPOpcode();
3828 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3829 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3830
3831 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3832 "Input type needs to be promoted!");
3833
3834 EVT VT = Op.getValueType();
3835
3836 if (VT == MVT::f16) {
3837 // Extend to f32.
3838 VT = MVT::f32;
3839 Op = fpExtendHelper(Op, Chain, IsStrict, VT, dl, DAG);
3840 }
3841
3842 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3843 if (N->getOpcode() == ISD::LROUND ||
3844 N->getOpcode() == ISD::STRICT_LROUND) {
3845 if (VT == MVT::f32)
3846 LC = RTLIB::LROUND_F32;
3847 else if (VT == MVT::f64)
3848 LC = RTLIB::LROUND_F64;
3849 else if (VT == MVT::f80)
3850 LC = RTLIB::LROUND_F80;
3851 else if (VT == MVT::f128)
3852 LC = RTLIB::LROUND_F128;
3853 else if (VT == MVT::ppcf128)
3854 LC = RTLIB::LROUND_PPCF128;
3855 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!");
3856 } else if (N->getOpcode() == ISD::LRINT ||
3857 N->getOpcode() == ISD::STRICT_LRINT) {
3858 if (VT == MVT::f32)
3859 LC = RTLIB::LRINT_F32;
3860 else if (VT == MVT::f64)
3861 LC = RTLIB::LRINT_F64;
3862 else if (VT == MVT::f80)
3863 LC = RTLIB::LRINT_F80;
3864 else if (VT == MVT::f128)
3865 LC = RTLIB::LRINT_F128;
3866 else if (VT == MVT::ppcf128)
3867 LC = RTLIB::LRINT_PPCF128;
3868 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!");
3869 } else if (N->getOpcode() == ISD::LLROUND ||
3870 N->getOpcode() == ISD::STRICT_LLROUND) {
3871 if (VT == MVT::f32)
3872 LC = RTLIB::LLROUND_F32;
3873 else if (VT == MVT::f64)
3874 LC = RTLIB::LLROUND_F64;
3875 else if (VT == MVT::f80)
3876 LC = RTLIB::LLROUND_F80;
3877 else if (VT == MVT::f128)
3878 LC = RTLIB::LLROUND_F128;
3879 else if (VT == MVT::ppcf128)
3880 LC = RTLIB::LLROUND_PPCF128;
3881 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3882 } else if (N->getOpcode() == ISD::LLRINT ||
3883 N->getOpcode() == ISD::STRICT_LLRINT) {
3884 if (VT == MVT::f32)
3885 LC = RTLIB::LLRINT_F32;
3886 else if (VT == MVT::f64)
3887 LC = RTLIB::LLRINT_F64;
3888 else if (VT == MVT::f80)
3889 LC = RTLIB::LLRINT_F80;
3890 else if (VT == MVT::f128)
3891 LC = RTLIB::LLRINT_F128;
3892 else if (VT == MVT::ppcf128)
3893 LC = RTLIB::LLRINT_PPCF128;
3894 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3895 } else
3896 llvm_unreachable("Unexpected opcode!");
3897
3898 EVT RetVT = N->getValueType(0);
3899
3901 CallOptions.setSExt(true);
3902 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3903 Op, CallOptions, dl,
3904 Chain);
3905 SplitInteger(Tmp.first, Lo, Hi);
3906
3907 if (N->isStrictFPOpcode())
3908 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3909}
3910
3911void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3912 SDValue &Lo, SDValue &Hi) {
3913 assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?");
3914
3915 if (ISD::isNormalLoad(N)) {
3916 ExpandRes_NormalLoad(N, Lo, Hi);
3917 return;
3918 }
3919
3920 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3921
3922 EVT VT = N->getValueType(0);
3923 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3924 SDValue Ch = N->getChain();
3925 SDValue Ptr = N->getBasePtr();
3926 ISD::LoadExtType ExtType = N->getExtensionType();
3927 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3928 AAMDNodes AAInfo = N->getAAInfo();
3929 SDLoc dl(N);
3930
3931 assert(NVT.isByteSized() && "Expanded type not byte sized!");
3932
3933 if (N->getMemoryVT().bitsLE(NVT)) {
3934 EVT MemVT = N->getMemoryVT();
3935
3936 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3937 N->getOriginalAlign(), MMOFlags, AAInfo);
3938
3939 // Remember the chain.
3940 Ch = Lo.getValue(1);
3941
3942 if (ExtType == ISD::SEXTLOAD) {
3943 // The high part is obtained by SRA'ing all but one of the bits of the
3944 // lo part.
3945 unsigned LoSize = Lo.getValueSizeInBits();
3946 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3947 DAG.getConstant(LoSize - 1, dl,
3948 TLI.getPointerTy(DAG.getDataLayout())));
3949 } else if (ExtType == ISD::ZEXTLOAD) {
3950 // The high part is just a zero.
3951 Hi = DAG.getConstant(0, dl, NVT);
3952 } else {
3953 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3954 // The high part is undefined.
3955 Hi = DAG.getUNDEF(NVT);
3956 }
3957 } else if (DAG.getDataLayout().isLittleEndian()) {
3958 // Little-endian - low bits are at low addresses.
3959 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3960 N->getOriginalAlign(), MMOFlags, AAInfo);
3961
3962 unsigned ExcessBits =
3963 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3964 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3965
3966 // Increment the pointer to the other half.
3967 unsigned IncrementSize = NVT.getSizeInBits()/8;
3968 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
3969 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3970 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3971 N->getOriginalAlign(), MMOFlags, AAInfo);
3972
3973 // Build a factor node to remember that this load is independent of the
3974 // other one.
3975 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3976 Hi.getValue(1));
3977 } else {
3978 // Big-endian - high bits are at low addresses. Favor aligned loads at
3979 // the cost of some bit-fiddling.
3980 EVT MemVT = N->getMemoryVT();
3981 unsigned EBytes = MemVT.getStoreSize();
3982 unsigned IncrementSize = NVT.getSizeInBits()/8;
3983 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3984
3985 // Load both the high bits and maybe some of the low bits.
3986 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3988 MemVT.getSizeInBits() - ExcessBits),
3989 N->getOriginalAlign(), MMOFlags, AAInfo);
3990
3991 // Increment the pointer to the other half.
3992 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
3993 // Load the rest of the low bits.
3994 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3995 N->getPointerInfo().getWithOffset(IncrementSize),
3996 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3997 N->getOriginalAlign(), MMOFlags, AAInfo);
3998
3999 // Build a factor node to remember that this load is independent of the
4000 // other one.
4001 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4002 Hi.getValue(1));
4003
4004 if (ExcessBits < NVT.getSizeInBits()) {
4005 // Transfer low bits from the bottom of Hi to the top of Lo.
4006 Lo = DAG.getNode(
4007 ISD::OR, dl, NVT, Lo,
4008 DAG.getNode(ISD::SHL, dl, NVT, Hi,
4009 DAG.getConstant(ExcessBits, dl,
4010 TLI.getPointerTy(DAG.getDataLayout()))));
4011 // Move high bits to the right position in Hi.
4012 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
4013 Hi,
4014 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4015 TLI.getPointerTy(DAG.getDataLayout())));
4016 }
4017 }
4018
4019 // Legalize the chain result - switch anything that used the old chain to
4020 // use the new one.
4021 ReplaceValueWith(SDValue(N, 1), Ch);
4022}
4023
4024void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
4025 SDValue &Lo, SDValue &Hi) {
4026 SDLoc dl(N);
4027 SDValue LL, LH, RL, RH;
4028 GetExpandedInteger(N->getOperand(0), LL, LH);
4029 GetExpandedInteger(N->getOperand(1), RL, RH);
4030 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
4031 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
4032}
4033
4034void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
4035 SDValue &Lo, SDValue &Hi) {
4036 EVT VT = N->getValueType(0);
4037 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4038 SDLoc dl(N);
4039
4040 SDValue LL, LH, RL, RH;
4041 GetExpandedInteger(N->getOperand(0), LL, LH);
4042 GetExpandedInteger(N->getOperand(1), RL, RH);
4043
4044 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
4046 LL, LH, RL, RH))
4047 return;
4048
4049 // If nothing else, we can make a libcall.
4050 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4051 if (VT == MVT::i16)
4052 LC = RTLIB::MUL_I16;
4053 else if (VT == MVT::i32)
4054 LC = RTLIB::MUL_I32;
4055 else if (VT == MVT::i64)
4056 LC = RTLIB::MUL_I64;
4057 else if (VT == MVT::i128)
4058 LC = RTLIB::MUL_I128;
4059
4060 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
4061 // Perform a wide multiplication where the wide type is the original VT and
4062 // the 4 parts are the split arguments.
4063 TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, VT, LL, LH, RL, RH, Lo,
4064 Hi);
4065 return;
4066 }
4067
4068 // Note that we don't need to do a wide MUL here since we don't care about the
4069 // upper half of the result if it exceeds VT.
4070 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4072 CallOptions.setSExt(true);
4073 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4074 Lo, Hi);
4075}
4076
4077void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo,
4078 SDValue &Hi) {
4079 SDLoc DL(N);
4080 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4081 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
4082 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
4083 Lo = R.getValue(0);
4084 Hi = R.getValue(1);
4085 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
4086}
4087
4088void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
4089 SDValue &Hi) {
4090 SDValue Result = TLI.expandAddSubSat(N, DAG);
4091 SplitInteger(Result, Lo, Hi);
4092}
4093
4094void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
4095 SDValue &Hi) {
4096 SDValue Result = TLI.expandShlSat(N, DAG);
4097 SplitInteger(Result, Lo, Hi);
4098}
4099
4100/// This performs an expansion of the integer result for a fixed point
4101/// multiplication. The default expansion performs rounding down towards
4102/// negative infinity, though targets that do care about rounding should specify
4103/// a target hook for rounding and provide their own expansion or lowering of
4104/// fixed point multiplication to be consistent with rounding.
4105void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
4106 SDValue &Hi) {
4107 SDLoc dl(N);
4108 EVT VT = N->getValueType(0);
4109 unsigned VTSize = VT.getScalarSizeInBits();
4110 SDValue LHS = N->getOperand(0);
4111 SDValue RHS = N->getOperand(1);
4112 uint64_t Scale = N->getConstantOperandVal(2);
4113 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
4114 N->getOpcode() == ISD::UMULFIXSAT);
4115 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
4116 N->getOpcode() == ISD::SMULFIXSAT);
4117
4118 // Handle special case when scale is equal to zero.
4119 if (!Scale) {
4121 if (!Saturating) {
4122 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
4123 } else {
4124 EVT BoolVT = getSetCCResultType(VT);
4125 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
4126 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
4127 SDValue Product = Result.getValue(0);
4128 SDValue Overflow = Result.getValue(1);
4129 if (Signed) {
4130 APInt MinVal = APInt::getSignedMinValue(VTSize);
4131 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
4132 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
4133 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4134 SDValue Zero = DAG.getConstant(0, dl, VT);
4135 // Xor the inputs, if resulting sign bit is 0 the product will be
4136 // positive, else negative.
4137 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4138 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
4139 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
4140 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
4141 } else {
4142 // For unsigned multiplication, we only need to check the max since we
4143 // can't really overflow towards zero.
4144 APInt MaxVal = APInt::getMaxValue(VTSize);
4145 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4146 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
4147 }
4148 }
4149 SplitInteger(Result, Lo, Hi);
4150 return;
4151 }
4152
4153 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
4154 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
4155 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
4156
4157 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4158 SDValue LL, LH, RL, RH;
4159 GetExpandedInteger(LHS, LL, LH);
4160 GetExpandedInteger(RHS, RL, RH);
4162
4163 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
4164 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4166 LL, LH, RL, RH)) {
4167 Result.clear();
4168 Result.resize(4);
4169
4170 SDValue LoTmp, HiTmp;
4171 TLI.forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, LoTmp, HiTmp);
4172 SplitInteger(LoTmp, Result[0], Result[1]);
4173 SplitInteger(HiTmp, Result[2], Result[3]);
4174 }
4175 assert(Result.size() == 4 && "Unexpected number of partlets in the result");
4176
4177 unsigned NVTSize = NVT.getScalarSizeInBits();
4178 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
4179 "the size of the current value type");
4180
4181 // After getting the multiplication result in 4 parts, we need to perform a
4182 // shift right by the amount of the scale to get the result in that scale.
4183 //
4184 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
4185 // 128 bits that are cut into 4 32-bit parts:
4186 //
4187 // HH HL LH LL
4188 // |---32---|---32---|---32---|---32---|
4189 // 128 96 64 32 0
4190 //
4191 // |------VTSize-----|
4192 //
4193 // |NVTSize-|
4194 //
4195 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
4196 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
4197 // amount and get Lo and Hi using two funnel shifts. Or for the special case
4198 // when Scale is a multiple of NVTSize we can just pick the result without
4199 // shifting.
4200 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
4201 if (Scale % NVTSize) {
4202 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl);
4203 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
4204 ShiftAmount);
4205 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
4206 ShiftAmount);
4207 } else {
4208 Lo = Result[Part0];
4209 Hi = Result[Part0 + 1];
4210 }
4211
4212 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
4213 if (!Saturating)
4214 return;
4215
4216 // Can not overflow when there is no integer part.
4217 if (Scale == VTSize)
4218 return;
4219
4220 // To handle saturation we must check for overflow in the multiplication.
4221 //
4222 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
4223 // aren't all zeroes.
4224 //
4225 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
4226 // aren't all ones or all zeroes.
4227 //
4228 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
4229 // highest bit of HH determines saturation direction in the event of signed
4230 // saturation.
4231
4232 SDValue ResultHL = Result[2];
4233 SDValue ResultHH = Result[3];
4234
4235 SDValue SatMax, SatMin;
4236 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
4237 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
4238 EVT BoolNVT = getSetCCResultType(NVT);
4239
4240 if (!Signed) {
4241 if (Scale < NVTSize) {
4242 // Overflow happened if ((HH | (HL >> Scale)) != 0).
4243 SDValue HLAdjusted =
4244 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4245 DAG.getShiftAmountConstant(Scale, NVT, dl));
4246 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
4247 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
4248 } else if (Scale == NVTSize) {
4249 // Overflow happened if (HH != 0).
4250 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
4251 } else if (Scale < VTSize) {
4252 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
4253 SDValue HLAdjusted =
4254 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4255 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl));
4256 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
4257 } else
4258 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
4259 "(and saturation can't happen with Scale==VTSize).");
4260
4261 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
4262 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
4263 return;
4264 }
4265
4266 if (Scale < NVTSize) {
4267 // The number of overflow bits we can check are VTSize - Scale + 1 (we
4268 // include the sign bit). If these top bits are > 0, then we overflowed past
4269 // the max value. If these top bits are < -1, then we overflowed past the
4270 // min value. Otherwise, we did not overflow.
4271 unsigned OverflowBits = VTSize - Scale + 1;
4272 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4273 "Extent of overflow bits must start within HL");
4274 SDValue HLHiMask = DAG.getConstant(
4275 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
4276 SDValue HLLoMask = DAG.getConstant(
4277 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
4278 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
4279 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4280 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4281 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
4282 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4283 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
4284 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
4285 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4286 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4287 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
4288 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4289 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
4290 } else if (Scale == NVTSize) {
4291 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
4292 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4293 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4294 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
4295 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4296 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
4297 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
4298 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4299 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4300 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
4301 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4302 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
4303 } else if (Scale < VTSize) {
4304 // This is similar to the case when we saturate if Scale < NVTSize, but we
4305 // only need to check HH.
4306 unsigned OverflowBits = VTSize - Scale + 1;
4307 SDValue HHHiMask = DAG.getConstant(
4308 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
4309 SDValue HHLoMask = DAG.getConstant(
4310 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
4311 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
4312 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
4313 } else
4314 llvm_unreachable("Illegal scale for signed fixed point mul.");
4315
4316 // Saturate to signed maximum.
4317 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
4318 APInt MaxLo = APInt::getAllOnes(NVTSize);
4319 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
4320 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
4321 // Saturate to signed minimum.
4322 APInt MinHi = APInt::getSignedMinValue(NVTSize);
4323 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
4324 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
4325}
4326
4327void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
4328 SDValue &Hi) {
4329 SDLoc dl(N);
4330 // Try expanding in the existing type first.
4331 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
4332 N->getOperand(1),
4333 N->getConstantOperandVal(2), DAG);
4334
4335 if (!Res)
4336 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
4337 N->getConstantOperandVal(2), TLI, DAG);
4338 SplitInteger(Res, Lo, Hi);
4339}
4340
4341void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
4342 SDValue &Lo, SDValue &Hi) {
4343 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
4344 "Node has unexpected Opcode");
4345 SDValue LHS = Node->getOperand(0);
4346 SDValue RHS = Node->getOperand(1);
4347 SDLoc dl(Node);
4348
4349 SDValue Ovf;
4350
4351 bool IsAdd = Node->getOpcode() == ISD::SADDO;
4352 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
4353
4354 bool HasCarryOp = TLI.isOperationLegalOrCustom(
4355 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
4356
4357 if (HasCarryOp) {
4358 // Expand the subcomponents.
4359 SDValue LHSL, LHSH, RHSL, RHSH;
4360 GetExpandedInteger(LHS, LHSL, LHSH);
4361 GetExpandedInteger(RHS, RHSL, RHSH);
4362 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
4363
4364 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
4365 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
4366
4367 Ovf = Hi.getValue(1);
4368 } else {
4369 // Expand the result by simply replacing it with the equivalent
4370 // non-overflow-checking operation.
4371 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
4372 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
4373 LHS, RHS);
4374 SplitInteger(Sum, Lo, Hi);
4375
4376 // Compute the overflow.
4377 //
4378 // LHSSign -> LHS < 0
4379 // RHSSign -> RHS < 0
4380 // SumSign -> Sum < 0
4381 //
4382 // Add:
4383 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
4384 // Sub:
4385 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
4386 //
4387 // To get better codegen we can rewrite this by doing bitwise math on
4388 // the integers and extract the final sign bit at the end. So the
4389 // above becomes:
4390 //
4391 // Add:
4392 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
4393 // Sub:
4394 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
4395 //
4396 // NOTE: This is different than the expansion we do in expandSADDSUBO
4397 // because it is more costly to determine the RHS is > 0 for SSUBO with the
4398 // integers split.
4399 EVT VT = LHS.getValueType();
4400 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4401 if (IsAdd)
4402 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
4403
4404 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
4405 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
4406 EVT OType = Node->getValueType(1);
4407 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
4408 }
4409
4410 // Use the calculated overflow everywhere.
4411 ReplaceValueWith(SDValue(Node, 1), Ovf);
4412}
4413
4414void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
4415 SDValue &Lo, SDValue &Hi) {
4416 EVT VT = N->getValueType(0);
4417 SDLoc dl(N);
4418 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4419
4421 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4422 SplitInteger(Res.getValue(0), Lo, Hi);
4423 return;
4424 }
4425
4426 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4427 if (VT == MVT::i16)
4428 LC = RTLIB::SDIV_I16;
4429 else if (VT == MVT::i32)
4430 LC = RTLIB::SDIV_I32;
4431 else if (VT == MVT::i64)
4432 LC = RTLIB::SDIV_I64;
4433 else if (VT == MVT::i128)
4434 LC = RTLIB::SDIV_I128;
4435 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
4436
4438 CallOptions.setSExt(true);
4439 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4440}
4441
4442void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo,
4443 SDValue &Hi) {
4444 SDLoc dl(N);
4445 SDValue Shiftee = N->getOperand(0);
4446 EVT VT = Shiftee.getValueType();
4447 SDValue ShAmt = N->getOperand(1);
4448 EVT ShAmtVT = ShAmt.getValueType();
4449
4450 // This legalization is optimal when the shift is by a multiple of byte width,
4451 // %x * 8 <-> %x << 3 so 3 low bits should be be known zero.
4452 bool ShiftByByteMultiple =
4453 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3;
4454
4455 // If we can't do it as one step, we'll have two uses of shift amount,
4456 // and thus must freeze it.
4457 if (!ShiftByByteMultiple)
4458 ShAmt = DAG.getFreeze(ShAmt);
4459
4460 unsigned VTBitWidth = VT.getScalarSizeInBits();
4461 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?");
4462 unsigned VTByteWidth = VTBitWidth / 8;
4463 assert(isPowerOf2_32(VTByteWidth) &&
4464 "Shiftee type size is not a power of two!");
4465 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4466 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4467 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth);
4468
4469 // Get a temporary stack slot 2x the width of our VT.
4470 // FIXME: reuse stack slots?
4471 // FIXME: should we be more picky about alignment?
4472 Align StackSlotAlignment(1);
4474 TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment);
4475 EVT PtrTy = StackPtr.getValueType();
4476 SDValue Ch = DAG.getEntryNode();
4477
4479 DAG.getMachineFunction(),
4480 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex());
4481
4482 // Extend the value, that is being shifted, to the entire stack slot's width.
4483 SDValue Init;
4484 if (N->getOpcode() != ISD::SHL) {
4485 unsigned WideningOpc =
4486 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4487 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4488 } else {
4489 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width.
4490 SDValue AllZeros = DAG.getConstant(0, dl, VT);
4491 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee);
4492 }
4493 // And spill it into the stack slot.
4494 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4495
4496 // Now, compute the full-byte offset into stack slot from where we can load.
4497 // We have shift amount, which is in bits, but in multiples of byte.
4498 // So just divide by CHAR_BIT.
4500 if (ShiftByByteMultiple)
4501 Flags.setExact(true);
4502 SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt,
4503 DAG.getConstant(3, dl, ShAmtVT), Flags);
4504 // And clamp it, because OOB load is an immediate UB,
4505 // while shift overflow would have *just* been poison.
4506 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset,
4507 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT));
4508 // We have exactly two strategies on indexing into stack slot here:
4509 // 1. upwards starting from the beginning of the slot
4510 // 2. downwards starting from the middle of the slot
4511 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift
4512 // and vice versa on big-endian machine.
4513 bool WillIndexUpwards = N->getOpcode() != ISD::SHL;
4514 if (DAG.getDataLayout().isBigEndian())
4515 WillIndexUpwards = !WillIndexUpwards;
4516
4517 SDValue AdjStackPtr;
4518 if (WillIndexUpwards) {
4519 AdjStackPtr = StackPtr;
4520 } else {
4521 AdjStackPtr = DAG.getMemBasePlusOffset(
4522 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl);
4523 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT);
4524 }
4525
4526 // Get the pointer somewhere into the stack slot from which we need to load.
4527 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy);
4528 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl);
4529
4530 // And load it! While the load is not legal, legalizing it is obvious.
4531 SDValue Res = DAG.getLoad(
4532 VT, dl, Ch, AdjStackPtr,
4534 // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_]
4535
4536 // If we may still have a less-than-CHAR_BIT to shift by, do so now.
4537 if (!ShiftByByteMultiple) {
4538 SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
4539 DAG.getConstant(7, dl, ShAmtVT));
4540 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem);
4541 }
4542
4543 // Finally, split the computed value.
4544 SplitInteger(Res, Lo, Hi);
4545}
4546
4547void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
4548 SDValue &Lo, SDValue &Hi) {
4549 EVT VT = N->getValueType(0);
4550 unsigned Opc = N->getOpcode();
4551 SDLoc dl(N);
4552
4553 // If we can emit an efficient shift operation, do so now. Check to see if
4554 // the RHS is a constant.
4555 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
4556 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
4557
4558 // If we can determine that the high bit of the shift is zero or one, even if
4559 // the low bits are variable, emit this shift in an optimized form.
4560 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
4561 return;
4562
4563 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
4564 unsigned PartsOpc;
4565 if (Opc == ISD::SHL) {
4566 PartsOpc = ISD::SHL_PARTS;
4567 } else if (Opc == ISD::SRL) {
4568 PartsOpc = ISD::SRL_PARTS;
4569 } else {
4570 assert(Opc == ISD::SRA && "Unknown shift!");
4571 PartsOpc = ISD::SRA_PARTS;
4572 }
4573
4574 // Next check to see if the target supports this SHL_PARTS operation or if it
4575 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
4576 // size, but create a libcall instead.
4577 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4579 const bool LegalOrCustom =
4580 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
4581 Action == TargetLowering::Custom;
4582
4583 unsigned ExpansionFactor = 1;
4584 // That VT->NVT expansion is one step. But will we re-expand NVT?
4585 for (EVT TmpVT = NVT;;) {
4586 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT);
4587 if (NewTMPVT == TmpVT)
4588 break;
4589 TmpVT = NewTMPVT;
4590 ++ExpansionFactor;
4591 }
4592
4594 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor);
4595
4597 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi);
4598
4599 if (LegalOrCustom &&
4601 // Expand the subcomponents.
4602 SDValue LHSL, LHSH;
4603 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4604 EVT VT = LHSL.getValueType();
4605
4606 // If the shift amount operand is coming from a vector legalization it may
4607 // have an illegal type. Fix that first by casting the operand, otherwise
4608 // the new SHL_PARTS operation would need further legalization.
4609 SDValue ShiftOp = N->getOperand(1);
4610 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
4611 if (ShiftOp.getValueType() != ShiftTy)
4612 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
4613
4614 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4615 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
4616 Hi = Lo.getValue(1);
4617 return;
4618 }
4619
4620 // Otherwise, emit a libcall.
4621 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4622 bool isSigned;
4623 if (Opc == ISD::SHL) {
4624 isSigned = false; /*sign irrelevant*/
4625 if (VT == MVT::i16)
4626 LC = RTLIB::SHL_I16;
4627 else if (VT == MVT::i32)
4628 LC = RTLIB::SHL_I32;
4629 else if (VT == MVT::i64)
4630 LC = RTLIB::SHL_I64;
4631 else if (VT == MVT::i128)
4632 LC = RTLIB::SHL_I128;
4633 } else if (Opc == ISD::SRL) {
4634 isSigned = false;
4635 if (VT == MVT::i16)
4636 LC = RTLIB::SRL_I16;
4637 else if (VT == MVT::i32)
4638 LC = RTLIB::SRL_I32;
4639 else if (VT == MVT::i64)
4640 LC = RTLIB::SRL_I64;
4641 else if (VT == MVT::i128)
4642 LC = RTLIB::SRL_I128;
4643 } else {
4644 assert(Opc == ISD::SRA && "Unknown shift!");
4645 isSigned = true;
4646 if (VT == MVT::i16)
4647 LC = RTLIB::SRA_I16;
4648 else if (VT == MVT::i32)
4649 LC = RTLIB::SRA_I32;
4650 else if (VT == MVT::i64)
4651 LC = RTLIB::SRA_I64;
4652 else if (VT == MVT::i128)
4653 LC = RTLIB::SRA_I128;
4654 }
4655
4656 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
4657 EVT ShAmtTy =
4659 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
4660 SDValue Ops[2] = {N->getOperand(0), ShAmt};
4662 CallOptions.setSExt(isSigned);
4663 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4664 return;
4665 }
4666
4667 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
4668 llvm_unreachable("Unsupported shift!");
4669}
4670
4671void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
4672 SDValue &Lo, SDValue &Hi) {
4673 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4674 SDLoc dl(N);
4675 SDValue Op = N->getOperand(0);
4676 if (Op.getValueType().bitsLE(NVT)) {
4677 // The low part is sign extension of the input (degenerates to a copy).
4678 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
4679 // The high part is obtained by SRA'ing all but one of the bits of low part.
4680 unsigned LoSize = NVT.getSizeInBits();
4681 Hi = DAG.getNode(
4682 ISD::SRA, dl, NVT, Lo,
4683 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
4684 } else {
4685 // For example, extension of an i48 to an i64. The operand type necessarily
4686 // promotes to the result type, so will end up being expanded too.
4687 assert(getTypeAction(Op.getValueType()) ==
4689 "Only know how to promote this result!");
4690 SDValue Res = GetPromotedInteger(Op);
4691 assert(Res.getValueType() == N->getValueType(0) &&
4692 "Operand over promoted?");
4693 // Split the promoted operand. This will simplify when it is expanded.
4694 SplitInteger(Res, Lo, Hi);
4695 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4696 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4698 ExcessBits)));
4699 }
4700}
4701
4702void DAGTypeLegalizer::
4703ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4704 SDLoc dl(N);
4705 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4706 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4707
4708 if (EVT.bitsLE(Lo.getValueType())) {
4709 // sext_inreg the low part if needed.
4710 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
4711 N->getOperand(1));
4712
4713 // The high part gets the sign extension from the lo-part. This handles
4714 // things like sextinreg V:i64 from i8.
4715 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
4716 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
4717 TLI.getPointerTy(DAG.getDataLayout())));
4718 } else {
4719 // For example, extension of an i48 to an i64. Leave the low part alone,
4720 // sext_inreg the high part.
4721 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
4722 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4724 ExcessBits)));
4725 }
4726}
4727
4728void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
4729 SDValue &Lo, SDValue &Hi) {
4730 EVT VT = N->getValueType(0);
4731 SDLoc dl(N);
4732 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4733
4735 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4736 SplitInteger(Res.getValue(1), Lo, Hi);
4737 return;
4738 }
4739
4740 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4741 if (VT == MVT::i16)
4742 LC = RTLIB::SREM_I16;
4743 else if (VT == MVT::i32)
4744 LC = RTLIB::SREM_I32;
4745 else if (VT == MVT::i64)
4746 LC = RTLIB::SREM_I64;
4747 else if (VT == MVT::i128)
4748 LC = RTLIB::SREM_I128;
4749 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
4750
4752 CallOptions.setSExt(true);
4753 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4754}
4755
4756void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
4757 SDValue &Lo, SDValue &Hi) {
4758 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4759 SDLoc dl(N);
4760 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
4761 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
4762 N->getOperand(0),
4763 DAG.getConstant(NVT.getSizeInBits(), dl,
4764 TLI.getPointerTy(DAG.getDataLayout())));
4765 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
4766}
4767
4768void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
4769 SDValue &Lo, SDValue &Hi) {
4770 EVT VT = N->getValueType(0);
4771 SDLoc dl(N);
4772
4773 if (N->getOpcode() == ISD::UMULO) {
4774 // This section expands the operation into the following sequence of
4775 // instructions. `iNh` here refers to a type which has half the bit width of
4776 // the type the original operation operated on.
4777 //
4778 // %0 = %LHS.HI != 0 && %RHS.HI != 0
4779 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
4780 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
4781 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
4782 // %4 = add iNh %1.0, %2.0 as iN
4783 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
4784 //
4785 // %lo = %3.LO
4786 // %hi = %5.0
4787 // %ovf = %0 || %1.1 || %2.1 || %5.1
4788 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
4789 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4790 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4791 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4792 EVT HalfVT = LHSLow.getValueType();
4793 EVT BitVT = N->getValueType(1);
4794 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
4795
4796 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
4797 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
4798 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
4799 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
4800
4801 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
4802 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
4803
4804 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
4805 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
4806
4807 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
4808
4809 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
4810 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
4811 // operation recursively legalized?).
4812 //
4813 // Many backends understand this pattern and will convert into LOHI
4814 // themselves, if applicable.
4815 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
4816 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
4817 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
4818 SplitInteger(Three, Lo, Hi);
4819
4820 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
4821 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
4822 ReplaceValueWith(SDValue(N, 1), Overflow);
4823 return;
4824 }
4825
4826 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
4827 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
4828 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
4829
4830 // Replace this with a libcall that will check overflow.
4831 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4832 if (VT == MVT::i32)
4833 LC = RTLIB::MULO_I32;
4834 else if (VT == MVT::i64)
4835 LC = RTLIB::MULO_I64;
4836 else if (VT == MVT::i128)
4837 LC = RTLIB::MULO_I128;
4838
4839 // If we don't have the libcall or if the function we are compiling is the
4840 // implementation of the expected libcall (avoid inf-loop), expand inline.
4841 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) ||
4842 TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) {
4843 // FIXME: This is not an optimal expansion, but better than crashing.
4844 EVT WideVT =
4846 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0));
4847 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1));
4848 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
4849 SDValue MulLo, MulHi;
4850 SplitInteger(Mul, MulLo, MulHi);
4851 SDValue SRA =
4852 DAG.getNode(ISD::SRA, dl, VT, MulLo,
4853 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT));
4854 SDValue Overflow =
4855 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
4856 SplitInteger(MulLo, Lo, Hi);
4857 ReplaceValueWith(SDValue(N, 1), Overflow);
4858 return;
4859 }
4860
4861 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
4862 // Temporary for the overflow value, default it to zero.
4863 SDValue Chain =
4864 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
4866
4869 for (const SDValue &Op : N->op_values()) {
4870 EVT ArgVT = Op.getValueType();
4871 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
4872 Entry.Node = Op;
4873 Entry.Ty = ArgTy;
4874 Entry.IsSExt = true;
4875 Entry.IsZExt = false;
4876 Args.push_back(Entry);
4877 }
4878
4879 // Also pass the address of the overflow check.
4880 Entry.Node = Temp;
4881 Entry.Ty = PointerType::getUnqual(PtrTy->getContext());
4882 Entry.IsSExt = true;
4883 Entry.IsZExt = false;
4884 Args.push_back(Entry);
4885
4886 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
4887
4889 CLI.setDebugLoc(dl)
4890 .setChain(Chain)
4891 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
4892 .setSExtResult();
4893
4894 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
4895
4896 SplitInteger(CallInfo.first, Lo, Hi);
4897 SDValue Temp2 =
4898 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
4899 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
4900 DAG.getConstant(0, dl, PtrVT),
4901 ISD::SETNE);
4902 // Use the overflow from the libcall everywhere.
4903 ReplaceValueWith(SDValue(N, 1), Ofl);
4904}
4905
4906void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
4907 SDValue &Lo, SDValue &Hi) {
4908 EVT VT = N->getValueType(0);
4909 SDLoc dl(N);
4910 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4911
4913 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4914 SplitInteger(Res.getValue(0), Lo, Hi);
4915 return;
4916 }
4917
4918 // Try to expand UDIV by constant.
4919 if (isa<ConstantSDNode>(N->getOperand(1))) {
4920 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4921 // Only if the new type is legal.
4922 if (isTypeLegal(NVT)) {
4923 SDValue InL, InH;
4924 GetExpandedInteger(N->getOperand(0), InL, InH);
4926 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
4927 Lo = Result[0];
4928 Hi = Result[1];
4929 return;
4930 }
4931 }
4932 }
4933
4934 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4935 if (VT == MVT::i16)
4936 LC = RTLIB::UDIV_I16;
4937 else if (VT == MVT::i32)
4938 LC = RTLIB::UDIV_I32;
4939 else if (VT == MVT::i64)
4940 LC = RTLIB::UDIV_I64;
4941 else if (VT == MVT::i128)
4942 LC = RTLIB::UDIV_I128;
4943 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
4944
4946 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4947}
4948
4949void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
4950 SDValue &Lo, SDValue &Hi) {
4951 EVT VT = N->getValueType(0);
4952 SDLoc dl(N);
4953 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4954
4956 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4957 SplitInteger(Res.getValue(1), Lo, Hi);
4958 return;
4959 }
4960
4961 // Try to expand UREM by constant.
4962 if (isa<ConstantSDNode>(N->getOperand(1))) {
4963 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4964 // Only if the new type is legal.
4965 if (isTypeLegal(NVT)) {
4966 SDValue InL, InH;
4967 GetExpandedInteger(N->getOperand(0), InL, InH);
4969 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
4970 Lo = Result[0];
4971 Hi = Result[1];
4972 return;
4973 }
4974 }
4975 }
4976
4977 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4978 if (VT == MVT::i16)
4979 LC = RTLIB::UREM_I16;
4980 else if (VT == MVT::i32)
4981 LC = RTLIB::UREM_I32;
4982 else if (VT == MVT::i64)
4983 LC = RTLIB::UREM_I64;
4984 else if (VT == MVT::i128)
4985 LC = RTLIB::UREM_I128;
4986 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
4987
4989 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4990}
4991
4992void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
4993 SDValue &Lo, SDValue &Hi) {
4994 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4995 SDLoc dl(N);
4996 SDValue Op = N->getOperand(0);
4997 if (Op.getValueType().bitsLE(NVT)) {
4998 // The low part is zero extension of the input (degenerates to a copy).
4999 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
5000 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
5001 } else {
5002 // For example, extension of an i48 to an i64. The operand type necessarily
5003 // promotes to the result type, so will end up being expanded too.
5004 assert(getTypeAction(Op.getValueType()) ==
5006 "Only know how to promote this result!");
5007 SDValue Res = GetPromotedInteger(Op);
5008 assert(Res.getValueType() == N->getValueType(0) &&
5009 "Operand over promoted?");
5010 // Split the promoted operand. This will simplify when it is expanded.
5011 SplitInteger(Res, Lo, Hi);
5012 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5013 Hi = DAG.getZeroExtendInReg(Hi, dl,
5015 ExcessBits));
5016 }
5017}
5018
5019void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
5020 SDValue &Lo, SDValue &Hi) {
5021 SDLoc dl(N);
5022 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
5023 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
5024 SDValue Zero = DAG.getConstant(0, dl, VT);
5025 SDValue Swap = DAG.getAtomicCmpSwap(
5027 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
5028 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
5029
5030 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
5031 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
5032}
5033
5034void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
5035 SDValue &Lo, SDValue &Hi) {
5036 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
5037 // both halves independently.
5038 SDValue Res = TLI.expandVecReduce(N, DAG);
5039 SplitInteger(Res, Lo, Hi);
5040}
5041
5042void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
5043 SDValue &Lo, SDValue &Hi) {
5044 // Delegate to funnel-shift expansion.
5045 SDLoc DL(N);
5046 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
5047 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0),
5048 N->getOperand(0), N->getOperand(1));
5049 SplitInteger(Res, Lo, Hi);
5050}
5051
5052void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
5053 SDValue &Hi) {
5054 // Values numbered from least significant to most significant.
5055 SDValue In1, In2, In3, In4;
5056 GetExpandedInteger(N->getOperand(0), In3, In4);
5057 GetExpandedInteger(N->getOperand(1), In1, In2);
5058 EVT HalfVT = In1.getValueType();
5059
5060 SDLoc DL(N);
5061 unsigned Opc = N->getOpcode();
5062 SDValue ShAmt = N->getOperand(2);
5063 EVT ShAmtVT = ShAmt.getValueType();
5064 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5065
5066 // If the shift amount is at least half the bitwidth, swap the inputs.
5067 unsigned HalfVTBits = HalfVT.getScalarSizeInBits();
5068 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt,
5069 DAG.getConstant(HalfVTBits, DL, ShAmtVT));
5070 SDValue Cond =
5071 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT),
5072 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ);
5073
5074 // Expand to a pair of funnel shifts.
5075 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5076 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT);
5077
5078 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2);
5079 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3);
5080 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4);
5081 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt);
5082 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
5083}
5084
5085void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
5086 SDValue &Hi) {
5087 EVT VT = N->getValueType(0);
5088 EVT HalfVT =
5089 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
5090 SDLoc dl(N);
5091
5092 // We assume VSCALE(1) fits into a legal integer.
5093 APInt One(HalfVT.getSizeInBits(), 1);
5094 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
5095 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
5096 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
5097 SplitInteger(Res, Lo, Hi);
5098}
5099
5100//===----------------------------------------------------------------------===//
5101// Integer Operand Expansion
5102//===----------------------------------------------------------------------===//
5103
5104/// ExpandIntegerOperand - This method is called when the specified operand of
5105/// the specified node is found to need expansion. At this point, all of the
5106/// result types of the node are known to be legal, but other operands of the
5107/// node may need promotion or expansion as well as the specified one.
5108bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
5109 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG));
5110 SDValue Res = SDValue();
5111
5112 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
5113 return false;
5114
5115 switch (N->getOpcode()) {
5116 default:
5117 #ifndef NDEBUG
5118 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
5119 N->dump(&DAG); dbgs() << "\n";
5120 #endif
5121 report_fatal_error("Do not know how to expand this operator's operand!");
5122
5123 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
5124 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
5125 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
5126 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
5127 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
5128 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
5129 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
5130 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
5131 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
5132 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
5134 case ISD::SINT_TO_FP:
5136 case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break;
5137 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
5138 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
5139
5140 case ISD::SHL:
5141 case ISD::SRA:
5142 case ISD::SRL:
5143 case ISD::ROTL:
5144 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
5145 case ISD::RETURNADDR:
5146 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
5147
5148 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
5149 case ISD::STACKMAP:
5150 Res = ExpandIntOp_STACKMAP(N, OpNo);
5151 break;
5152 case ISD::PATCHPOINT:
5153 Res = ExpandIntOp_PATCHPOINT(N, OpNo);
5154 break;
5155 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5156 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5157 Res = ExpandIntOp_VP_STRIDED(N, OpNo);
5158 break;
5159 }
5160
5161 // If the result is null, the sub-method took care of registering results etc.
5162 if (!Res.getNode()) return false;
5163
5164 // If the result is N, the sub-method updated N in place. Tell the legalizer
5165 // core about this.
5166 if (Res.getNode() == N)
5167 return true;
5168
5169 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
5170 "Invalid operand expansion");
5171
5172 ReplaceValueWith(SDValue(N, 0), Res);
5173 return false;
5174}
5175
5176/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
5177/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
5178void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
5179 SDValue &NewRHS,
5180 ISD::CondCode &CCCode,
5181 const SDLoc &dl) {
5182 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5183 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5184 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5185
5186 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
5187 if (RHSLo == RHSHi && isAllOnesConstant(RHSLo)) {
5188 // Equality comparison to -1.
5189 NewLHS = DAG.getNode(ISD::AND, dl, LHSLo.getValueType(), LHSLo, LHSHi);
5190 NewRHS = RHSLo;
5191 return;
5192 }
5193
5194 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
5195 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
5196 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
5197 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
5198 return;
5199 }
5200
5201 // If this is a comparison of the sign bit, just look at the top part.
5202 // X > -1, x < 0
5203 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
5204 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
5205 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
5206 NewLHS = LHSHi;
5207 NewRHS = RHSHi;
5208 return;
5209 }
5210
5211 // FIXME: This generated code sucks.
5212 ISD::CondCode LowCC;
5213 switch (CCCode) {
5214 default: llvm_unreachable("Unknown integer setcc!");
5215 case ISD::SETLT:
5216 case ISD::SETULT: LowCC = ISD::SETULT; break;
5217 case ISD::SETGT:
5218 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
5219 case ISD::SETLE:
5220 case ISD::SETULE: LowCC = ISD::SETULE; break;
5221 case ISD::SETGE:
5222 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
5223 }
5224
5225 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
5226 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
5227 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
5228
5229 // NOTE: on targets without efficient SELECT of bools, we can always use
5230 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
5231 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
5232 nullptr);
5233 SDValue LoCmp, HiCmp;
5234 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
5235 TLI.isTypeLegal(RHSLo.getValueType()))
5236 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
5237 RHSLo, LowCC, false, DagCombineInfo, dl);
5238 if (!LoCmp.getNode())
5239 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
5240 RHSLo, LowCC);
5241 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
5242 TLI.isTypeLegal(RHSHi.getValueType()))
5243 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
5244 RHSHi, CCCode, false, DagCombineInfo, dl);
5245 if (!HiCmp.getNode())
5246 HiCmp =
5247 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
5248 LHSHi, RHSHi, DAG.getCondCode(CCCode));
5249
5250 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
5251 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
5252
5253 bool EqAllowed = ISD::isTrueWhenEqual(CCCode);
5254
5255 // FIXME: Is the HiCmpC->isOne() here correct for
5256 // ZeroOrNegativeOneBooleanContent.
5257 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
5258 (!EqAllowed &&
5259 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
5260 // For LE / GE, if high part is known false, ignore the low part.
5261 // For LT / GT: if low part is known false, return the high part.
5262 // if high part is known true, ignore the low part.
5263 NewLHS = HiCmp;
5264 NewRHS = SDValue();
5265 return;
5266 }
5267
5268 if (LHSHi == RHSHi) {
5269 // Comparing the low bits is enough.
5270 NewLHS = LoCmp;
5271 NewRHS = SDValue();
5272 return;
5273 }
5274
5275 // Lower with SETCCCARRY if the target supports it.
5276 EVT HiVT = LHSHi.getValueType();
5277 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
5278 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
5279
5280 // FIXME: Make all targets support this, then remove the other lowering.
5281 if (HasSETCCCARRY) {
5282 // SETCCCARRY can detect < and >= directly. For > and <=, flip
5283 // operands and condition code.
5284 bool FlipOperands = false;
5285 switch (CCCode) {
5286 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
5287 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
5288 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
5289 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
5290 default: break;
5291 }
5292 if (FlipOperands) {
5293 std::swap(LHSLo, RHSLo);
5294 std::swap(LHSHi, RHSHi);
5295 }
5296 // Perform a wide subtraction, feeding the carry from the low part into
5297 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
5298 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
5299 // zero or positive iff LHS >= RHS.
5300 EVT LoVT = LHSLo.getValueType();
5301 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
5302 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
5303 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
5304 LHSHi, RHSHi, LowCmp.getValue(1),
5305 DAG.getCondCode(CCCode));
5306 NewLHS = Res;
5307 NewRHS = SDValue();
5308 return;
5309 }
5310
5311 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
5312 false, DagCombineInfo, dl);
5313 if (!NewLHS.getNode())
5314 NewLHS =
5315 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
5316 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
5317 NewRHS = SDValue();
5318}
5319
5320SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
5321 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
5322 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
5323 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5324
5325 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5326 // against zero to select between true and false values.
5327 if (!NewRHS.getNode()) {
5328 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
5329 CCCode = ISD::SETNE;
5330 }
5331
5332 // Update N to have the operands specified.
5333 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
5334 DAG.getCondCode(CCCode), NewLHS, NewRHS,
5335 N->getOperand(4)), 0);
5336}
5337
5338SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
5339 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
5340 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
5341 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5342
5343 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5344 // against zero to select between true and false values.
5345 if (!NewRHS.getNode()) {
5346 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
5347 CCCode = ISD::SETNE;
5348 }
5349
5350 // Update N to have the operands specified.
5351 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
5352 N->getOperand(2), N->getOperand(3),
5353 DAG.getCondCode(CCCode)), 0);
5354}
5355
5356SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
5357 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
5358 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
5359 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5360
5361 // If ExpandSetCCOperands returned a scalar, use it.
5362 if (!NewRHS.getNode()) {
5363 assert(NewLHS.getValueType() == N->getValueType(0) &&
5364 "Unexpected setcc expansion!");
5365 return NewLHS;
5366 }
5367
5368 // Otherwise, update N to have the operands specified.
5369 return SDValue(
5370 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
5371}
5372
5373SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
5374 SDValue LHS = N->getOperand(0);
5375 SDValue RHS = N->getOperand(1);
5376 SDValue Carry = N->getOperand(2);
5377 SDValue Cond = N->getOperand(3);
5378 SDLoc dl = SDLoc(N);
5379
5380 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5381 GetExpandedInteger(LHS, LHSLo, LHSHi);
5382 GetExpandedInteger(RHS, RHSLo, RHSHi);
5383
5384 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high.
5385 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
5386 SDValue LowCmp =
5387 DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry);
5388 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
5389 LowCmp.getValue(1), Cond);
5390}
5391
5392SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
5393 // Split the operand and replace with SPLAT_VECTOR_PARTS.
5394 SDValue Lo, Hi;
5395 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5396 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
5397 Hi);
5398}
5399
5400SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
5401 // The value being shifted is legal, but the shift amount is too big.
5402 // It follows that either the result of the shift is undefined, or the
5403 // upper half of the shift amount is zero. Just use the lower half.
5404 SDValue Lo, Hi;
5405 GetExpandedInteger(N->getOperand(1), Lo, Hi);
5406 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
5407}
5408
5409SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
5410 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
5411 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
5412 // constant to valid type.
5413 SDValue Lo, Hi;
5414 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5415 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
5416}
5417
5418SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) {
5419 bool IsStrict = N->isStrictFPOpcode();
5420 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
5421 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
5422 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
5423 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
5424 EVT DstVT = N->getValueType(0);
5425 RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(Op.getValueType(), DstVT)
5426 : RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
5427 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5428 "Don't know how to expand this XINT_TO_FP!");
5430 CallOptions.setSExt(true);
5431 std::pair<SDValue, SDValue> Tmp =
5432 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
5433
5434 if (!IsStrict)
5435 return Tmp.first;
5436
5437 ReplaceValueWith(SDValue(N, 1), Tmp.second);
5438 ReplaceValueWith(SDValue(N, 0), Tmp.first);
5439 return SDValue();
5440}
5441
5442SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
5443 assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?");
5444
5445 if (ISD::isNormalStore(N))
5446 return ExpandOp_NormalStore(N, OpNo);
5447
5448 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
5449 assert(OpNo == 1 && "Can only expand the stored value so far");
5450
5451 EVT VT = N->getOperand(1).getValueType();
5452 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5453 SDValue Ch = N->getChain();
5454 SDValue Ptr = N->getBasePtr();
5455 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
5456 AAMDNodes AAInfo = N->getAAInfo();
5457 SDLoc dl(N);
5458 SDValue Lo, Hi;
5459
5460 assert(NVT.isByteSized() && "Expanded type not byte sized!");
5461
5462 if (N->getMemoryVT().bitsLE(NVT)) {
5463 GetExpandedInteger(N->getValue(), Lo, Hi);
5464 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
5465 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
5466 AAInfo);
5467 }
5468
5469 if (DAG.getDataLayout().isLittleEndian()) {
5470 // Little-endian - low bits are at low addresses.
5471 GetExpandedInteger(N->getValue(), Lo, Hi);
5472
5473 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
5474 N->getOriginalAlign(), MMOFlags, AAInfo);
5475
5476 unsigned ExcessBits =
5477 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
5478 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
5479
5480 // Increment the pointer to the other half.
5481 unsigned IncrementSize = NVT.getSizeInBits()/8;
5482 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
5483 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
5484 N->getPointerInfo().getWithOffset(IncrementSize),
5485 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
5486 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
5487 }
5488
5489 // Big-endian - high bits are at low addresses. Favor aligned stores at
5490 // the cost of some bit-fiddling.
5491 GetExpandedInteger(N->getValue(), Lo, Hi);
5492
5493 EVT ExtVT = N->getMemoryVT();
5494 unsigned EBytes = ExtVT.getStoreSize();
5495 unsigned IncrementSize = NVT.getSizeInBits()/8;
5496 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5497 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
5498 ExtVT.getSizeInBits() - ExcessBits);
5499
5500 if (ExcessBits < NVT.getSizeInBits()) {
5501 // Transfer high bits from the top of Lo to the bottom of Hi.
5502 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
5503 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
5504 TLI.getPointerTy(DAG.getDataLayout())));
5505 Hi = DAG.getNode(
5506 ISD::OR, dl, NVT, Hi,
5507 DAG.getNode(ISD::SRL, dl, NVT, Lo,
5508 DAG.getConstant(ExcessBits, dl,
5509 TLI.getPointerTy(DAG.getDataLayout()))));
5510 }
5511
5512 // Store both the high bits and maybe some of the low bits.
5513 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
5514 N->getOriginalAlign(), MMOFlags, AAInfo);
5515
5516 // Increment the pointer to the other half.
5517 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
5518 // Store the lowest ExcessBits bits in the second half.
5519 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
5520 N->getPointerInfo().getWithOffset(IncrementSize),
5521 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
5522 N->getOriginalAlign(), MMOFlags, AAInfo);
5523 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
5524}
5525
5526SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
5527 SDValue InL, InH;
5528 GetExpandedInteger(N->getOperand(0), InL, InH);
5529 // Just truncate the low part of the source.
5530 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
5531}
5532
5533SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
5534 SDLoc dl(N);
5535 SDValue Swap =
5536 DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(),
5537 N->getOperand(0), N->getOperand(2), N->getOperand(1),
5538 cast<AtomicSDNode>(N)->getMemOperand());
5539 return Swap.getValue(1);
5540}
5541
5542SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
5543 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5544 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5545
5546 SDValue Hi; // The upper half is dropped out.
5547 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
5548 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi);
5549
5550 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
5551}
5552
5553SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
5554 SDLoc dl(N);
5555
5556 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5557 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5558 EVT OutVT = V0.getValueType();
5559
5560 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
5561}
5562
5563SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
5564 SDLoc dl(N);
5565
5566 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5567 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5568 EVT ResVT = V0.getValueType();
5569 SDValue Res = DAG.getNode(N->getOpcode(), dl,
5570 DAG.getVTList(ResVT, ResVT), V0, V1);
5571 SetPromotedInteger(SDValue(N, 0), Res.getValue(0));
5572 SetPromotedInteger(SDValue(N, 1), Res.getValue(1));
5573 return SDValue();
5574}
5575
5576SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
5577
5578 EVT OutVT = N->getValueType(0);
5579 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5580 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5581 EVT NOutVTElem = NOutVT.getVectorElementType();
5582
5583 SDLoc dl(N);
5584 SDValue BaseIdx = N->getOperand(1);
5585
5586 // TODO: We may be able to use this for types other than scalable
5587 // vectors and fix those tests that expect BUILD_VECTOR to be used
5588 if (OutVT.isScalableVector()) {
5589 SDValue InOp0 = N->getOperand(0);
5590 EVT InVT = InOp0.getValueType();
5591
5592 // Try and extract from a smaller type so that it eventually falls
5593 // into the promotion code below.
5594 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
5595 getTypeAction(InVT) == TargetLowering::TypeLegal) {
5596 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
5597 unsigned NElts = NInVT.getVectorMinNumElements();
5598 uint64_t IdxVal = BaseIdx->getAsZExtVal();
5599
5600 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
5601 DAG.getConstant(alignDown(IdxVal, NElts), dl,
5602 BaseIdx.getValueType()));
5603 SDValue Step2 = DAG.getNode(
5604 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
5605 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
5606 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
5607 }
5608
5609 // Try and extract from a widened type.
5610 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
5611 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5612 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
5613 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
5614 }
5615
5616 // Promote operands and see if this is handled by target lowering,
5617 // Otherwise, use the BUILD_VECTOR approach below
5618 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
5619 // Collect the (promoted) operands
5620 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5621
5622 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
5623 assert(PromEltVT.bitsLE(NOutVTElem) &&
5624 "Promoted operand has an element type greater than result");
5625
5626 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
5627 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
5628 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
5629 }
5630 }
5631
5632 if (OutVT.isScalableVector())
5633 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
5634
5635 SDValue InOp0 = N->getOperand(0);
5636 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
5637 InOp0 = GetPromotedInteger(N->getOperand(0));
5638
5639 EVT InVT = InOp0.getValueType();
5640
5641 unsigned OutNumElems = OutVT.getVectorNumElements();
5643 Ops.reserve(OutNumElems);
5644 for (unsigned i = 0; i != OutNumElems; ++i) {
5645
5646 // Extract the element from the original vector.
5647 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
5648 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
5650 InVT.getVectorElementType(), N->getOperand(0), Index);
5651
5652 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
5653 // Insert the converted element to the new vector.
5654 Ops.push_back(Op);
5655 }
5656
5657 return DAG.getBuildVector(NOutVT, dl, Ops);
5658}
5659
5660SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
5661 EVT OutVT = N->getValueType(0);
5662 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5663 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5664
5665 SDLoc dl(N);
5666 SDValue Vec = N->getOperand(0);
5667 SDValue SubVec = N->getOperand(1);
5668 SDValue Idx = N->getOperand(2);
5669
5670 EVT SubVecVT = SubVec.getValueType();
5671 EVT NSubVT =
5673 SubVecVT.getVectorElementCount());
5674
5675 Vec = GetPromotedInteger(Vec);
5676 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);
5677
5678 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
5679}
5680
5681SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
5682 SDLoc dl(N);
5683
5684 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5685 EVT OutVT = V0.getValueType();
5686
5687 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
5688}
5689
5690SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
5691 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
5692 EVT VT = N->getValueType(0);
5693 SDLoc dl(N);
5694
5695 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
5696
5697 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5698 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5699 EVT OutVT = V0.getValueType();
5700
5701 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
5702}
5703
5704SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
5705 EVT OutVT = N->getValueType(0);
5706 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5707 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5708 unsigned NumElems = N->getNumOperands();
5709 EVT NOutVTElem = NOutVT.getVectorElementType();
5710 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT);
5711 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType);
5712 SDLoc dl(N);
5713
5715 Ops.reserve(NumElems);
5716 for (unsigned i = 0; i != NumElems; ++i) {
5717 SDValue Op = N->getOperand(i);
5718 EVT OpVT = Op.getValueType();
5719 // BUILD_VECTOR integer operand types are allowed to be larger than the
5720 // result's element type. This may still be true after the promotion. For
5721 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
5722 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
5723 if (OpVT.bitsLT(NOutVTElem)) {
5724 unsigned ExtOpc = ISD::ANY_EXTEND;
5725 // Attempt to extend constant bool vectors to match target's BooleanContent.
5726 // While not necessary, this improves chances of the constant correctly
5727 // folding with compare results (e.g. for NOT patterns).
5728 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant)
5729 ExtOpc = NOutExtOpc;
5730 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op);
5731 }
5732 Ops.push_back(Op);
5733 }
5734
5735 return DAG.getBuildVector(NOutVT, dl, Ops);
5736}
5737
5738SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) {
5739
5740 SDLoc dl(N);
5741
5742 assert(!N->getOperand(0).getValueType().isVector() &&
5743 "Input must be a scalar");
5744
5745 EVT OutVT = N->getValueType(0);
5746 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5747 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5748 EVT NOutElemVT = NOutVT.getVectorElementType();
5749
5750 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0));
5751
5752 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op);
5753}
5754
5755SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
5756 SDLoc dl(N);
5757 EVT OutVT = N->getValueType(0);
5758 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5759 assert(NOutVT.isScalableVector() &&
5760 "Type must be promoted to a scalable vector type");
5761 const APInt &StepVal = N->getConstantOperandAPInt(0);
5762 return DAG.getStepVector(dl, NOutVT,
5763 StepVal.sext(NOutVT.getScalarSizeInBits()));
5764}
5765
5766SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
5767 SDLoc dl(N);
5768
5769 EVT OutVT = N->getValueType(0);
5770 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5771 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5772
5773 unsigned NumOperands = N->getNumOperands();
5774 unsigned NumOutElem = NOutVT.getVectorMinNumElements();
5775 EVT OutElemTy = NOutVT.getVectorElementType();
5776 if (OutVT.isScalableVector()) {
5777 // Find the largest promoted element type for each of the operands.
5778 SDUse *MaxSizedValue = std::max_element(
5779 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) {
5780 EVT AVT = A.getValueType().getVectorElementType();
5781 EVT BVT = B.getValueType().getVectorElementType();
5782 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5783 });
5784 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType();
5785
5786 // Then promote all vectors to the largest element type.
5788 for (unsigned I = 0; I < NumOperands; ++I) {
5789 SDValue Op = N->getOperand(I);
5790 EVT OpVT = Op.getValueType();
5791 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger)
5792 Op = GetPromotedInteger(Op);
5793 else
5794 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal &&
5795 "Unhandled legalization type");
5796
5798 MaxElementVT.getScalarSizeInBits())
5799 Op = DAG.getAnyExtOrTrunc(Op, dl,
5800 OpVT.changeVectorElementType(MaxElementVT));
5801 Ops.push_back(Op);
5802 }
5803
5804 // Do the CONCAT on the promoted type and finally truncate to (the promoted)
5805 // NOutVT.
5806 return DAG.getAnyExtOrTrunc(
5808 OutVT.changeVectorElementType(MaxElementVT), Ops),
5809 dl, NOutVT);
5810 }
5811
5812 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
5813 assert(NumElem * NumOperands == NumOutElem &&
5814 "Unexpected number of elements");
5815
5816 // Take the elements from the first vector.
5817 SmallVector<SDValue, 8> Ops(NumOutElem);
5818 for (unsigned i = 0; i < NumOperands; ++i) {
5819 SDValue Op = N->getOperand(i);
5820 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
5821 Op = GetPromotedInteger(Op);
5822 EVT SclrTy = Op.getValueType().getVectorElementType();
5823 assert(NumElem == Op.getValueType().getVectorNumElements() &&
5824 "Unexpected number of elements");
5825
5826 for (unsigned j = 0; j < NumElem; ++j) {
5827 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
5828 DAG.getVectorIdxConstant(j, dl));
5829 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
5830 }
5831 }
5832
5833 return DAG.getBuildVector(NOutVT, dl, Ops);
5834}
5835
5836SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
5837 EVT VT = N->getValueType(0);
5838 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5839 assert(NVT.isVector() && "This type must be promoted to a vector type");
5840
5841 SDLoc dl(N);
5842
5843 // For operands whose TypeAction is to promote, extend the promoted node
5844 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
5845 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
5846 // type..
5847 if (getTypeAction(N->getOperand(0).getValueType())
5849 SDValue Promoted;
5850
5851 switch(N->getOpcode()) {
5853 Promoted = SExtPromotedInteger(N->getOperand(0));
5854 break;
5856 Promoted = ZExtPromotedInteger(N->getOperand(0));
5857 break;
5859 Promoted = GetPromotedInteger(N->getOperand(0));
5860 break;
5861 default:
5862 llvm_unreachable("Node has unexpected Opcode");
5863 }
5864 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
5865 }
5866
5867 // Directly extend to the appropriate transform-to type.
5868 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
5869}
5870
5871SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
5872 EVT OutVT = N->getValueType(0);
5873 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5874 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5875
5876 EVT NOutVTElem = NOutVT.getVectorElementType();
5877
5878 SDLoc dl(N);
5879 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5880
5881 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
5882 NOutVTElem, N->getOperand(1));
5883 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
5884 V0, ConvElem, N->getOperand(2));
5885}
5886
5887SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
5888 // The VECREDUCE result size may be larger than the element size, so
5889 // we can simply change the result type.
5890 SDLoc dl(N);
5891 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5892 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
5893}
5894
5895SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
5896 // The VP_REDUCE result size may be larger than the element size, so we can
5897 // simply change the result type. However the start value and result must be
5898 // the same.
5899 SDLoc DL(N);
5900 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
5901 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
5902 N->getOperand(1), N->getOperand(2), N->getOperand(3));
5903}
5904
5905SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
5906 SDLoc dl(N);
5907 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5908 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
5909 TLI.getVectorIdxTy(DAG.getDataLayout()));
5911 V0->getValueType(0).getScalarType(), V0, V1);
5912
5913 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
5914 // element types. If this is the case then we need to expand the outgoing
5915 // value and not truncate it.
5916 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5917}
5918
5919SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
5920 SDLoc dl(N);
5921 // The result type is equal to the first input operand's type, so the
5922 // type that needs promoting must be the second source vector.
5923 SDValue V0 = N->getOperand(0);
5924 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5925 SDValue Idx = N->getOperand(2);
5926 EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
5929 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
5930 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
5931 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5932}
5933
5934SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
5935 SDLoc dl(N);
5936 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5937 MVT InVT = V0.getValueType().getSimpleVT();
5939 N->getValueType(0).getVectorNumElements());
5940 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
5941 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
5942}
5943
5944SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
5945 SDLoc dl(N);
5946
5947 EVT ResVT = N->getValueType(0);
5948 unsigned NumElems = N->getNumOperands();
5949
5950 if (ResVT.isScalableVector()) {
5951 SDValue ResVec = DAG.getUNDEF(ResVT);
5952
5953 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5954 SDValue Op = N->getOperand(OpIdx);
5955 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
5956 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
5957 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
5958 }
5959
5960 return ResVec;
5961 }
5962
5963 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
5964
5966 NewOps.reserve(NumElems);
5967
5968 // For each incoming vector
5969 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5970 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
5971 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
5972 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
5973
5974 for (unsigned i=0; i<NumElem; ++i) {
5975 // Extract element from incoming vector
5976 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
5977 DAG.getVectorIdxConstant(i, dl));
5978 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
5979 NewOps.push_back(Tr);
5980 }
5981 }
5982
5983 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
5984}
5985
5986SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
5987 assert(OpNo > 1);
5988 SDValue Op = N->getOperand(OpNo);
5989
5990 // FIXME: Non-constant operands are not yet handled:
5991 // - https://github.com/llvm/llvm-project/issues/26431
5992 // - https://github.com/llvm/llvm-project/issues/55957
5993 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
5994 if (!CN)
5995 return SDValue();
5996
5997 // Copy operands before the one being expanded.
5998 SmallVector<SDValue> NewOps;
5999 for (unsigned I = 0; I < OpNo; I++)
6000 NewOps.push_back(N->getOperand(I));
6001
6002 EVT Ty = Op.getValueType();
6003 SDLoc DL = SDLoc(N);
6004 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6005 NewOps.push_back(
6006 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6007 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6008 } else {
6009 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6010 return SDValue();
6011 }
6012
6013 // Copy remaining operands.
6014 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6015 NewOps.push_back(N->getOperand(I));
6016
6017 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6018
6019 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6020 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6021
6022 return SDValue(); // Signal that we have replaced the node already.
6023}
6024
6025SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
6026 assert(OpNo >= 7);
6027 SDValue Op = N->getOperand(OpNo);
6028
6029 // FIXME: Non-constant operands are not yet handled:
6030 // - https://github.com/llvm/llvm-project/issues/26431
6031 // - https://github.com/llvm/llvm-project/issues/55957
6032 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
6033 if (!CN)
6034 return SDValue();
6035
6036 // Copy operands before the one being expanded.
6037 SmallVector<SDValue> NewOps;
6038 for (unsigned I = 0; I < OpNo; I++)
6039 NewOps.push_back(N->getOperand(I));
6040
6041 EVT Ty = Op.getValueType();
6042 SDLoc DL = SDLoc(N);
6043 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6044 NewOps.push_back(
6045 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6046 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6047 } else {
6048 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6049 return SDValue();
6050 }
6051
6052 // Copy remaining operands.
6053 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6054 NewOps.push_back(N->getOperand(I));
6055
6056 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6057
6058 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6059 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6060
6061 return SDValue(); // Signal that we have replaced the node already.
6062}
#define Success
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:531
#define I(x, y, z)
Definition: MD5.cpp:58
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1463
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:184
unsigned countLeadingOnes() const
Definition: APInt.h:1574
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1160
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1227
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
unsigned countTrailingZeros() const
Definition: APInt.h:1597
unsigned countLeadingZeros() const
Definition: APInt.h:1556
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:954
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1235
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:274
unsigned countTrailingOnes() const
Definition: APInt.h:1612
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:829
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1199
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:195
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:145
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
Definition: Constant.h:41
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:238
bool isBigEndian() const
Definition: DataLayout.h:239
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:954
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
Definition: SmallVector.h:676
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
Definition: TypeSize.h:256
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
Definition: TypeSize.h:264
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:239
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:820
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:885
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1188
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:758
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:586
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:646
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1225
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:880
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1266
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1329
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1248
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:809
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:591
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1182
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:925
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition: ISDOpcodes.h:636
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:831
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1070
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:580
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:313
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1581
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:1556
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
Definition: ISDOpcodes.h:1562
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Definition: ISDOpcodes.h:1568
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ AfterLegalizeTypes
Definition: DAGCombine.h:17
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:428
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:246
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:101
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:77
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:238
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
Definition: KnownBits.h:292
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)