LLVM 24.0.0git
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type. For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type. For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//
19
20#include "LegalizeTypes.h"
29#include <algorithm>
30using namespace llvm;
31
32#define DEBUG_TYPE "legalize-types"
33
34//===----------------------------------------------------------------------===//
35// Integer Result Promotion
36//===----------------------------------------------------------------------===//
37
38/// PromoteIntegerResult - This method is called when a result of a node is
39/// found to be in need of promotion to a larger type. At this point, the node
40/// may also have invalid operands or may have other results that need
41/// expansion, we just know that (at least) one result needs promotion.
42void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
43 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
44 SDValue Res = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
48 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
49 return;
50 }
51
52 switch (N->getOpcode()) {
53 default:
54#ifndef NDEBUG
55 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
56 N->dump(&DAG); dbgs() << "\n";
57#endif
58 report_fatal_error("Do not know how to promote this operator!");
59 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
60 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
61 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
62 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
63 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
64 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
65 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
66 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
68 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
69 case ISD::CTLS: Res = PromoteIntRes_CTLS(N); break;
70 case ISD::PARITY:
71 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
73 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
75 case ISD::CTTZ_ELTS:
76 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
77 case ISD::VP_CTTZ_ELTS:
78 Res = PromoteIntRes_VP_CttzElements(N);
79 break;
81 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
82 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
83 case ISD::VP_LOAD:
84 Res = PromoteIntRes_VP_LOAD(cast<VPLoadSDNode>(N));
85 break;
86 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
87 break;
88 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
89 break;
91 Res = PromoteIntRes_VECTOR_COMPRESS(N);
92 break;
94 Res = PromoteIntRes_VECTOR_SHUFFLE_VAR(N);
95 break;
96 case ISD::SELECT:
97 case ISD::VSELECT:
98 case ISD::VP_MERGE:
99 Res = PromoteIntRes_Select(N);
100 break;
101 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
104 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
105
106 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
108 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
109 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
110 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
111 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
112 case ISD::POISON:
113 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
114 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
115 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
116
118 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
120 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
122 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
124 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
127 Res = PromoteIntRes_VECTOR_SPLICE(N);
128 break;
129 case ISD::EXPERIMENTAL_VP_SPLICE:
130 Res = PromoteIntRes_VP_SPLICE(N);
131 break;
133 Res = PromoteIntRes_VECTOR_REPEAT(N);
134 break;
137 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
138 return;
140 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
142 Res = PromoteIntRes_BUILD_VECTOR(N);
143 break;
146 Res = PromoteIntRes_ScalarOp(N);
147 break;
148 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
150 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
151
155 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
156
158 Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(N);
159 break;
160
162 Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
163 break;
165 Res = PromoteIntRes_MASK_BEFOREFIRST(N);
166 break;
168 Res = PromoteIntRes_VECTOR_MATCH(N);
169 break;
170
174 Res = PromoteIntRes_PARTIAL_REDUCE_MLA(N);
175 break;
176
177 case ISD::SIGN_EXTEND:
178 case ISD::ZERO_EXTEND:
179 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
180
183 case ISD::FP_TO_SINT:
184 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
185
188 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
189
190 case ISD::FP_TO_BF16:
191 case ISD::FP_TO_FP16:
192 Res = PromoteIntRes_FP_TO_FP16_BF16(N);
193 break;
195 Res = PromoteIntRes_CONVERT_TO_ARBITRARY_FP(N);
196 break;
199 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
200 break;
201 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
202
203 case ISD::AND:
204 case ISD::OR:
205 case ISD::XOR:
206 case ISD::ADD:
207 case ISD::SUB:
208 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
209
210 case ISD::SMIN:
211 case ISD::SMAX:
212 case ISD::ABDS:
213 case ISD::AVGCEILS:
214 case ISD::AVGFLOORS:
215 case ISD::SDIV:
216 case ISD::SREM:
217 case ISD::VP_SDIV:
218 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
219
220 case ISD::AVGCEILU:
221 case ISD::AVGFLOORU:
222 case ISD::UDIV:
223 case ISD::UREM:
224 case ISD::VP_UDIV:
225 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
226
227 case ISD::UMIN:
228 case ISD::UMAX:
229 case ISD::ABDU: Res = PromoteIntRes_SExtOrZExtIntBinOp(N); break;
230
231 case ISD::MASKED_UDIV:
232 case ISD::MASKED_UREM:
233 Res = PromoteIntRes_ZExtMaskedIntBinOp(N);
234 break;
235 case ISD::MASKED_SDIV:
236 case ISD::MASKED_SREM:
237 Res = PromoteIntRes_SExtMaskedIntBinOp(N);
238 break;
239
240 case ISD::SADDO:
241 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
242 case ISD::UADDO:
243 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
244 case ISD::SMULO:
245 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
246
247 case ISD::ADDE:
248 case ISD::SUBE:
249 case ISD::UADDO_CARRY:
250 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
251
252 case ISD::SADDO_CARRY:
253 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
254
255 case ISD::SADDSAT:
256 case ISD::UADDSAT:
257 case ISD::SSUBSAT:
258 case ISD::USUBSAT:
259 case ISD::SSHLSAT:
260 case ISD::USHLSAT:
261 Res = PromoteIntRes_ADDSUBSHLSAT(N);
262 break;
263
264 case ISD::SCMP:
265 case ISD::UCMP:
266 Res = PromoteIntRes_CMP(N);
267 break;
268
269 case ISD::SMULFIX:
270 case ISD::SMULFIXSAT:
271 case ISD::UMULFIX:
272 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
273
274 case ISD::SDIVFIX:
275 case ISD::SDIVFIXSAT:
276 case ISD::UDIVFIX:
277 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
278
279 case ISD::ABS:
281 Res = PromoteIntRes_ABS(N);
282 break;
283
284 case ISD::ATOMIC_LOAD:
285 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
286
298 case ISD::ATOMIC_SWAP:
299 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
300
303 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
304 break;
305
315 Res = PromoteIntRes_VECREDUCE(N);
316 break;
317
318 case ISD::VP_REDUCE_ADD:
319 case ISD::VP_REDUCE_MUL:
320 case ISD::VP_REDUCE_AND:
321 case ISD::VP_REDUCE_OR:
322 case ISD::VP_REDUCE_XOR:
323 case ISD::VP_REDUCE_SMAX:
324 case ISD::VP_REDUCE_SMIN:
325 case ISD::VP_REDUCE_UMAX:
326 case ISD::VP_REDUCE_UMIN:
327 Res = PromoteIntRes_VP_REDUCE(N);
328 break;
329
332 Res = PromoteIntRes_LOOP_DEPENDENCE_MASK(N);
333 break;
334
335 case ISD::FREEZE:
336 Res = PromoteIntRes_FREEZE(N);
337 break;
338
339 case ISD::ROTL:
340 case ISD::ROTR:
341 Res = PromoteIntRes_Rotate(N);
342 break;
343
344 case ISD::FSHL:
345 case ISD::FSHR:
346 Res = PromoteIntRes_FunnelShift(N);
347 break;
348
349 case ISD::CLMUL:
350 case ISD::CLMULH:
351 case ISD::CLMULR:
352 Res = PromoteIntRes_CLMUL(N);
353 break;
354
355 case ISD::PEXT:
356 Res = PromoteIntRes_PEXT(N);
357 break;
358
359 case ISD::PDEP:
360 Res = PromoteIntRes_PDEP(N);
361 break;
362
363 case ISD::MULHS:
364 case ISD::MULHU:
365 Res = PromoteIntRes_MULH(N);
366 break;
367
368 case ISD::IS_FPCLASS:
369 Res = PromoteIntRes_IS_FPCLASS(N);
370 break;
371 case ISD::FFREXP:
372 Res = PromoteIntRes_FFREXP(N);
373 break;
374
375 case ISD::LRINT:
376 case ISD::LLRINT:
377 Res = PromoteIntRes_XRINT(N);
378 break;
379
380 case ISD::PATCHPOINT:
381 Res = PromoteIntRes_PATCHPOINT(N);
382 break;
384 Res = PromoteIntRes_READ_REGISTER(N);
385 break;
386 }
387
388 // If the result is null then the sub-method took care of registering it.
389 if (Res.getNode())
390 SetPromotedInteger(SDValue(N, ResNo), Res);
391}
392
393SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
394 unsigned ResNo) {
395 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
396 return GetPromotedInteger(Op);
397}
398
399SDValue DAGTypeLegalizer::PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
400 EVT VT = N->getValueType(0);
401 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
402 return DAG.getNode(N->getOpcode(), SDLoc(N), NewVT, N->ops());
403}
404
405SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
406 // Sign-extend the new bits, and continue the assertion.
407 SDValue Op = SExtPromotedInteger(N->getOperand(0));
408 return DAG.getNode(ISD::AssertSext, SDLoc(N),
409 Op.getValueType(), Op, N->getOperand(1));
410}
411
412SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
413 // Zero the new bits, and continue the assertion.
414 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
415 return DAG.getNode(ISD::AssertZext, SDLoc(N),
416 Op.getValueType(), Op, N->getOperand(1));
417}
418
419SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
420 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
421 ISD::LoadExtType ExtType = N->getExtensionType();
422 if (ExtType == ISD::NON_EXTLOAD) {
423 switch (TLI.getExtendForAtomicOps()) {
424 case ISD::SIGN_EXTEND:
425 ExtType = ISD::SEXTLOAD;
426 break;
427 case ISD::ZERO_EXTEND:
428 ExtType = ISD::ZEXTLOAD;
429 break;
430 case ISD::ANY_EXTEND:
431 ExtType = ISD::EXTLOAD;
432 break;
433 default:
434 llvm_unreachable("Invalid atomic op extension");
435 }
436 }
437
438 SDValue Res =
439 DAG.getAtomicLoad(ExtType, SDLoc(N), N->getMemoryVT(), ResVT,
440 N->getChain(), N->getBasePtr(), N->getMemOperand());
441
442 // Legalize the chain result - switch anything that used the old chain to
443 // use the new one.
444 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
445 return Res;
446}
447
448SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
449 SDValue Op2 = N->getOperand(2);
450 switch (TLI.getExtendForAtomicRMWArg(N->getOpcode())) {
451 case ISD::SIGN_EXTEND:
452 Op2 = SExtPromotedInteger(Op2);
453 break;
454 case ISD::ZERO_EXTEND:
455 Op2 = ZExtPromotedInteger(Op2);
456 break;
457 case ISD::ANY_EXTEND:
458 Op2 = GetPromotedInteger(Op2);
459 break;
460 default:
461 llvm_unreachable("Invalid atomic op extension");
462 }
463 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
464 N->getMemoryVT(),
465 N->getChain(), N->getBasePtr(),
466 Op2, N->getMemOperand());
467 // Legalize the chain result - switch anything that used the old chain to
468 // use the new one.
469 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
470 return Res;
471}
472
473SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
474 unsigned ResNo) {
475 if (ResNo == 1) {
477 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
479
480 // Only use the result of getSetCCResultType if it is legal,
481 // otherwise just use the promoted result type (NVT).
482 if (!TLI.isTypeLegal(SVT))
483 SVT = NVT;
484
485 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
486 SDValue Res = DAG.getAtomicCmpSwap(
487 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
488 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
489 N->getMemOperand());
490 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
491 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
492 return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT);
493 }
494
495 // Op2 is used for the comparison and thus must be extended according to the
496 // target's atomic operations. Op3 is merely stored and so can be left alone.
497 SDValue Op2 = N->getOperand(2);
498 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
499 switch (TLI.getExtendForAtomicCmpSwapArg()) {
500 case ISD::SIGN_EXTEND:
501 Op2 = SExtPromotedInteger(Op2);
502 break;
503 case ISD::ZERO_EXTEND:
504 Op2 = ZExtPromotedInteger(Op2);
505 break;
506 case ISD::ANY_EXTEND:
507 Op2 = GetPromotedInteger(Op2);
508 break;
509 default:
510 llvm_unreachable("Invalid atomic op extension");
511 }
512
513 SDVTList VTs =
514 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
515 SDValue Res = DAG.getAtomicCmpSwap(
516 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
517 N->getBasePtr(), Op2, Op3, N->getMemOperand());
518 // Update the use to N with the newly created Res.
519 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
520 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
521 return Res;
522}
523
524SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
525 SDValue InOp = N->getOperand(0);
526 EVT InVT = InOp.getValueType();
527 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
528 EVT OutVT = N->getValueType(0);
529 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
530 SDLoc dl(N);
531
532 switch (getTypeAction(InVT)) {
534 break;
536 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
537 // The input promotes to the same size. Convert the promoted value.
538 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
539 break;
541 // Promote the integer operand by hand.
542 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
544 // Promote the integer operand by hand.
545 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
548 break;
550 // Convert the element to an integer and promote it by hand.
551 if (!NOutVT.isVector())
552 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
553 BitConvertToInteger(GetScalarizedVector(InOp)));
554 break;
556 report_fatal_error("scalarization of scalable vectors is not supported");
558 if (!NOutVT.isVector()) {
559 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
560 // pieces of the input into integers and reassemble in the final type.
561 SDValue Lo, Hi;
562 GetSplitVector(N->getOperand(0), Lo, Hi);
563 Lo = BitConvertToInteger(Lo);
564 Hi = BitConvertToInteger(Hi);
565
566 if (DAG.getDataLayout().isBigEndian())
567 std::swap(Lo, Hi);
568
569 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
570 EVT::getIntegerVT(*DAG.getContext(),
571 NOutVT.getSizeInBits()),
572 JoinIntegers(Lo, Hi));
573 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
574 }
575 break;
576 }
578 // The input is widened to the same size. Convert to the widened value.
579 // Make sure that the outgoing value is not a vector, because this would
580 // make us bitcast between two vectors which are legalized in different ways.
581 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
582 SDValue Res =
583 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
584
585 // For big endian targets we need to shift the casted value or the
586 // interesting bits will end up at the wrong place.
587 if (DAG.getDataLayout().isBigEndian()) {
588 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
589 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
590 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
591 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
592 }
593 return Res;
594 }
595 // If the output type is also a vector and widening it to the same size
596 // as the widened input type would be a legal type, we can widen the bitcast
597 // and handle the promotion after.
598 if (NOutVT.isVector()) {
599 TypeSize WidenInSize = NInVT.getSizeInBits();
600 TypeSize OutSize = OutVT.getSizeInBits();
601 if (WidenInSize.hasKnownScalarFactor(OutSize)) {
602 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
603 EVT WideOutVT =
604 EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(),
605 OutVT.getVectorElementCount() * Scale);
606 if (isTypeLegal(WideOutVT)) {
607 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
608 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
609 DAG.getVectorIdxConstant(0, dl));
610 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
611 }
612 }
613 }
614 }
615
616 // TODO: Handle big endian
617 if (!NOutVT.isVector() && InOp.getValueType().isVector() &&
618 DAG.getDataLayout().isLittleEndian()) {
619 // Pad the vector operand with undef and cast to a wider integer.
620 EVT EltVT = InOp.getValueType().getVectorElementType();
621 TypeSize EltSize = EltVT.getSizeInBits();
622 TypeSize OutSize = NOutVT.getSizeInBits();
623
624 if (OutSize.hasKnownScalarFactor(EltSize)) {
625 unsigned NumEltsWithPadding = OutSize.getKnownScalarFactor(EltSize);
626 EVT WideVecVT =
627 EVT::getVectorVT(*DAG.getContext(), EltVT, NumEltsWithPadding);
628
629 if (isTypeLegal(WideVecVT)) {
630 SDValue Inserted = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT,
631 DAG.getUNDEF(WideVecVT), InOp,
632 DAG.getVectorIdxConstant(0, dl));
633
634 return DAG.getNode(ISD::BITCAST, dl, NOutVT, Inserted);
635 }
636 }
637 }
638
639 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
640 CreateStackStoreLoad(InOp, OutVT));
641}
642
643SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
644 SDValue V = GetPromotedInteger(N->getOperand(0));
645 return DAG.getNode(ISD::FREEZE, SDLoc(N),
646 V.getValueType(), V);
647}
648
649SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
650 SDValue Op = GetPromotedInteger(N->getOperand(0));
651 EVT OVT = N->getValueType(0);
652 EVT NVT = Op.getValueType();
653 SDLoc dl(N);
654
655 // If the larger BSWAP isn't supported by the target, try to expand now.
656 // If we expand later we'll end up with more operations since we lost the
657 // original type. We only do this for scalars since we have a shuffle
658 // based lowering for vectors in LegalizeVectorOps.
659 if (!OVT.isVector() &&
660 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) {
661 if (SDValue Res = TLI.expandBSWAP(N, DAG))
662 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
663 }
664
665 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
666 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
667 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
668 ShAmt);
669}
670
671SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
672 SDValue Op = GetPromotedInteger(N->getOperand(0));
673 EVT OVT = N->getValueType(0);
674 EVT NVT = Op.getValueType();
675 SDLoc dl(N);
676
677 // If the larger BITREVERSE isn't supported by the target, try to expand now.
678 // If we expand later we'll end up with more operations since we lost the
679 // original type. We only do this for scalars since we have a shuffle
680 // based lowering for vectors in LegalizeVectorOps.
681 if (!OVT.isVector() && OVT.isSimple() &&
682 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) {
683 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
684 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
685 }
686
687 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
688 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
689 return DAG.getNode(ISD::SRL, dl, NVT,
690 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt);
691}
692
693SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
694 // The pair element type may be legal, or may not promote to the same type as
695 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
696 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
697 TLI.getTypeToTransformTo(*DAG.getContext(),
698 N->getValueType(0)), JoinIntegers(N->getOperand(0),
699 N->getOperand(1)));
700}
701
702SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
703 EVT VT = N->getValueType(0);
704 // FIXME there is no actual debug info here
705 SDLoc dl(N);
706 // Zero extend things like i1, sign extend everything else. It shouldn't
707 // matter in theory which one we pick, but this tends to give better code?
709 SDValue Result = DAG.getNode(Opc, dl,
710 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
711 SDValue(N, 0));
712 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
713 return Result;
714}
715
716SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
717 EVT OVT = N->getValueType(0);
718 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
719 SDLoc dl(N);
720
721 // If the larger CTLZ isn't supported by the target, try to expand now.
722 // If we expand later we'll end up with more operations since we lost the
723 // original type.
724 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
725 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) &&
726 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_POISON, NVT)) {
727 if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
728 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
729 return Result;
730 }
731 }
732
733 unsigned CtlzOpcode = N->getOpcode();
734 if (CtlzOpcode == ISD::CTLZ) {
735 // Subtract off the extra leading bits in the bigger type.
736 SDValue ExtractLeadingBits = DAG.getConstant(
737 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
738 // Zero extend to the promoted type and do the count there.
739 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
740
741 // At this stage SUB is guaranteed to be positive no-wrap,
742 // that to be used in further KnownBits optimizations.
743 return DAG.getNode(ISD::SUB, dl, NVT,
744 DAG.getNode(N->getOpcode(), dl, NVT, Op),
745 ExtractLeadingBits, SDNodeFlags::NoUnsignedWrap);
746 }
747 if (CtlzOpcode == ISD::CTLZ_ZERO_POISON) {
748 // Any Extend the argument
749 SDValue Op = GetPromotedInteger(N->getOperand(0));
750 // Op = Op << (sizeinbits(NVT) - sizeinbits(Old VT))
751 unsigned SHLAmount = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
752 auto ShiftConst =
753 DAG.getShiftAmountConstant(SHLAmount, Op.getValueType(), dl);
754 Op = DAG.getNode(ISD::SHL, dl, NVT, Op, ShiftConst);
755 return DAG.getNode(CtlzOpcode, dl, NVT, Op);
756 }
757 llvm_unreachable("Invalid CTLZ Opcode");
758}
759
760SDValue DAGTypeLegalizer::PromoteIntRes_CTLS(SDNode *N) {
761 EVT OVT = N->getValueType(0);
762 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
763 SDLoc dl(N);
764
765 SDValue ExtractLeadingBits = DAG.getConstant(
766 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
767
768 SDValue Op = SExtPromotedInteger(N->getOperand(0));
769 return DAG.getNode(ISD::SUB, dl, NVT, DAG.getNode(ISD::CTLS, dl, NVT, Op),
770 ExtractLeadingBits);
771}
772
773SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
774 EVT OVT = N->getValueType(0);
775 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
776
777 // If the larger CTPOP isn't supported by the target, try to expand now.
778 // If we expand later we'll end up with more operations since we lost the
779 // original type.
780 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
781 // TargetLowering.
782 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
783 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) {
784 if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
785 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
786 return Result;
787 }
788 }
789
790 // Zero extend to the promoted type and do the count or parity there.
791 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
792 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
793}
794
795SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
796 SDValue Op = GetPromotedInteger(N->getOperand(0));
797 EVT OVT = N->getValueType(0);
798 EVT NVT = Op.getValueType();
799 SDLoc dl(N);
800
801 // If the larger CTTZ isn't supported by the target, try to expand now.
802 // If we expand later we'll end up with more operations since we lost the
803 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
804 // larger type.
805 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
806 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) &&
807 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_POISON, NVT) &&
808 !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
809 !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
810 if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
811 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
812 return Result;
813 }
814 }
815
816 unsigned NewOpc = N->getOpcode();
817 if (NewOpc == ISD::CTTZ) {
818 // The count is the same in the promoted type except if the original
819 // value was zero. This can be handled by setting the bit just off
820 // the top of the original type.
821 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
822 OVT.getScalarSizeInBits());
823 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
824 NewOpc = ISD::CTTZ_ZERO_POISON;
825 }
826 return DAG.getNode(NewOpc, dl, NVT, Op);
827}
828
829SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(SDNode *N) {
830 SDLoc DL(N);
831 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
832 return DAG.getNode(N->getOpcode(), DL, NewVT, N->ops());
833}
834
835SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
836 SDLoc dl(N);
837 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
838
839 SDValue Op0 = N->getOperand(0);
840 SDValue Op1 = N->getOperand(1);
841
842 // If the input also needs to be promoted, do that first so we can get a
843 // get a good idea for the output type.
844 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
846 SDValue In = GetPromotedInteger(Op0);
847
848 // If the new type is larger than NVT, use it. We probably won't need to
849 // promote it again.
850 EVT SVT = In.getValueType().getScalarType();
851 if (SVT.bitsGE(NVT)) {
852 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
853 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
854 }
855 }
856
857 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
858}
859
860SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
861 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
862 unsigned NewOpc =
863 TLI.getPreferredFPToIntOpcode(N->getOpcode(), N->getValueType(0), NVT);
864 SDLoc dl(N);
865
866 SDValue Res;
867 if (N->isStrictFPOpcode()) {
868 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
869 {N->getOperand(0), N->getOperand(1)});
870 // Legalize the chain result - switch anything that used the old chain to
871 // use the new one.
872 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
873 } else {
874 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
875 }
876
877 // Assert that the converted value fits in the original type. If it doesn't
878 // (eg: because the value being converted is too big), then the result of the
879 // original operation was undefined anyway, so the assert is still correct.
880 //
881 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
882 // before legalization: fp-to-uint16, 65534. -> 0xfffe
883 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
884 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
885 N->getOpcode() == ISD::STRICT_FP_TO_UINT)
888 dl, NVT, Res,
889 DAG.getValueType(N->getValueType(0).getScalarType()));
890}
891
892SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
893 // Promote the result type, while keeping the original width in Op1.
894 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
895 SDLoc dl(N);
896 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
897 N->getOperand(1));
898}
899
900SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
901 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
902 SDLoc dl(N);
903
904 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
905}
906
907// TODO: CONVERT_TO_ARBITRARY_FP also needs an ExpandIntegerResult handler for
908// wider arbitrary FP formats whose integer result requires expansion.
909SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
910 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
911 SDLoc dl(N);
912
913 return DAG.getNode(ISD::CONVERT_TO_ARBITRARY_FP, dl, NVT, N->getOperand(0),
914 N->getOperand(1), N->getOperand(2), N->getOperand(3));
915}
916
917SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
918 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
919 SDLoc dl(N);
920
921 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
922 N->getOperand(0), N->getOperand(1));
923 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
924 return Res;
925}
926
927SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
928 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
929 SDLoc dl(N);
930 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
931}
932
933SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
934 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
935 SDLoc dl(N);
936
937 SDValue Res =
938 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
939
940 // Legalize the chain result - switch anything that used the old chain to
941 // use the new one.
942 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
943 return Res;
944}
945
946SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
947 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
948 SDLoc dl(N);
949
950 if (getTypeAction(N->getOperand(0).getValueType())
952 SDValue Res = GetPromotedInteger(N->getOperand(0));
953 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
954
955 // If the result and operand types are the same after promotion, simplify
956 // to an in-register extension. Unless this is a VP_*_EXTEND.
957 if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
958 // The high bits are not guaranteed to be anything. Insert an extend.
959 if (N->getOpcode() == ISD::SIGN_EXTEND)
960 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
961 DAG.getValueType(N->getOperand(0).getValueType()));
962 if (N->getOpcode() == ISD::ZERO_EXTEND)
963 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
964 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
965 return Res;
966 }
967 }
968
969 // Otherwise, just extend the original operand all the way to the larger type.
970 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
971}
972
973SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
974 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
975 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
976 ISD::LoadExtType ExtType =
977 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
978 SDLoc dl(N);
979 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
980 N->getMemoryVT(), N->getMemOperand());
981
982 // Legalize the chain result - switch anything that used the old chain to
983 // use the new one.
984 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
985 return Res;
986}
987
988SDValue DAGTypeLegalizer::PromoteIntRes_VP_LOAD(VPLoadSDNode *N) {
989 assert(!N->isIndexed() && "Indexed vp_load during type legalization!");
990 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
991 ISD::LoadExtType ExtType = (N->getExtensionType() == ISD::NON_EXTLOAD)
993 : N->getExtensionType();
994 SDLoc dl(N);
995 SDValue Res =
996 DAG.getExtLoadVP(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
997 N->getMask(), N->getVectorLength(), N->getMemoryVT(),
998 N->getMemOperand(), N->isExpandingLoad());
999 // Legalize the chain result - switch anything that used the old chain to
1000 // use the new one.
1001 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1002 return Res;
1003}
1004
1005SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
1006 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1007 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
1008
1009 ISD::LoadExtType ExtType = N->getExtensionType();
1010 if (ExtType == ISD::NON_EXTLOAD)
1011 ExtType = ISD::EXTLOAD;
1012
1013 SDLoc dl(N);
1014 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
1015 N->getOffset(), N->getMask(), ExtPassThru,
1016 N->getMemoryVT(), N->getMemOperand(),
1017 N->getAddressingMode(), ExtType,
1018 N->isExpandingLoad());
1019 // Legalize the chain result - switch anything that used the old chain to
1020 // use the new one.
1021 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1022 return Res;
1023}
1024
1025SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
1026 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1027 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
1028 assert(NVT == ExtPassThru.getValueType() &&
1029 "Gather result type and the passThru argument type should be the same");
1030
1031 ISD::LoadExtType ExtType = N->getExtensionType();
1032 if (ExtType == ISD::NON_EXTLOAD)
1033 ExtType = ISD::EXTLOAD;
1034
1035 SDLoc dl(N);
1036 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
1037 N->getIndex(), N->getScale() };
1038 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
1039 N->getMemoryVT(), dl, Ops,
1040 N->getMemOperand(), N->getIndexType(),
1041 ExtType);
1042 // Legalize the chain result - switch anything that used the old chain to
1043 // use the new one.
1044 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1045 return Res;
1046}
1047
1048SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(SDNode *N) {
1049 SDValue Vec = GetPromotedInteger(N->getOperand(0));
1050 SDValue Passthru = GetPromotedInteger(N->getOperand(2));
1051 return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), Vec.getValueType(), Vec,
1052 N->getOperand(1), Passthru);
1053}
1054
1055SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
1056 SDValue V = GetPromotedInteger(N->getOperand(0));
1057 return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, SDLoc(N), V.getValueType(), V,
1058 N->getOperand(1));
1059}
1060
1061/// Promote the overflow flag of an overflowing arithmetic node.
1062SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
1063 // Change the return type of the boolean result while obeying
1064 // getSetCCResultType.
1065 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1066 EVT VT = N->getValueType(0);
1067 EVT SVT = getSetCCResultType(VT);
1068 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
1069 unsigned NumOps = N->getNumOperands();
1070 assert(NumOps <= 3 && "Too many operands");
1071 if (NumOps == 3)
1072 Ops[2] = PromoteTargetBoolean(N->getOperand(2), VT);
1073
1074 SDLoc dl(N);
1075 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
1076 ArrayRef(Ops, NumOps));
1077
1078 // Modified the sum result - switch anything that used the old sum to use
1079 // the new one.
1080 ReplaceValueWith(SDValue(N, 0), Res);
1081
1082 // Convert to the expected type.
1083 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
1084}
1085
1086SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
1087 // If the promoted type is legal, we can convert this to:
1088 // 1. ANY_EXTEND iN to iM
1089 // 2. SHL by M-N
1090 // 3. [US][ADD|SUB|SHL]SAT
1091 // 4. L/ASHR by M-N
1092 // Else it is more efficient to convert this to a min and a max
1093 // operation in the higher precision arithmetic.
1094 SDLoc dl(N);
1095 SDValue Op1 = N->getOperand(0);
1096 SDValue Op2 = N->getOperand(1);
1097
1098 unsigned Opcode = N->getOpcode();
1099 unsigned OldBits = Op1.getScalarValueSizeInBits();
1100
1101 // USUBSAT can always be promoted as long as we have zero/sign-extended the
1102 // args.
1103 if (Opcode == ISD::USUBSAT) {
1104 SExtOrZExtPromotedOperands(Op1, Op2);
1105 return DAG.getNode(ISD::USUBSAT, dl, Op1.getValueType(), Op1, Op2);
1106 }
1107
1108 if (Opcode == ISD::UADDSAT) {
1109 EVT OVT = Op1.getValueType();
1110 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1111 // We can promote if we use sign-extend. Do this if the target prefers.
1112 if (TLI.isSExtCheaperThanZExt(OVT, NVT)) {
1113 Op1 = SExtPromotedInteger(Op1);
1114 Op2 = SExtPromotedInteger(Op2);
1115 return DAG.getNode(ISD::UADDSAT, dl, NVT, Op1, Op2);
1116 }
1117
1118 Op1 = ZExtPromotedInteger(Op1);
1119 Op2 = ZExtPromotedInteger(Op2);
1120 unsigned NewBits = NVT.getScalarSizeInBits();
1121 APInt MaxVal = APInt::getLowBitsSet(NewBits, OldBits);
1122 SDValue SatMax = DAG.getConstant(MaxVal, dl, NVT);
1123 SDValue Add = DAG.getNode(ISD::ADD, dl, NVT, Op1, Op2);
1124 return DAG.getNode(ISD::UMIN, dl, NVT, Add, SatMax);
1125 }
1126
1127 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
1128
1129 // FIXME: We need vp-aware PromotedInteger functions.
1130 if (IsShift) {
1131 Op1 = GetPromotedInteger(Op1);
1132 if (getTypeAction(Op2.getValueType()) == TargetLowering::TypePromoteInteger)
1133 Op2 = ZExtPromotedInteger(Op2);
1134 } else {
1135 Op1 = SExtPromotedInteger(Op1);
1136 Op2 = SExtPromotedInteger(Op2);
1137 }
1138 EVT PromotedType = Op1.getValueType();
1139 unsigned NewBits = PromotedType.getScalarSizeInBits();
1140
1141 // Shift cannot use a min/max expansion, we can't detect overflow if all of
1142 // the bits have been shifted out.
1143 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
1144 unsigned ShiftOp;
1145 switch (Opcode) {
1146 case ISD::SADDSAT:
1147 case ISD::SSUBSAT:
1148 case ISD::SSHLSAT:
1149 ShiftOp = ISD::SRA;
1150 break;
1151 case ISD::USHLSAT:
1152 ShiftOp = ISD::SRL;
1153 break;
1154 default:
1155 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
1156 "addition, subtraction or left shift");
1157 }
1158
1159 unsigned SHLAmount = NewBits - OldBits;
1160 SDValue ShiftAmount =
1161 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
1162 Op1 = DAG.getNode(ISD::SHL, dl, PromotedType, Op1, ShiftAmount);
1163 if (!IsShift)
1164 Op2 = DAG.getNode(ISD::SHL, dl, PromotedType, Op2, ShiftAmount);
1165
1166 SDValue Result = DAG.getNode(Opcode, dl, PromotedType, Op1, Op2);
1167 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1168 }
1169
1170 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
1171 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
1172 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
1173 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
1174 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1175 SDValue Result = DAG.getNode(AddOp, dl, PromotedType, Op1, Op2);
1176 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
1177 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
1178 return Result;
1179}
1180
1181SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
1182 // Can just promote the operands then continue with operation.
1183 SDLoc dl(N);
1184 SDValue Op1Promoted, Op2Promoted;
1185 bool Signed =
1186 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
1187 bool Saturating =
1188 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
1189 if (Signed) {
1190 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1191 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1192 } else {
1193 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1194 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1195 }
1196 EVT OldType = N->getOperand(0).getValueType();
1197 EVT PromotedType = Op1Promoted.getValueType();
1198 unsigned DiffSize =
1199 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
1200
1201 if (Saturating) {
1202 // Promoting the operand and result values changes the saturation width,
1203 // which is extends the values that we clamp to on saturation. This could be
1204 // resolved by shifting one of the operands the same amount, which would
1205 // also shift the result we compare against, then shifting back.
1206 Op1Promoted =
1207 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1208 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1209 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1210 Op2Promoted, N->getOperand(2));
1211 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
1212 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
1213 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1214 }
1215 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1216 N->getOperand(2));
1217}
1218
1220 unsigned SatW, bool Signed,
1221 const TargetLowering &TLI,
1222 SelectionDAG &DAG) {
1223 EVT VT = V.getValueType();
1224 unsigned VTW = VT.getScalarSizeInBits();
1225
1226 if (!Signed) {
1227 // Saturate to the unsigned maximum by getting the minimum of V and the
1228 // maximum.
1229 return DAG.getNode(ISD::UMIN, dl, VT, V,
1230 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
1231 dl, VT));
1232 }
1233
1234 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
1235 // signed minimum of it and V.
1236 V = DAG.getNode(ISD::SMIN, dl, VT, V,
1237 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
1238 dl, VT));
1239 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
1240 // signed maximum of it and V.
1241 V = DAG.getNode(ISD::SMAX, dl, VT, V,
1242 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
1243 dl, VT));
1244 return V;
1245}
1246
1248 unsigned Scale, const TargetLowering &TLI,
1249 SelectionDAG &DAG, unsigned SatW = 0) {
1250 EVT VT = LHS.getValueType();
1251 unsigned VTSize = VT.getScalarSizeInBits();
1252 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1253 N->getOpcode() == ISD::SDIVFIXSAT;
1254 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1255 N->getOpcode() == ISD::UDIVFIXSAT;
1256
1257 SDLoc dl(N);
1258 // Widen the types by a factor of two. This is guaranteed to expand, since it
1259 // will always have enough high bits in the LHS to shift into.
1260 EVT WideVT = VT.changeElementType(
1261 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), VTSize * 2));
1262 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT);
1263 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT);
1264 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
1265 DAG);
1266 assert(Res && "Expanding DIVFIX with wide type failed?");
1267 if (Saturating) {
1268 // If the caller has told us to saturate at something less, use that width
1269 // instead of the type before doubling. However, it cannot be more than
1270 // what we just widened!
1271 assert(SatW <= VTSize &&
1272 "Tried to saturate to more than the original type?");
1273 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1274 TLI, DAG);
1275 }
1276 return DAG.getZExtOrTrunc(Res, dl, VT);
1277}
1278
1279SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1280 SDLoc dl(N);
1281 SDValue Op1Promoted, Op2Promoted;
1282 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1283 N->getOpcode() == ISD::SDIVFIXSAT;
1284 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1285 N->getOpcode() == ISD::UDIVFIXSAT;
1286 if (Signed) {
1287 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1288 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1289 } else {
1290 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1291 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1292 }
1293 EVT PromotedType = Op1Promoted.getValueType();
1294 unsigned Scale = N->getConstantOperandVal(2);
1295
1296 // If the type is already legal and the operation is legal in that type, we
1297 // should not early expand.
1298 if (TLI.isTypeLegal(PromotedType)) {
1300 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1301 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1302 unsigned Diff = PromotedType.getScalarSizeInBits() -
1303 N->getValueType(0).getScalarSizeInBits();
1304 if (Saturating)
1305 Op1Promoted =
1306 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1307 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1308 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1309 Op2Promoted, N->getOperand(2));
1310 if (Saturating)
1311 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1312 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1313 return Res;
1314 }
1315 }
1316
1317 // See if we can perform the division in this type without expanding.
1318 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1319 Op2Promoted, Scale, DAG)) {
1320 if (Saturating)
1321 Res = SaturateWidenedDIVFIX(Res, dl,
1322 N->getValueType(0).getScalarSizeInBits(),
1323 Signed, TLI, DAG);
1324 return Res;
1325 }
1326 // If we cannot, expand it to twice the type width. If we are saturating, give
1327 // it the original width as a saturating width so we don't need to emit
1328 // two saturations.
1329 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1330 N->getValueType(0).getScalarSizeInBits());
1331}
1332
1333SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1334 if (ResNo == 1)
1335 return PromoteIntRes_Overflow(N);
1336
1337 // The operation overflowed iff the result in the larger type is not the
1338 // sign extension of its truncation to the original type.
1339 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1340 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1341 EVT OVT = N->getOperand(0).getValueType();
1342 EVT NVT = LHS.getValueType();
1343 SDLoc dl(N);
1344
1345 // Do the arithmetic in the larger type.
1346 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1347 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1348
1349 // Calculate the overflow flag: sign extend the arithmetic result from
1350 // the original type.
1351 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1352 DAG.getValueType(OVT));
1353 // Overflowed if and only if this is not equal to Res.
1354 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1355
1356 // Use the calculated overflow everywhere.
1357 ReplaceValueWith(SDValue(N, 1), Ofl);
1358
1359 return Res;
1360}
1361
1362SDValue DAGTypeLegalizer::PromoteIntRes_CMP(SDNode *N) {
1363 EVT PromotedResultTy =
1364 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1365 return DAG.getNode(N->getOpcode(), SDLoc(N), PromotedResultTy,
1366 N->getOperand(0), N->getOperand(1));
1367}
1368
1369SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1370 SDValue Mask = N->getOperand(0);
1371
1372 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1373 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1374
1375 unsigned Opcode = N->getOpcode();
1376 if (Opcode == ISD::VP_MERGE)
1377 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
1378 N->getOperand(3));
1379 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
1380}
1381
1382SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1383 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1384 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1385 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1386 LHS.getValueType(), N->getOperand(0),
1387 N->getOperand(1), LHS, RHS, N->getOperand(4));
1388}
1389
1390SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1391 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1392 EVT InVT = N->getOperand(OpNo).getValueType();
1393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1394
1395 EVT SVT = getSetCCResultType(InVT);
1396
1397 // If we got back a type that needs to be promoted, this likely means the
1398 // the input type also needs to be promoted. So get the promoted type for
1399 // the input and try the query again.
1400 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1401 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1402 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1403 SVT = getSetCCResultType(InVT);
1404 } else {
1405 // Input type isn't promoted, just use the default promoted type.
1406 SVT = NVT;
1407 }
1408 }
1409
1410 SDLoc dl(N);
1411 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1412 "Vector compare must return a vector result!");
1413
1414 // Get the SETCC result using the canonical SETCC type.
1415 SDValue SetCC;
1416 if (N->isStrictFPOpcode()) {
1417 SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
1418 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1419 N->getOperand(2), N->getOperand(3)};
1420 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
1421 // Legalize the chain result - switch anything that used the old chain to
1422 // use the new one.
1423 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1424 } else
1425 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1426 N->getOperand(1), N->getOperand(2), N->getFlags());
1427
1428 // Convert to the expected type.
1429 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1430}
1431
1432SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1433 SDLoc DL(N);
1434 SDValue Arg = N->getOperand(0);
1435 SDValue Test = N->getOperand(1);
1436 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1437 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
1438}
1439
1440SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
1441 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1442 EVT VT = N->getValueType(0);
1443
1444 SDLoc dl(N);
1445 SDValue Res =
1446 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0));
1447
1448 ReplaceValueWith(SDValue(N, 0), Res);
1449 return Res.getValue(1);
1450}
1451
1452SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1453 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1454 SDValue RHS = N->getOperand(1);
1455 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1456 RHS = ZExtPromotedInteger(RHS);
1457 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1458}
1459
1460SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1461 SDValue Op = GetPromotedInteger(N->getOperand(0));
1462 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1463 Op.getValueType(), Op, N->getOperand(1));
1464}
1465
1466SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1467 // The input may have strange things in the top bits of the registers, but
1468 // these operations don't care. They may have weird bits going out, but
1469 // that too is okay if they are integer operations.
1470 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1471 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1472 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1473}
1474
1475SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1476 // Sign extend the input.
1477 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1478 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1479 if (N->getNumOperands() == 2)
1480 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1481 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1482 assert((N->getOpcode() == ISD::VP_SDIV || N->getOpcode() == ISD::VP_SREM) &&
1483 "Expected VP opcode");
1484 SDValue Mask = N->getOperand(2);
1485 SDValue EVL = N->getOperand(3);
1486 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1487 Mask, EVL);
1488}
1489
1490SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1491 // Zero extend the input.
1492 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1493 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1494 if (N->getNumOperands() == 2)
1495 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1496 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1497 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_UREM) &&
1498 "Expected VP opcode");
1499 // Zero extend the input.
1500 SDValue Mask = N->getOperand(2);
1501 SDValue EVL = N->getOperand(3);
1502 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1503 Mask, EVL);
1504}
1505
1506SDValue DAGTypeLegalizer::PromoteIntRes_ZExtMaskedIntBinOp(SDNode *N) {
1507 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1508 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1509 SDValue Mask = N->getOperand(2);
1510 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1511 Mask);
1512}
1513
1514SDValue DAGTypeLegalizer::PromoteIntRes_SExtMaskedIntBinOp(SDNode *N) {
1515 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1516 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1517 SDValue Mask = N->getOperand(2);
1518 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1519 Mask);
1520}
1521
1522SDValue DAGTypeLegalizer::PromoteIntRes_SExtOrZExtIntBinOp(SDNode *N) {
1523 SDValue LHS = N->getOperand(0);
1524 SDValue RHS = N->getOperand(1);
1525
1526 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1527 // whatever is best for the target and the promoted operands.
1528 SExtOrZExtPromotedOperands(LHS, RHS);
1529
1530 return DAG.getNode(N->getOpcode(), SDLoc(N),
1531 LHS.getValueType(), LHS, RHS);
1532}
1533
1534SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1535 // The input value must be properly sign extended.
1536 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1537 SDValue RHS = N->getOperand(1);
1538 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1539 RHS = ZExtPromotedInteger(RHS);
1540 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1541}
1542
1543SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1544 SDValue RHS = N->getOperand(1);
1545 // The input value must be properly zero extended.
1546 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1547 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1548 RHS = ZExtPromotedInteger(RHS);
1549 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1550}
1551
1552SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1553 EVT OldVT = N->getValueType(0);
1554 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
1555 SDValue Amt = N->getOperand(1);
1556 unsigned Opcode = N->getOpcode();
1557 unsigned OldBits = OldVT.getScalarSizeInBits();
1558 unsigned NewBits = VT.getScalarSizeInBits();
1559
1560 // If the promoted type is twice the size (or more), then we can concatenate
1561 // the value with itself and treat this similar to a funnel shift. This isn't
1562 // necessary if the rotate amount is constant or if shl/srl of the original
1563 // type are custom lowered.
1564 // rotl(x,amt) -> (((aext(x) << bw) | zext(x)) << (amt % bw)) >> bw.
1565 // rotr(x,amt) -> (((aext(x) << bw) | zext(x)) >> (amt % bw)).
1566 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1567 !TLI.isOperationLegalOrCustom(Opcode, VT) &&
1568 TLI.getOperationAction(ISD::SHL, OldVT) != TargetLowering::Custom &&
1569 TLI.getOperationAction(ISD::SRL, OldVT) != TargetLowering::Custom) {
1570 SDValue Op0 = GetPromotedInteger(N->getOperand(0));
1571 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1572 Amt = ZExtPromotedInteger(Amt);
1573 EVT AmtVT = Amt.getValueType();
1574
1575 SDLoc DL(N);
1576 // Amount has to be interpreted modulo the old bit width.
1577 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1578 DAG.getConstant(OldBits, DL, AmtVT));
1579 SDValue HiShift = DAG.getShiftAmountConstant(OldBits, VT, DL);
1580 SDValue Hi = DAG.getNode(ISD::SHL, DL, VT, Op0, HiShift);
1581 SDValue Lo = DAG.getZeroExtendInReg(Op0, DL, OldVT);
1582 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1583 bool IsROTR = N->getOpcode() == ISD::ROTR;
1584 Res = DAG.getNode(IsROTR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1585 // FIXME: We can avoid this by using ROTL when the promoted type is exactly
1586 // twice the size.
1587 if (!IsROTR)
1588 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1589 return Res;
1590 }
1591
1592 // Lower the rotate to shifts and ORs which can be promoted.
1593 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
1594 ReplaceValueWith(SDValue(N, 0), Res);
1595 return SDValue();
1596}
1597
1598SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1599 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1600 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1601 SDValue Amt = N->getOperand(2);
1602 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1603 Amt = ZExtPromotedInteger(Amt);
1604 EVT AmtVT = Amt.getValueType();
1605
1606 SDLoc DL(N);
1607 EVT OldVT = N->getOperand(0).getValueType();
1608 EVT VT = Lo.getValueType();
1609 unsigned Opcode = N->getOpcode();
1610 bool IsFSHR = Opcode == ISD::FSHR;
1611 unsigned OldBits = OldVT.getScalarSizeInBits();
1612 unsigned NewBits = VT.getScalarSizeInBits();
1613
1614 // Amount has to be interpreted modulo the old bit width.
1615 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1616 DAG.getConstant(OldBits, DL, AmtVT));
1617
1618 // If the promoted type is twice the size (or more), then we use the
1619 // traditional funnel 'double' shift codegen. This isn't necessary if the
1620 // shift amount is constant.
1621 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1622 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1623 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1624 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1625 SDValue HiShift = DAG.getShiftAmountConstant(OldBits, VT, DL);
1626 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1627 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1628 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1629 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1630 if (!IsFSHR)
1631 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1632 return Res;
1633 }
1634
1635 // Shift Lo up to occupy the upper bits of the promoted type.
1636 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo,
1637 DAG.getShiftAmountConstant(NewBits - OldBits, VT, DL));
1638
1639 // Increase Amount to shift the result into the lower bits of the promoted
1640 // type.
1641 if (IsFSHR)
1642 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt,
1643 DAG.getConstant(NewBits - OldBits, DL, AmtVT));
1644
1645 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1646}
1647
1648SDValue DAGTypeLegalizer::PromoteIntRes_CLMUL(SDNode *N) {
1649 unsigned Opcode = N->getOpcode();
1650
1651 SDLoc DL(N);
1652 EVT OldVT = N->getOperand(0).getValueType();
1653 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
1654
1655 if (Opcode == ISD::CLMUL) {
1656 // Avoid the generic expansion if the cross-product expansion in
1657 // ExpandIntRes_CLMUL would produce a better result.
1658 if (!TLI.isOperationLegalOrCustomOrPromote(ISD::CLMUL, VT) &&
1659 !(getTypeAction(VT) == TargetLowering::TypeExpandInteger &&
1660 TLI.isOperationLegalOrCustom(
1661 ISD::CLMUL, TLI.getRegisterType(*DAG.getContext(), VT)))) {
1662 if (SDValue Res = TLI.expandCLMUL(N, DAG))
1663 return DAG.getNode(ISD::ANY_EXTEND, DL, VT, Res);
1664 }
1665 SDValue X = GetPromotedInteger(N->getOperand(0));
1666 SDValue Y = GetPromotedInteger(N->getOperand(1));
1667 return DAG.getNode(ISD::CLMUL, DL, VT, X, Y);
1668 }
1669
1670 SDValue X = ZExtPromotedInteger(N->getOperand(0));
1671 SDValue Y = ZExtPromotedInteger(N->getOperand(1));
1672
1673 unsigned OldBits = OldVT.getScalarSizeInBits();
1674 unsigned NewBits = VT.getScalarSizeInBits();
1675 if (NewBits < 2 * OldBits) {
1676 SDValue Clmul = DAG.getNode(ISD::CLMUL, DL, VT, X, Y);
1677 unsigned ShAmt = Opcode == ISD::CLMULH ? OldBits : OldBits - 1;
1678 SDValue Lo = DAG.getNode(ISD::SRL, DL, VT, Clmul,
1679 DAG.getShiftAmountConstant(ShAmt, VT, DL));
1680 SDValue Clmulh = DAG.getNode(ISD::CLMULH, DL, VT, X, Y);
1681 ShAmt = Opcode == ISD::CLMULH ? NewBits - OldBits : NewBits - OldBits + 1;
1682 SDValue Hi = DAG.getNode(ISD::SHL, DL, VT, Clmulh,
1683 DAG.getShiftAmountConstant(ShAmt, VT, DL));
1684 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
1685 }
1686
1687 SDValue Clmul = DAG.getNode(ISD::CLMUL, DL, VT, X, Y);
1688 unsigned ShAmt = Opcode == ISD::CLMULH ? OldBits : OldBits - 1;
1689 return DAG.getNode(ISD::SRL, DL, VT, Clmul,
1690 DAG.getShiftAmountConstant(ShAmt, VT, DL));
1691}
1692
1693SDValue DAGTypeLegalizer::PromoteIntRes_PEXT(SDNode *N) {
1694 SDLoc DL(N);
1695 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1696 if (!TLI.isOperationLegalOrCustomOrPromote(ISD::PEXT, VT)) {
1697 if (SDValue Res = TLI.expandPEXT(N, DAG))
1698 return DAG.getNode(ISD::ANY_EXTEND, DL, VT, Res);
1699 }
1700 // Only the mask operand needs zero-extension because the implicit AND from
1701 // masking clears the corresponding bits in X anyway.
1702 SDValue X = GetPromotedInteger(N->getOperand(0));
1703 SDValue Y = ZExtPromotedInteger(N->getOperand(1));
1704 return DAG.getNode(ISD::PEXT, DL, VT, X, Y);
1705}
1706
1707SDValue DAGTypeLegalizer::PromoteIntRes_PDEP(SDNode *N) {
1708 SDLoc DL(N);
1709 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1710 if (!TLI.isOperationLegalOrCustomOrPromote(ISD::PDEP, VT)) {
1711 if (SDValue Res = TLI.expandPDEP(N, DAG))
1712 return DAG.getNode(ISD::ANY_EXTEND, DL, VT, Res);
1713 }
1714 // Neither operand needs zero-extension because the upper operand bits could
1715 // only result in depositing result bits that will be discarded.
1716 SDValue X = GetPromotedInteger(N->getOperand(0));
1717 SDValue Y = GetPromotedInteger(N->getOperand(1));
1718 return DAG.getNode(ISD::PDEP, DL, VT, X, Y);
1719}
1720
1721SDValue DAGTypeLegalizer::PromoteIntRes_MULH(SDNode *N) {
1722 SDLoc dl(N);
1723 EVT VT = N->getValueType(0);
1724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1725 bool IsSigned = N->getOpcode() == ISD::MULHS;
1726 unsigned BW = VT.getScalarSizeInBits();
1727 unsigned NBW = NVT.getScalarSizeInBits();
1728
1729 SDValue LHS, RHS;
1730 if (IsSigned) {
1731 LHS = SExtPromotedInteger(N->getOperand(0));
1732 RHS = SExtPromotedInteger(N->getOperand(1));
1733 } else {
1734 LHS = ZExtPromotedInteger(N->getOperand(0));
1735 RHS = ZExtPromotedInteger(N->getOperand(1));
1736 }
1737
1738 // If the promoted type is already wide enough, emit a MUL.
1739 if (NBW >= 2 * BW)
1740 return DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, NVT,
1741 DAG.getNode(ISD::MUL, dl, NVT, LHS, RHS),
1742 DAG.getShiftAmountConstant(BW, NVT, dl));
1743
1744 // Otherwise, align an operand with a left-shift and emit a MULH.
1745 LHS = DAG.getNode(ISD::SHL, dl, NVT, LHS,
1746 DAG.getShiftAmountConstant(NBW - BW, NVT, dl));
1747 return DAG.getNode(N->getOpcode(), dl, NVT, LHS, RHS);
1748}
1749
1750SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1751 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1752 SDValue Res;
1753 SDValue InOp = N->getOperand(0);
1754 SDLoc dl(N);
1755
1756 switch (getTypeAction(InOp.getValueType())) {
1757 default: llvm_unreachable("Unknown type action!");
1760 Res = InOp;
1761 break;
1763 Res = GetPromotedInteger(InOp);
1764 break;
1766 EVT InVT = InOp.getValueType();
1767 assert(InVT.isVector() && "Cannot split scalar types");
1768 ElementCount NumElts = InVT.getVectorElementCount();
1769 assert(NumElts == NVT.getVectorElementCount() &&
1770 "Dst and Src must have the same number of elements");
1771 assert(NumElts.isPowerOf2() &&
1772 "Promoted vector type must be a power of two");
1773
1774 SDValue EOp1, EOp2;
1775 GetSplitVector(InOp, EOp1, EOp2);
1776
1777 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1778 NumElts.divideCoefficientBy(2));
1779 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1780 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1781 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1782 }
1784 SDValue WideInOp = GetWidenedVector(InOp);
1785
1786 // Truncate widened InOp.
1787 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1788 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1789 N->getValueType(0).getScalarType(), NumElem);
1790 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1791
1792 // Zero extend so that the elements are of same type as those of NVT
1793 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1794 NumElem);
1795 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1796
1797 // Extract the low NVT subvector.
1798 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1799 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1800 }
1801 }
1802
1803 // Truncate to NVT instead of VT
1804 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1805}
1806
1807SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1808 if (ResNo == 1)
1809 return PromoteIntRes_Overflow(N);
1810
1811 // The operation overflowed iff the result in the larger type is not the
1812 // zero extension of its truncation to the original type.
1813 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1814 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1815 EVT OVT = N->getOperand(0).getValueType();
1816 EVT NVT = LHS.getValueType();
1817 SDLoc dl(N);
1818
1819 // Do the arithmetic in the larger type.
1820 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1821 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1822
1823 // Calculate the overflow flag: zero extend the arithmetic result from
1824 // the original type.
1825 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1826 // Overflowed if and only if this is not equal to Res.
1827 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1828
1829 // Use the calculated overflow everywhere.
1830 ReplaceValueWith(SDValue(N, 1), Ofl);
1831
1832 return Res;
1833}
1834
1835// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
1836// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1837// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
1838SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
1839 unsigned ResNo) {
1840 if (ResNo == 1)
1841 return PromoteIntRes_Overflow(N);
1842
1843 // We need to sign-extend the operands so the carry value computed by the
1844 // wide operation will be equivalent to the carry value computed by the
1845 // narrow operation.
1846 // An UADDO_CARRY can generate carry only if any of the operands has its
1847 // most significant bit set. Sign extension propagates the most significant
1848 // bit into the higher bits which means the extra bit that the narrow
1849 // addition would need (i.e. the carry) will be propagated through the higher
1850 // bits of the wide addition.
1851 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
1852 // be preserved by sign extension.
1853 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1854 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1855
1856 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1857
1858 // Do the arithmetic in the wide type.
1859 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1860 LHS, RHS, N->getOperand(2));
1861
1862 // Update the users of the original carry/borrow value.
1863 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1864
1865 return SDValue(Res.getNode(), 0);
1866}
1867
1868SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1869 unsigned ResNo) {
1870 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1871 return PromoteIntRes_Overflow(N);
1872}
1873
1874SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1875 EVT OVT = N->getValueType(0);
1876 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1877
1878 // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1879 // If we expand later we'll end up sign extending more than just the sra input
1880 // in sra+xor+sub expansion.
1881 if (!OVT.isVector() &&
1882 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) &&
1883 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS_MIN_POISON, NVT) &&
1884 !TLI.isOperationLegal(ISD::SMAX, NVT)) {
1885 if (SDValue Res = TLI.expandABS(N, DAG))
1886 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
1887 }
1888
1889 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1890 return DAG.getNode(ISD::ABS_MIN_POISON, SDLoc(N), Op0.getValueType(), Op0);
1891}
1892
1893SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1894 // Promote the overflow bit trivially.
1895 if (ResNo == 1)
1896 return PromoteIntRes_Overflow(N);
1897
1898 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1899 SDLoc DL(N);
1900 EVT SmallVT = LHS.getValueType();
1901
1902 // To determine if the result overflowed in a larger type, we extend the
1903 // input to the larger type, do the multiply (checking if it overflows),
1904 // then also check the high bits of the result to see if overflow happened
1905 // there.
1906 if (N->getOpcode() == ISD::SMULO) {
1907 LHS = SExtPromotedInteger(LHS);
1908 RHS = SExtPromotedInteger(RHS);
1909 } else {
1910 LHS = ZExtPromotedInteger(LHS);
1911 RHS = ZExtPromotedInteger(RHS);
1912 }
1913 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1914 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1915
1916 // Overflow occurred if it occurred in the larger type, or if the high part
1917 // of the result does not zero/sign-extend the low part. Check this second
1918 // possibility first.
1919 SDValue Overflow;
1920 if (N->getOpcode() == ISD::UMULO) {
1921 // Unsigned overflow occurred if the high part is non-zero.
1922 unsigned Shift = SmallVT.getScalarSizeInBits();
1923 SDValue Hi =
1924 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1925 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
1926 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1927 DAG.getConstant(0, DL, Hi.getValueType()),
1928 ISD::SETNE);
1929 } else {
1930 // Signed overflow occurred if the high part does not sign extend the low.
1931 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1932 Mul, DAG.getValueType(SmallVT));
1933 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1934 }
1935
1936 // The only other way for overflow to occur is if the multiplication in the
1937 // larger type itself overflowed.
1938 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1939 SDValue(Mul.getNode(), 1));
1940
1941 // Use the calculated overflow everywhere.
1942 ReplaceValueWith(SDValue(N, 1), Overflow);
1943 return Mul;
1944}
1945
1946SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1947 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1948 N->getValueType(0)));
1949}
1950
1951SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1952 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1953
1954 const APInt &MulImm = N->getConstantOperandAPInt(0);
1955 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
1956}
1957
1958SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1959 SDValue Chain = N->getOperand(0); // Get the chain.
1960 SDValue Ptr = N->getOperand(1); // Get the pointer.
1961 EVT VT = N->getValueType(0);
1962 SDLoc dl(N);
1963
1964 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1965 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1966 // The argument is passed as NumRegs registers of type RegVT.
1967
1968 SmallVector<SDValue, 8> Parts(NumRegs);
1969 for (unsigned i = 0; i < NumRegs; ++i) {
1970 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1971 N->getConstantOperandVal(3));
1972 Chain = Parts[i].getValue(1);
1973 }
1974
1975 // Handle endianness of the load.
1976 if (DAG.getDataLayout().isBigEndian())
1977 std::reverse(Parts.begin(), Parts.end());
1978
1979 // Assemble the parts in the promoted type.
1980 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1981 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1982 for (unsigned i = 1; i < NumRegs; ++i) {
1983 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1984 // Shift it to the right position and "or" it in.
1985 Part = DAG.getNode(
1986 ISD::SHL, dl, NVT, Part,
1987 DAG.getShiftAmountConstant(i * RegVT.getSizeInBits(), NVT, dl));
1988 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1989 }
1990
1991 // Modified the chain result - switch anything that used the old chain to
1992 // use the new one.
1993 ReplaceValueWith(SDValue(N, 1), Chain);
1994
1995 return Res;
1996}
1997
1998//===----------------------------------------------------------------------===//
1999// Integer Operand Promotion
2000//===----------------------------------------------------------------------===//
2001
2002/// PromoteIntegerOperand - This method is called when the specified operand of
2003/// the specified node is found to need promotion. At this point, all of the
2004/// result types of the node are known to be legal, but other operands of the
2005/// node may need promotion or expansion as well as the specified one.
2006bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
2007 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
2008 SDValue Res = SDValue();
2009 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2010 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2011 return false;
2012 }
2013
2014 switch (N->getOpcode()) {
2015 default:
2016 #ifndef NDEBUG
2017 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
2018 N->dump(&DAG); dbgs() << "\n";
2019 #endif
2020 report_fatal_error("Do not know how to promote this operator's operand!");
2021
2022 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
2024 Res = PromoteIntOp_ANY_EXTEND_VECTOR_INREG(N);
2025 break;
2026 case ISD::ATOMIC_STORE:
2027 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
2028 break;
2029 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
2030 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
2031 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
2032 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
2033 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
2034 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
2035 case ISD::COND_LOOP:
2036 Res = PromoteIntOp_COND_LOOP(N, OpNo);
2037 break;
2038 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
2039 case ISD::FAKE_USE:
2040 Res = PromoteIntOp_FAKE_USE(N);
2041 break;
2043 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
2044 break;
2045 case ISD::SPLAT_VECTOR:
2047 Res = PromoteIntOp_ScalarOp(N);
2048 break;
2049 case ISD::VSELECT:
2050 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
2051 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
2052 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
2053 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
2054 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
2055 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
2056 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
2057 OpNo); break;
2058 case ISD::VP_STORE:
2059 Res = PromoteIntOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2060 break;
2061 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
2062 OpNo); break;
2063 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
2064 OpNo); break;
2065 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
2066 OpNo); break;
2067 case ISD::VP_GATHER:
2068 Res = PromoteIntOp_VP_GATHER(cast<VPGatherSDNode>(N), OpNo);
2069 break;
2070 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
2071 OpNo); break;
2072 case ISD::VP_SCATTER:
2073 Res = PromoteIntOp_VP_SCATTER(cast<VPScatterSDNode>(N), OpNo);
2074 break;
2076 Res = PromoteIntOp_VECTOR_COMPRESS(N, OpNo);
2077 break;
2079 Res = PromoteIntOp_VECTOR_SHUFFLE_VAR(N, OpNo);
2080 break;
2081 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
2082 case ISD::BF16_TO_FP:
2083 case ISD::FP16_TO_FP:
2084 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
2086 Res = PromoteIntOp_CONVERT_FROM_ARBITRARY_FP(N);
2087 break;
2089 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
2090 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
2091 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
2092 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
2093
2094 case ISD::SHL:
2095 case ISD::SRA:
2096 case ISD::SRL:
2097 case ISD::ROTL:
2098 case ISD::ROTR:
2099 case ISD::SSHLSAT:
2100 case ISD::USHLSAT:
2101 Res = PromoteIntOp_Shift(N);
2102 break;
2103
2104 case ISD::SCMP:
2105 case ISD::UCMP: Res = PromoteIntOp_CMP(N); break;
2106
2107 case ISD::FSHL:
2108 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
2109
2110 case ISD::FRAMEADDR:
2111 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
2112
2113 case ISD::SMULFIX:
2114 case ISD::SMULFIXSAT:
2115 case ISD::UMULFIX:
2116 case ISD::UMULFIXSAT:
2117 case ISD::SDIVFIX:
2118 case ISD::SDIVFIXSAT:
2119 case ISD::UDIVFIX:
2120 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
2121 case ISD::FPOWI:
2122 case ISD::STRICT_FPOWI:
2123 case ISD::FLDEXP:
2124 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
2125 case ISD::VECREDUCE_ADD:
2126 case ISD::VECREDUCE_MUL:
2127 case ISD::VECREDUCE_AND:
2128 case ISD::VECREDUCE_OR:
2129 case ISD::VECREDUCE_XOR:
2133 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
2134 case ISD::VP_REDUCE_ADD:
2135 case ISD::VP_REDUCE_MUL:
2136 case ISD::VP_REDUCE_AND:
2137 case ISD::VP_REDUCE_OR:
2138 case ISD::VP_REDUCE_XOR:
2139 case ISD::VP_REDUCE_SMAX:
2140 case ISD::VP_REDUCE_SMIN:
2141 case ISD::VP_REDUCE_UMAX:
2142 case ISD::VP_REDUCE_UMIN:
2143 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
2144 break;
2145
2146 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
2147 case ISD::STACKMAP:
2148 Res = PromoteIntOp_STACKMAP(N, OpNo);
2149 break;
2150 case ISD::PATCHPOINT:
2151 Res = PromoteIntOp_PATCHPOINT(N, OpNo);
2152 break;
2154 Res = PromoteIntOp_WRITE_REGISTER(N, OpNo);
2155 break;
2156 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2157 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2158 Res = PromoteIntOp_VP_STRIDED(N, OpNo);
2159 break;
2160 case ISD::EXPERIMENTAL_VP_SPLICE:
2161 Res = PromoteIntOp_VP_SPLICE(N, OpNo);
2162 break;
2164 Res = PromoteIntOp_VECTOR_HISTOGRAM(N, OpNo);
2165 break;
2167 case ISD::CTTZ_ELTS:
2169 Res = PromoteIntOp_UnaryBooleanVectorOp(N, OpNo);
2170 break;
2172 Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N);
2173 break;
2174 case ISD::VECTOR_MATCH:
2175 Res = PromoteIntOp_VECTOR_MATCH(N, OpNo);
2176 break;
2177 case ISD::MASKED_UDIV:
2178 case ISD::MASKED_SDIV:
2179 case ISD::MASKED_UREM:
2180 case ISD::MASKED_SREM:
2181 Res = PromoteIntOp_MaskedBinOp(N, OpNo);
2182 break;
2186 Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N);
2187 break;
2188 case ISD::VECTOR_REPEAT:
2189 Res = PromoteIntOp_VECTOR_REPEAT(N);
2190 break;
2193 Res = PromoteIntOp_LOOP_DEPENDENCE_MASK(N);
2194 break;
2195 }
2196
2197 // If the result is null, the sub-method took care of registering results etc.
2198 if (!Res.getNode()) return false;
2199
2200 // If the result is N, the sub-method updated N in place. Tell the legalizer
2201 // core about this.
2202 if (Res.getNode() == N)
2203 return true;
2204
2205 const bool IsStrictFp = N->isStrictFPOpcode();
2206 assert(Res.getValueType() == N->getValueType(0) &&
2207 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2208 "Invalid operand expansion");
2209 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
2210 Res.dump());
2211
2212 ReplaceValueWith(SDValue(N, 0), Res);
2213 if (IsStrictFp)
2214 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2215
2216 return false;
2217}
2218
2219// These operands can be either sign extended or zero extended as long as we
2220// treat them the same. If an extension is free, choose that. Otherwise, follow
2221// target preference.
2222void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) {
2223 SDValue OpL = GetPromotedInteger(LHS);
2224 SDValue OpR = GetPromotedInteger(RHS);
2225
2226 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
2227 // The target would prefer to promote the comparison operand with sign
2228 // extension. Honor that unless the promoted values are already zero
2229 // extended.
2230 unsigned OpLEffectiveBits =
2231 DAG.computeKnownBits(OpL).countMaxActiveBits();
2232 unsigned OpREffectiveBits =
2233 DAG.computeKnownBits(OpR).countMaxActiveBits();
2234 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
2235 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
2236 LHS = OpL;
2237 RHS = OpR;
2238 return;
2239 }
2240
2241 // The promoted values aren't zero extended, use a sext_inreg.
2242 LHS = SExtPromotedInteger(LHS);
2243 RHS = SExtPromotedInteger(RHS);
2244 return;
2245 }
2246
2247 // Prefer to promote the comparison operand with zero extension.
2248
2249 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
2250 // than the width of LHS/RHS, we can avoid inserting a zext_inreg operation
2251 // that we might not be able to remove.
2252 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
2253 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
2254 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
2255 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
2256 LHS = OpL;
2257 RHS = OpR;
2258 return;
2259 }
2260
2261 // Otherwise, use zext_inreg.
2262 LHS = ZExtPromotedInteger(LHS);
2263 RHS = ZExtPromotedInteger(RHS);
2264}
2265
2266/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
2267/// shared among BR_CC, SELECT_CC, and SETCC handlers.
2268void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
2269 ISD::CondCode CCCode) {
2270 // We have to insert explicit sign or zero extends. Note that we could
2271 // insert sign extends for ALL conditions. For those operations where either
2272 // zero or sign extension would be valid, we ask the target which extension
2273 // it would prefer.
2274
2275 // Signed comparisons always require sign extension.
2276 if (ISD::isSignedIntSetCC(CCCode)) {
2277 LHS = SExtPromotedInteger(LHS);
2278 RHS = SExtPromotedInteger(RHS);
2279 return;
2280 }
2281
2283 "Unknown integer comparison!");
2284
2285 SExtOrZExtPromotedOperands(LHS, RHS);
2286}
2287
2288SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
2289 SDValue Op = GetPromotedInteger(N->getOperand(0));
2290 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
2291}
2292
2293SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND_VECTOR_INREG(SDNode *N) {
2294 SDValue Op = GetPromotedInteger(N->getOperand(0));
2295 EVT ResVT = N->getValueType(0);
2296 EVT OpVT = Op.getValueType();
2297 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), OpVT.getScalarType(),
2298 ResVT.getVectorNumElements());
2299 Op = DAG.getExtractSubvector(SDLoc(Op), NewVT, Op, 0);
2300 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), ResVT, Op);
2301}
2302
2303SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
2304 SDValue Op1 = GetPromotedInteger(N->getOperand(1));
2305 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
2306 N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
2307}
2308
2309SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
2310 EVT OutVT = N->getValueType(0);
2311 SDValue InOp = N->getOperand(0);
2312 EVT InVT = InOp.getValueType();
2313 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
2314 SDLoc dl(N);
2315
2316 switch (getTypeAction(InVT)) {
2318 if (InVT.isFixedLengthVector() && !OutVT.isScalableVector()) {
2319 unsigned InEltBits = InVT.getScalarSizeInBits();
2320 unsigned OutEltBits = OutVT.getScalarSizeInBits();
2321 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), OutEltBits);
2322 if (OutEltBits % InEltBits == 0 && isTypeLegal(IntVT)) {
2323 // Promotion adds padding to each input element. Pack only the original
2324 // bits into each output element instead of using a stack slot.
2325 SDValue Promoted = GetPromotedInteger(InOp);
2326 unsigned PromotedEltBits = NInVT.getScalarSizeInBits();
2327 unsigned PromotedBits = NInVT.getSizeInBits();
2328 if (PromotedEltBits % InEltBits == 0) {
2329 unsigned Scale = PromotedEltBits / InEltBits;
2330 // BITCAST preserves the total bit width, so PromotedBits equals
2331 // OutVT.getSizeInBits() * Scale and is divisible by OutEltBits.
2332 EVT PackedVT =
2333 EVT::getVectorVT(*DAG.getContext(), InVT.getScalarType(),
2334 InVT.getVectorNumElements() * Scale);
2335 EVT CastVT =
2336 EVT::getVectorVT(*DAG.getContext(), OutVT.getScalarType(),
2337 PromotedBits / OutEltBits);
2338 if (isTypeLegal(PackedVT) && isTypeLegal(CastVT) &&
2339 TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, PackedVT)) {
2340 // Preserve vector packing when the target has a legal shuffle.
2341 // For v4i8 promoted to v4i16, reinterpret as v8i8 and select
2342 // 0, 2, 4, 6 (little endian) or 1, 3, 5, 7 (big endian).
2343 SmallVector<int, 16> Mask(PackedVT.getVectorNumElements(), -1);
2344 for (unsigned I = 0; I != InVT.getVectorNumElements(); ++I)
2345 Mask[I] = I * Scale +
2346 (DAG.getDataLayout().isBigEndian() ? Scale - 1 : 0);
2347 if (TLI.isShuffleMaskLegal(Mask, PackedVT)) {
2348 SDValue Packed =
2349 DAG.getNode(ISD::BITCAST, dl, PackedVT, Promoted);
2350 Packed = DAG.getVectorShuffle(PackedVT, dl, Packed,
2351 DAG.getUNDEF(PackedVT), Mask);
2352 Packed = DAG.getNode(ISD::BITCAST, dl, CastVT, Packed);
2353 if (OutVT.isVector())
2354 return DAG.getExtractSubvector(dl, OutVT, Packed, 0);
2355 return DAG.getExtractVectorElt(dl, OutVT, Packed, 0);
2356 }
2357 }
2358 }
2359 // Pack promoted elements into each output element.
2360 unsigned PartsPerElt = OutEltBits / InEltBits;
2361 unsigned NumElts = OutVT.isVector() ? OutVT.getVectorNumElements() : 1;
2363 Elts.reserve(NumElts);
2364 for (unsigned I = 0; I != NumElts; ++I) {
2365 SDValue Packed = DAG.getConstant(0, dl, IntVT);
2366 for (unsigned J = 0; J != PartsPerElt; ++J) {
2367 SDValue Part = DAG.getExtractVectorElt(
2368 dl, Promoted.getValueType().getScalarType(), Promoted,
2369 I * PartsPerElt + J);
2370 Part = DAG.getZeroExtendInReg(Part, dl, InVT.getScalarType());
2371 Part = DAG.getZExtOrTrunc(Part, dl, IntVT);
2372 unsigned Shift = DAG.getDataLayout().isBigEndian()
2373 ? (PartsPerElt - 1 - J) * InEltBits
2374 : J * InEltBits;
2375 Part = DAG.getNode(ISD::SHL, dl, IntVT, Part,
2376 DAG.getShiftAmountConstant(Shift, IntVT, dl));
2377 Packed = DAG.getNode(ISD::OR, dl, IntVT, Packed, Part);
2378 }
2379 Elts.push_back(Packed);
2380 }
2381 SDValue Packed = Elts[0];
2382 if (OutVT.isVector()) {
2383 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), IntVT, NumElts);
2384 Packed = DAG.getBuildVector(VecVT, dl, Elts);
2385 }
2386 return DAG.getNode(ISD::BITCAST, dl, OutVT, Packed);
2387 }
2388 }
2389 // TODO: Handle big endian scalar input.
2390 if (OutVT.isVector() && !InVT.isVector() &&
2391 DAG.getDataLayout().isLittleEndian()) {
2392 EVT EltVT = OutVT.getVectorElementType();
2393 TypeSize EltSize = EltVT.getSizeInBits();
2394 TypeSize NInSize = NInVT.getSizeInBits();
2395
2396 if (NInSize.hasKnownScalarFactor(EltSize)) {
2397 unsigned NumEltsWithPadding = NInSize.getKnownScalarFactor(EltSize);
2398 EVT WideVecVT =
2399 EVT::getVectorVT(*DAG.getContext(), EltVT, NumEltsWithPadding);
2400
2401 if (isTypeLegal(WideVecVT)) {
2402 SDValue Promoted = GetPromotedInteger(InOp);
2403 SDValue Cast = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Promoted);
2404 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, Cast,
2405 DAG.getVectorIdxConstant(0, dl));
2406 }
2407 }
2408 }
2409
2410 break;
2411 }
2412 default:
2413 break;
2414 }
2415
2416 // This should only occur in unusual situations like bitcasting to an
2417 // x86_fp80, so just turn it into a store+load
2418 return CreateStackStoreLoad(InOp, OutVT);
2419}
2420
2421SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
2422 assert(OpNo == 2 && "Don't know how to promote this operand!");
2423
2424 SDValue LHS = N->getOperand(2);
2425 SDValue RHS = N->getOperand(3);
2426 PromoteSetCCOperands(LHS, RHS, N->getOperandCondCode(1));
2427
2428 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
2429 // legal types.
2430 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2431 N->getOperand(1), LHS, RHS, N->getOperand(4)),
2432 0);
2433}
2434
2435SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
2436 assert(OpNo == 1 && "only know how to promote condition");
2437
2438 // Promote all the way up to the canonical SetCC type.
2439 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
2440
2441 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
2442 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
2443 N->getOperand(2)), 0);
2444}
2445
2446SDValue DAGTypeLegalizer::PromoteIntOp_COND_LOOP(SDNode *N, unsigned OpNo) {
2447 assert(OpNo == 1 && "only know how to promote condition");
2448
2449 // Promote all the way up to the canonical SetCC type.
2450 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
2451
2452 // The chain (Op#0) is always a legal type.
2453 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond), 0);
2454}
2455
2456SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
2457 // Since the result type is legal, the operands must promote to it.
2458 EVT OVT = N->getOperand(0).getValueType();
2459 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
2460 SDValue Hi = GetPromotedInteger(N->getOperand(1));
2461 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
2462 SDLoc dl(N);
2463
2464 Hi = DAG.getNode(
2465 ISD::SHL, dl, N->getValueType(0), Hi,
2466 DAG.getShiftAmountConstant(OVT.getSizeInBits(), N->getValueType(0), dl));
2467 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
2468}
2469
2470SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
2471 // The vector type is legal but the element type is not. This implies
2472 // that the vector is a power-of-two in length and that the element
2473 // type does not have a strange size (eg: it is not i1).
2474 EVT VecVT = N->getValueType(0);
2475 unsigned NumElts = VecVT.getVectorNumElements();
2476 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2477 "Legal vector of one illegal element?");
2478
2479 // Promote the inserted value. The type does not need to match the
2480 // vector element type. Check that any extra bits introduced will be
2481 // truncated away.
2482 assert(N->getOperand(0).getValueSizeInBits() >=
2483 N->getValueType(0).getScalarSizeInBits() &&
2484 "Type of inserted value narrower than vector element type!");
2485
2487 for (unsigned i = 0; i < NumElts; ++i)
2488 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
2489
2490 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2491}
2492
2493SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
2494 unsigned OpNo) {
2495 if (OpNo == 1) {
2496 // Promote the inserted value. This is valid because the type does not
2497 // have to match the vector element type.
2498
2499 // Check that any extra bits introduced will be truncated away.
2500 assert(N->getOperand(1).getValueSizeInBits() >=
2501 N->getValueType(0).getScalarSizeInBits() &&
2502 "Type of inserted value narrower than vector element type!");
2503 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2504 GetPromotedInteger(N->getOperand(1)),
2505 N->getOperand(2)),
2506 0);
2507 }
2508
2509 assert(OpNo == 2 && "Different operand and result vector types?");
2510
2511 // Promote the index.
2512 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
2513 TLI.getVectorIdxTy(DAG.getDataLayout()));
2514 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2515 N->getOperand(1), Idx), 0);
2516}
2517
2518SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2519 SDValue Op = GetPromotedInteger(N->getOperand(0));
2520
2521 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
2522 // so just promote the operand in place.
2523 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2524}
2525
2526SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
2527 assert(OpNo == 0 && "Only know how to promote the condition!");
2528 SDValue Cond = N->getOperand(0);
2529 EVT OpTy = N->getOperand(1).getValueType();
2530
2531 if (N->getOpcode() == ISD::VSELECT)
2532 if (SDValue Res = WidenVSELECTMask(N))
2533 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2534 Res, N->getOperand(1), N->getOperand(2));
2535
2536 // Promote all the way up to the canonical SetCC type.
2537 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
2538 Cond = PromoteTargetBoolean(Cond, OpVT);
2539
2540 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
2541 N->getOperand(2)), 0);
2542}
2543
2544SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2545 assert(OpNo == 0 && "Don't know how to promote this operand!");
2546
2547 SDValue LHS = N->getOperand(0);
2548 SDValue RHS = N->getOperand(1);
2549 PromoteSetCCOperands(LHS, RHS, N->getOperandCondCode(4));
2550
2551 // The CC (#4) and the possible return values (#2 and #3) have legal types.
2552 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2553 N->getOperand(3), N->getOperand(4)), 0);
2554}
2555
2556SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
2557 assert(OpNo == 0 && "Don't know how to promote this operand!");
2558
2559 SDValue LHS = N->getOperand(0);
2560 SDValue RHS = N->getOperand(1);
2561 PromoteSetCCOperands(LHS, RHS, N->getOperandCondCode(2));
2562
2563 // The CC (#2) is always legal.
2564 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
2565}
2566
2567SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
2568 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2569 ZExtPromotedInteger(N->getOperand(1))), 0);
2570}
2571
2572SDValue DAGTypeLegalizer::PromoteIntOp_CMP(SDNode *N) {
2573 SDValue LHS = N->getOperand(0);
2574 SDValue RHS = N->getOperand(1);
2575
2576 if (N->getOpcode() == ISD::SCMP) {
2577 LHS = SExtPromotedInteger(LHS);
2578 RHS = SExtPromotedInteger(RHS);
2579 } else {
2580 SExtOrZExtPromotedOperands(LHS, RHS);
2581 }
2582
2583 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS), 0);
2584}
2585
2586SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
2587 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2588 ZExtPromotedInteger(N->getOperand(2))), 0);
2589}
2590
2591SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
2592 SDValue Op = GetPromotedInteger(N->getOperand(0));
2593 SDLoc dl(N);
2594 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2595 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
2596 Op, DAG.getValueType(N->getOperand(0).getValueType()));
2597}
2598
2599SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2600 return SDValue(DAG.UpdateNodeOperands(N,
2601 SExtPromotedInteger(N->getOperand(0))), 0);
2602}
2603
2604SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2605 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2606 SExtPromotedInteger(N->getOperand(1))), 0);
2607}
2608
2609SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2610 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2611 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2612 SDLoc dl(N);
2613
2614 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
2615
2616 // Truncate the value and store the result.
2617 return DAG.getTruncStore(Ch, dl, Val, Ptr,
2618 N->getMemoryVT(), N->getMemOperand());
2619}
2620
2621SDValue DAGTypeLegalizer::PromoteIntOp_VP_STORE(VPStoreSDNode *N,
2622 unsigned OpNo) {
2623
2624 assert(OpNo == 1 && "Unexpected operand for promotion");
2625 assert(!N->isIndexed() && "expecting unindexed vp_store!");
2626
2627 SDValue DataOp = GetPromotedInteger(N->getValue());
2628 return DAG.getTruncStoreVP(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2629 N->getMask(), N->getVectorLength(),
2630 N->getMemoryVT(), N->getMemOperand(),
2631 N->isCompressingStore());
2632}
2633
2634SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2635 unsigned OpNo) {
2636 SDValue DataOp = N->getValue();
2637 SDValue Mask = N->getMask();
2638
2639 if (OpNo == 4) {
2640 // The Mask. Update in place.
2641 EVT DataVT = DataOp.getValueType();
2642 Mask = PromoteTargetBoolean(Mask, DataVT);
2643 SmallVector<SDValue, 4> NewOps(N->ops());
2644 NewOps[4] = Mask;
2645 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2646 }
2647
2648 assert(OpNo == 1 && "Unexpected operand for promotion");
2649 DataOp = GetPromotedInteger(DataOp);
2650
2651 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2652 N->getOffset(), Mask, N->getMemoryVT(),
2653 N->getMemOperand(), N->getAddressingMode(),
2654 /*IsTruncating*/ true, N->isCompressingStore());
2655}
2656
2657SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2658 unsigned OpNo) {
2659 assert(OpNo == 3 && "Only know how to promote the mask!");
2660 EVT DataVT = N->getValueType(0);
2661 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2662 SmallVector<SDValue, 4> NewOps(N->ops());
2663 NewOps[OpNo] = Mask;
2664 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2665 if (Res == N)
2666 return SDValue(Res, 0);
2667
2668 // Update triggered CSE, do our own replacement since caller can't.
2669 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2670 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2671 return SDValue();
2672}
2673
2674SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2675 unsigned OpNo) {
2676 SmallVector<SDValue, 5> NewOps(N->ops());
2677
2678 if (OpNo == 2) {
2679 // The Mask
2680 EVT DataVT = N->getValueType(0);
2681 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2682 } else if (OpNo == 4) {
2683 // The Index
2684 if (N->isIndexSigned())
2685 // Need to sign extend the index since the bits will likely be used.
2686 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2687 else
2688 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2689 } else
2690 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2691
2692 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2693 if (Res == N)
2694 return SDValue(Res, 0);
2695
2696 // Update triggered CSE, do our own replacement since caller can't.
2697 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2698 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2699 return SDValue();
2700}
2701
2702SDValue DAGTypeLegalizer::PromoteIntOp_VP_GATHER(VPGatherSDNode *N,
2703 unsigned OpNo) {
2704 assert(OpNo == 2 && "Only the index operand can be promoted");
2705 SmallVector<SDValue, 6> NewOps(N->ops());
2706 // Extend the index since the high bits will be used for address computation.
2707 NewOps[OpNo] = N->isIndexSigned() ? SExtPromotedInteger(N->getOperand(OpNo))
2708 : ZExtPromotedInteger(N->getOperand(OpNo));
2709 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2710}
2711
2712SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2713 unsigned OpNo) {
2714 bool TruncateStore = N->isTruncatingStore();
2715 SmallVector<SDValue, 5> NewOps(N->ops());
2716
2717 if (OpNo == 2) {
2718 // The Mask
2719 EVT DataVT = N->getValue().getValueType();
2720 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2721 } else if (OpNo == 4) {
2722 // The Index
2723 if (N->isIndexSigned())
2724 // Need to sign extend the index since the bits will likely be used.
2725 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2726 else
2727 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2728 } else {
2729 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2730 TruncateStore = true;
2731 }
2732
2733 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2734 SDLoc(N), NewOps, N->getMemOperand(),
2735 N->getIndexType(), TruncateStore);
2736}
2737
2738SDValue DAGTypeLegalizer::PromoteIntOp_VP_SCATTER(VPScatterSDNode *N,
2739 unsigned OpNo) {
2740 assert(OpNo == 3 && "Only the index operand can be promoted");
2741 SmallVector<SDValue, 7> NewOps(N->ops());
2742 // Extend the index since the high bits will be used for address computation.
2743 NewOps[OpNo] = N->isIndexSigned() ? SExtPromotedInteger(N->getOperand(OpNo))
2744 : ZExtPromotedInteger(N->getOperand(OpNo));
2745 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2746}
2747
2748SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_COMPRESS(SDNode *N,
2749 unsigned OpNo) {
2750 assert(OpNo == 1 && "Can only promote VECTOR_COMPRESS mask.");
2751 SDValue Vec = N->getOperand(0);
2752 EVT VT = Vec.getValueType();
2753 SDValue Passthru = N->getOperand(2);
2754 SDValue Mask = PromoteTargetBoolean(N->getOperand(1), VT);
2755 return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), VT, Vec, Mask, Passthru);
2756}
2757
2758SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_SHUFFLE_VAR(SDNode *N,
2759 unsigned OpNo) {
2760 assert(OpNo == 1 && "The source shares the result's type, so promoting it "
2761 "is a result promotion.");
2762 // Mask elements are unsigned.
2763 SDValue Mask = ZExtPromotedInteger(N->getOperand(1));
2764 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Mask), 0);
2765}
2766
2767SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2768 SDValue Op = GetPromotedInteger(N->getOperand(0));
2769 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2770}
2771
2772SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2773 return SDValue(DAG.UpdateNodeOperands(N,
2774 ZExtPromotedInteger(N->getOperand(0))), 0);
2775}
2776
2777SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
2778 return SDValue(DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0)),
2779 N->getOperand(1)),
2780 0);
2781}
2782
2783SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2784 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2785 ZExtPromotedInteger(N->getOperand(1))), 0);
2786}
2787
2788SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2789 SDLoc dl(N);
2790 SDValue Src = N->getOperand(0);
2791 SDValue Op = GetPromotedInteger(Src);
2792 EVT VT = N->getValueType(0);
2793
2794 // If this zext has the nneg flag and the target prefers sext, see if the
2795 // promoted input is already sign extended.
2796 // TODO: Should we have some way to set nneg on ISD::AND instead?
2797 if (N->getFlags().hasNonNeg() && Op.getValueType() == VT &&
2798 TLI.isSExtCheaperThanZExt(Src.getValueType(), VT)) {
2799 unsigned OpEffectiveBits = DAG.ComputeMaxSignificantBits(Op);
2800 if (OpEffectiveBits <= Src.getScalarValueSizeInBits())
2801 return Op;
2802 }
2803
2804 Op = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op);
2805 return DAG.getZeroExtendInReg(Op, dl, Src.getValueType());
2806}
2807
2808SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2809 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2810 return SDValue(
2811 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2812}
2813
2814SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2815 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2816 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2817 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2818}
2819
2820SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
2821 bool IsStrict = N->isStrictFPOpcode();
2822 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2823
2824 bool IsPowI =
2825 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
2826 unsigned OpOffset = IsStrict ? 1 : 0;
2827
2828 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result
2829 // and floating point operand is already type legalized).
2830 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
2831 : RTLIB::getLDEXP(N->getValueType(0));
2832
2833 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
2834 if (LCImpl == RTLIB::Unsupported) {
2835 // Scalarize vector FPOWI instead of promoting the type. This allows the
2836 // scalar FPOWIs to be visited and converted to libcalls before promoting
2837 // the type.
2838 // FIXME: This should be done in LegalizeVectorOps/LegalizeDAG, but call
2839 // lowering needs the unpromoted EVT.
2840 if (IsPowI && N->getValueType(0).isVector())
2841 return DAG.UnrollVectorOp(N);
2842 SmallVector<SDValue, 3> NewOps(N->ops());
2843 NewOps[1 + OpOffset] = SExtPromotedInteger(N->getOperand(1 + OpOffset));
2844 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2845 }
2846
2847 // We can't just promote the exponent type in FPOWI, since we want to lower
2848 // the node to a libcall and we if we promote to a type larger than
2849 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2850 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2851 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2852
2853 // A wider-than-int exponent can't be passed in an int (there's no wider
2854 // libcall), so bail like the soften/expand paths. A narrower one is
2855 // sign-extended to int by the makeLibCall below.
2856 if (N->getOperand(1 + OpOffset).getScalarValueSizeInBits() >
2857 DAG.getLibInfo().getIntSize()) {
2858 const Function &Fn = DAG.getMachineFunction().getFunction();
2859 Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure(
2860 Twine(IsPowI ? "powi" : "ldexp") +
2861 " exponent does not match sizeof(int)",
2862 Fn, N->getDebugLoc()));
2863 if (IsStrict)
2864 ReplaceValueWith(SDValue(N, 1), Chain);
2865 ReplaceValueWith(SDValue(N, 0), DAG.getPOISON(N->getValueType(0)));
2866 return SDValue();
2867 }
2868
2869 TargetLowering::MakeLibCallOptions CallOptions;
2870 CallOptions.setIsSigned(true);
2871 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2872 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2873 DAG, LCImpl, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
2874 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2875 if (IsStrict)
2876 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2877 return SDValue();
2878}
2879
2881 switch (N->getOpcode()) {
2882 default:
2883 llvm_unreachable("Expected integer vector reduction");
2884 case ISD::VECREDUCE_ADD:
2885 case ISD::VECREDUCE_MUL:
2886 case ISD::VECREDUCE_AND:
2887 case ISD::VECREDUCE_OR:
2888 case ISD::VECREDUCE_XOR:
2889 case ISD::VP_REDUCE_ADD:
2890 case ISD::VP_REDUCE_MUL:
2891 case ISD::VP_REDUCE_AND:
2892 case ISD::VP_REDUCE_OR:
2893 case ISD::VP_REDUCE_XOR:
2894 return ISD::ANY_EXTEND;
2897 case ISD::VP_REDUCE_SMAX:
2898 case ISD::VP_REDUCE_SMIN:
2899 return ISD::SIGN_EXTEND;
2902 case ISD::VP_REDUCE_UMAX:
2903 case ISD::VP_REDUCE_UMIN:
2904 return ISD::ZERO_EXTEND;
2905 }
2906}
2907
2908SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2909 switch (getExtendForIntVecReduction(N)) {
2910 default:
2911 llvm_unreachable("Impossible extension kind for integer reduction");
2912 case ISD::ANY_EXTEND:
2913 return GetPromotedInteger(V);
2914 case ISD::SIGN_EXTEND:
2915 return SExtPromotedInteger(V);
2916 case ISD::ZERO_EXTEND:
2917 return ZExtPromotedInteger(V);
2918 }
2919}
2920
2921SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2922 SDLoc dl(N);
2923 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2924
2925 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
2926 EVT InVT = Op.getValueType();
2927 EVT EltVT = InVT.getVectorElementType();
2928 EVT ResVT = N->getValueType(0);
2929 unsigned Opcode = N->getOpcode();
2930
2931 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2932 // vecreduce_xor is not legal
2933 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2934 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) &&
2935 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT))
2936 Opcode = ISD::VECREDUCE_ADD;
2937
2938 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2939 // vecreduce_or is not legal
2940 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2941 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) &&
2942 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) {
2943 Opcode = ISD::VECREDUCE_UMAX;
2944 // Can't use promoteTargetBoolean here because we still need
2945 // to either sign_ext or zero_ext in the undefined case.
2946 switch (TLI.getBooleanContents(InVT)) {
2949 Op = ZExtPromotedInteger(N->getOperand(0));
2950 break;
2952 Op = SExtPromotedInteger(N->getOperand(0));
2953 break;
2954 }
2955 }
2956
2957 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2958 // vecreduce_and is not legal
2959 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2960 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) &&
2961 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) {
2962 Opcode = ISD::VECREDUCE_UMIN;
2963 // Can't use promoteTargetBoolean here because we still need
2964 // to either sign_ext or zero_ext in the undefined case.
2965 switch (TLI.getBooleanContents(InVT)) {
2968 Op = ZExtPromotedInteger(N->getOperand(0));
2969 break;
2971 Op = SExtPromotedInteger(N->getOperand(0));
2972 break;
2973 }
2974 }
2975
2976 if (ResVT.bitsGE(EltVT))
2977 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
2978
2979 // Result size must be >= element size. If this is not the case after
2980 // promotion, also promote the result type and then truncate.
2981 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
2982 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
2983}
2984
2985SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2986 SDLoc DL(N);
2987 SDValue Op = N->getOperand(OpNo);
2988 SmallVector<SDValue, 4> NewOps(N->ops());
2989
2990 if (OpNo == 2) { // Mask
2991 // Update in place.
2992 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2993 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2994 }
2995
2996 assert(OpNo == 1 && "Unexpected operand for promotion");
2997
2998 Op = PromoteIntOpVectorReduction(N, Op);
2999
3000 NewOps[OpNo] = Op;
3001
3002 EVT VT = N->getValueType(0);
3003 EVT EltVT = Op.getValueType().getScalarType();
3004
3005 if (VT.bitsGE(EltVT))
3006 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
3007
3008 // Result size must be >= element/start-value size. If this is not the case
3009 // after promotion, also promote both the start value and result type and
3010 // then truncate.
3011 NewOps[0] =
3012 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
3013 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
3014 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
3015}
3016
3017SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
3018 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
3019 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
3020}
3021
3022SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
3023 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3024 SmallVector<SDValue> NewOps(N->ops());
3025 NewOps[OpNo] = GetPromotedInteger(NewOps[OpNo]);
3026 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3027}
3028
3029SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
3030 assert(OpNo >= 7);
3031 SmallVector<SDValue> NewOps(N->ops());
3032 NewOps[OpNo] = GetPromotedInteger(NewOps[OpNo]);
3033 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3034}
3035
3036SDValue DAGTypeLegalizer::PromoteIntOp_WRITE_REGISTER(SDNode *N,
3037 unsigned OpNo) {
3038 const Function &Fn = DAG.getMachineFunction().getFunction();
3039 Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure(
3040 "cannot use llvm.write_register with illegal type", Fn,
3041 N->getDebugLoc()));
3042 return N->getOperand(0);
3043}
3044
3045SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
3046 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
3047 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
3048
3049 SmallVector<SDValue, 8> NewOps(N->ops());
3050 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
3051 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
3052 if (Res == N)
3053 return SDValue(Res, 0);
3054
3055 // Update triggered CSE, do our own replacement since caller can't.
3056 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
3057 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
3058 return SDValue();
3059}
3060
3061SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) {
3062 SmallVector<SDValue, 6> NewOps(N->ops());
3063
3064 if (OpNo == 2) { // Offset operand
3065 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
3066 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3067 }
3068
3069 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion");
3070
3071 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
3072 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3073}
3074
3075SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N,
3076 unsigned OpNo) {
3077 assert(OpNo == 1 && "Unexpected operand for promotion");
3078 SmallVector<SDValue, 7> NewOps(N->ops());
3079 NewOps[1] = GetPromotedInteger(N->getOperand(1));
3080 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3081}
3082
3083SDValue DAGTypeLegalizer::PromoteIntOp_UnaryBooleanVectorOp(SDNode *N,
3084 unsigned OpNo) {
3085 assert(OpNo == 0 && "Unexpected operand for promotion");
3086 SDValue Op = N->getOperand(0);
3087
3088 SDValue NewOp;
3089 if (TLI.getBooleanContents(Op.getValueType()) ==
3091 NewOp = SExtPromotedInteger(Op);
3092 else
3093 NewOp = ZExtPromotedInteger(Op);
3094
3095 return SDValue(DAG.UpdateNodeOperands(N, NewOp), 0);
3096}
3097
3098SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
3099 SmallVector<SDValue, 1> NewOps(N->ops());
3100 NewOps[0] = ZExtPromotedInteger(N->getOperand(0));
3101 NewOps[1] = ZExtPromotedInteger(N->getOperand(1));
3102 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3103}
3104
3105SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
3106 assert(OpNo < 3 && "Unexpected operand for promotion");
3107 if (OpNo != 2)
3108 return TLI.expandVectorMatch(N, DAG);
3109
3110 SmallVector<SDValue, 3> NewOps(N->ops());
3111 NewOps[2] = PromoteTargetBoolean(N->getOperand(2), N->getValueType(0));
3112 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3113}
3114
3115SDValue DAGTypeLegalizer::PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
3116 assert(OpNo == 2);
3117 SmallVector<SDValue, 3> NewOps(N->ops());
3118 NewOps[2] = PromoteTargetBoolean(NewOps[2], N->getValueType(0));
3119 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3120}
3121
3122SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N) {
3123 SmallVector<SDValue, 1> NewOps(N->ops());
3124 switch (N->getOpcode()) {
3126 NewOps[1] = SExtPromotedInteger(N->getOperand(1));
3127 NewOps[2] = SExtPromotedInteger(N->getOperand(2));
3128 break;
3130 NewOps[1] = ZExtPromotedInteger(N->getOperand(1));
3131 NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
3132 break;
3134 NewOps[1] = SExtPromotedInteger(N->getOperand(1));
3135 NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
3136 break;
3137 default:
3138 llvm_unreachable("unexpected opcode");
3139 }
3140 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3141}
3142
3143SDValue DAGTypeLegalizer::PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N) {
3144 SDValue NewOps[4];
3145 NewOps[0] = ZExtPromotedInteger(N->getOperand(0));
3146 NewOps[1] = ZExtPromotedInteger(N->getOperand(1));
3147 NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
3148 NewOps[3] = N->getOperand(3);
3149 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
3150}
3151
3152SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_REPEAT(SDNode *N) {
3153 SDLoc DL(N);
3154 SDValue Src = GetPromotedInteger(N->getOperand(0));
3155 EVT SrcVT = Src.getValueType();
3156 EVT OrigVT = N->getValueType(0);
3157 EVT NewVT = OrigVT.changeVectorElementType(*DAG.getContext(),
3158 SrcVT.getVectorElementType());
3159 SDValue Res = DAG.getNode(ISD::VECTOR_REPEAT, DL, NewVT, Src);
3160 return DAG.getNode(ISD::TRUNCATE, DL, OrigVT, Res);
3161}
3162
3163//===----------------------------------------------------------------------===//
3164// Integer Result Expansion
3165//===----------------------------------------------------------------------===//
3166
3167/// ExpandIntegerResult - This method is called when the specified result of the
3168/// specified node is found to need expansion. At this point, the node may also
3169/// have invalid operands or may have other results that need promotion, we just
3170/// know that (at least) one result needs expansion.
3171void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
3172 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG));
3173 SDValue Lo, Hi;
3174 Lo = Hi = SDValue();
3175
3176 // See if the target wants to custom expand this node.
3177 if (CustomLowerNode(N, N->getValueType(ResNo), true))
3178 return;
3179
3180 switch (N->getOpcode()) {
3181 default:
3182#ifndef NDEBUG
3183 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
3184 N->dump(&DAG); dbgs() << "\n";
3185#endif
3186 report_fatal_error("do not know how to expand the result of this "
3187 "operator!");
3188
3189 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
3190 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
3191 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
3192 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
3193 case ISD::POISON:
3194 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
3195 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
3196 case ISD::SETCC: ExpandIntRes_SETCC(N, Lo, Hi); break;
3197
3198 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
3199 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
3200 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
3201 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
3202 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
3203
3204 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
3205 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
3206 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
3207 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
3208 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
3209 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
3210 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
3211 case ISD::ABS:
3213 ExpandIntRes_ABS(N, Lo, Hi);
3214 break;
3215 case ISD::ABDS:
3216 case ISD::ABDU: ExpandIntRes_ABD(N, Lo, Hi); break;
3218 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
3219 case ISD::CTLS: ExpandIntRes_CTLS(N, Lo, Hi); break;
3220 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
3222 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
3223 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break;
3225 case ISD::FP_TO_SINT:
3227 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break;
3229 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
3230 case ISD::STRICT_LROUND:
3231 case ISD::STRICT_LRINT:
3232 case ISD::LROUND:
3233 case ISD::LRINT:
3235 case ISD::STRICT_LLRINT:
3236 case ISD::LLROUND:
3237 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break;
3238 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
3239 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
3241 case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break;
3242 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
3243 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
3244 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
3245 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
3246 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
3247 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
3248 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
3249 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
3250 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
3251
3263 case ISD::ATOMIC_SWAP:
3264 case ISD::ATOMIC_CMP_SWAP: {
3265 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
3266 SplitInteger(Tmp.first, Lo, Hi);
3267 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3268 break;
3269 }
3271 AtomicSDNode *AN = cast<AtomicSDNode>(N);
3272 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
3273 SDValue Tmp = DAG.getAtomicCmpSwap(
3274 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
3275 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
3276 AN->getMemOperand());
3277
3278 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
3279 // success simply by comparing the loaded value against the ingoing
3280 // comparison.
3281 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
3282 N->getOperand(2), ISD::SETEQ);
3283
3284 SplitInteger(Tmp, Lo, Hi);
3285 ReplaceValueWith(SDValue(N, 1), Success);
3286 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
3287 break;
3288 }
3289
3290 case ISD::AND:
3291 case ISD::OR:
3292 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
3293
3294 case ISD::UMAX:
3295 case ISD::SMAX:
3296 case ISD::UMIN:
3297 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
3298
3299 case ISD::SCMP:
3300 case ISD::UCMP: ExpandIntRes_CMP(N, Lo, Hi); break;
3301
3302 case ISD::ADD:
3303 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
3304
3305 case ISD::ADDC:
3306 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
3307
3308 case ISD::ADDE:
3309 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
3310
3311 case ISD::UADDO_CARRY:
3312 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
3313
3314 case ISD::SADDO_CARRY:
3315 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
3316
3317 case ISD::SHL:
3318 case ISD::SRA:
3319 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
3320
3321 case ISD::SADDO:
3322 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
3323 case ISD::UADDO:
3324 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
3325 case ISD::UMULO:
3326 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
3327
3328 case ISD::SADDSAT:
3329 case ISD::UADDSAT:
3330 case ISD::SSUBSAT:
3331 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
3332
3333 case ISD::SSHLSAT:
3334 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
3335
3336 case ISD::AVGCEILS:
3337 case ISD::AVGCEILU:
3338 case ISD::AVGFLOORS:
3339 case ISD::AVGFLOORU: ExpandIntRes_AVG(N, Lo, Hi); break;
3340
3341 case ISD::SMULFIX:
3342 case ISD::SMULFIXSAT:
3343 case ISD::UMULFIX:
3344 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
3345
3346 case ISD::SDIVFIX:
3347 case ISD::SDIVFIXSAT:
3348 case ISD::UDIVFIX:
3349 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
3350
3351 case ISD::VECREDUCE_ADD:
3352 case ISD::VECREDUCE_MUL:
3353 case ISD::VECREDUCE_AND:
3354 case ISD::VECREDUCE_OR:
3355 case ISD::VECREDUCE_XOR:
3359 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
3360
3361 case ISD::ROTL:
3362 case ISD::ROTR:
3363 ExpandIntRes_Rotate(N, Lo, Hi);
3364 break;
3365
3366 case ISD::FSHL:
3367 case ISD::FSHR:
3368 ExpandIntRes_FunnelShift(N, Lo, Hi);
3369 break;
3370
3371 case ISD::CLMUL:
3372 case ISD::CLMULR:
3373 case ISD::CLMULH:
3374 ExpandIntRes_CLMUL(N, Lo, Hi);
3375 break;
3376
3377 case ISD::PEXT:
3378 ExpandIntRes_PEXT(N, Lo, Hi);
3379 break;
3380
3381 case ISD::PDEP:
3382 ExpandIntRes_PDEP(N, Lo, Hi);
3383 break;
3384
3385 case ISD::MULHS:
3386 case ISD::MULHU:
3387 ExpandIntRes_MULH(N, Lo, Hi);
3388 break;
3389
3390 case ISD::VSCALE:
3391 ExpandIntRes_VSCALE(N, Lo, Hi);
3392 break;
3393
3394 case ISD::READ_REGISTER:
3395 ExpandIntRes_READ_REGISTER(N, Lo, Hi);
3396 break;
3397
3398 case ISD::CTTZ_ELTS:
3400 ExpandIntRes_CTTZ_ELTS(N, Lo, Hi);
3401 break;
3402 }
3403
3404 // If Lo/Hi is null, the sub-method took care of registering results etc.
3405 if (Lo.getNode())
3406 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
3407}
3408
3409/// Lower an atomic node to the appropriate builtin call.
3410std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
3411 unsigned Opc = Node->getOpcode();
3412 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3413 assert(TLI.getOperationAction(Opc, VT) == TargetLowering::LibCall &&
3414 "expanded atomic requires a libcall action for the memory type");
3415 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3416 RTLIB::AtomicLibcall LC =
3417 RTLIB::selectAtomicLibcall(DAG.getLibcalls(), Opc, order, VT);
3418 EVT RetVT = Node->getValueType(0);
3419 TargetLowering::MakeLibCallOptions CallOptions;
3421 if (LC.IsOutline) {
3422 Ops.append(Node->op_begin() + 2, Node->op_end());
3423 Ops.push_back(Node->getOperand(1));
3424 } else {
3425 Ops.append(Node->op_begin() + 1, Node->op_end());
3426 }
3427 return TLI.makeLibCall(DAG, LC.Impl, RetVT, Ops, CallOptions, SDLoc(Node),
3428 Node->getOperand(0));
3429}
3430
3431/// N is a shift by a value that needs to be expanded,
3432/// and the shift amount is a constant 'Amt'. Expand the operation.
3433void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
3434 SDValue &Lo, SDValue &Hi) {
3435 SDLoc DL(N);
3436 // Expand the incoming operand to be shifted, so that we have its parts
3437 SDValue InL, InH;
3438 GetExpandedInteger(N->getOperand(0), InL, InH);
3439
3440 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
3441 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
3442 if (!Amt) {
3443 Lo = InL;
3444 Hi = InH;
3445 return;
3446 }
3447
3448 EVT NVT = InL.getValueType();
3449 unsigned VTBits = N->getValueType(0).getSizeInBits();
3450 unsigned NVTBits = NVT.getSizeInBits();
3451
3452 if (N->getOpcode() == ISD::SHL) {
3453 if (Amt.uge(VTBits)) {
3454 Lo = Hi = DAG.getConstant(0, DL, NVT);
3455 } else if (Amt.ugt(NVTBits)) {
3456 Lo = DAG.getConstant(0, DL, NVT);
3457 Hi = DAG.getNode(ISD::SHL, DL, NVT, InL,
3458 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
3459 } else if (Amt == NVTBits) {
3460 Lo = DAG.getConstant(0, DL, NVT);
3461 Hi = InL;
3462 } else {
3463 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL,
3464 DAG.getShiftAmountConstant(Amt, NVT, DL));
3465 // Use FSHL if legal so we don't need to combine it later.
3466 if (TLI.isOperationLegal(ISD::FSHL, NVT)) {
3467 Hi = DAG.getNode(ISD::FSHL, DL, NVT, InH, InL,
3468 DAG.getShiftAmountConstant(Amt, NVT, DL));
3469 } else {
3470 Hi = DAG.getNode(
3471 ISD::OR, DL, NVT,
3472 DAG.getNode(ISD::SHL, DL, NVT, InH,
3473 DAG.getShiftAmountConstant(Amt, NVT, DL)),
3474 DAG.getNode(ISD::SRL, DL, NVT, InL,
3475 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
3476 }
3477 }
3478 return;
3479 }
3480
3481 if (N->getOpcode() == ISD::SRL) {
3482 if (Amt.uge(VTBits)) {
3483 Lo = Hi = DAG.getConstant(0, DL, NVT);
3484 } else if (Amt.ugt(NVTBits)) {
3485 Lo = DAG.getNode(ISD::SRL, DL, NVT, InH,
3486 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
3487 Hi = DAG.getConstant(0, DL, NVT);
3488 } else if (Amt == NVTBits) {
3489 Lo = InH;
3490 Hi = DAG.getConstant(0, DL, NVT);
3491 } else {
3492 // Use FSHR if legal so we don't need to combine it later.
3493 if (TLI.isOperationLegal(ISD::FSHR, NVT)) {
3494 Lo = DAG.getNode(ISD::FSHR, DL, NVT, InH, InL,
3495 DAG.getShiftAmountConstant(Amt, NVT, DL));
3496 } else {
3497 Lo = DAG.getNode(
3498 ISD::OR, DL, NVT,
3499 DAG.getNode(ISD::SRL, DL, NVT, InL,
3500 DAG.getShiftAmountConstant(Amt, NVT, DL)),
3501 DAG.getNode(ISD::SHL, DL, NVT, InH,
3502 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
3503 }
3504 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH,
3505 DAG.getShiftAmountConstant(Amt, NVT, DL));
3506 }
3507 return;
3508 }
3509
3510 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3511 if (Amt.uge(VTBits)) {
3512 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
3513 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
3514 } else if (Amt.ugt(NVTBits)) {
3515 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
3516 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
3517 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
3518 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
3519 } else if (Amt == NVTBits) {
3520 Lo = InH;
3521 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
3522 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
3523 } else {
3524 // Use FSHR if legal so we don't need to combine it later.
3525 if (TLI.isOperationLegal(ISD::FSHR, NVT)) {
3526 Lo = DAG.getNode(ISD::FSHR, DL, NVT, InH, InL,
3527 DAG.getShiftAmountConstant(Amt, NVT, DL));
3528 } else {
3529 Lo = DAG.getNode(
3530 ISD::OR, DL, NVT,
3531 DAG.getNode(ISD::SRL, DL, NVT, InL,
3532 DAG.getShiftAmountConstant(Amt, NVT, DL)),
3533 DAG.getNode(ISD::SHL, DL, NVT, InH,
3534 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
3535 }
3536 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
3537 DAG.getShiftAmountConstant(Amt, NVT, DL));
3538 }
3539}
3540
3541/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
3542/// this shift based on knowledge of the high bit of the shift amount. If we
3543/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
3544/// shift amount.
3545bool DAGTypeLegalizer::
3546ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3547 unsigned Opc = N->getOpcode();
3548 SDValue In = N->getOperand(0);
3549 SDValue Amt = N->getOperand(1);
3550 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3551 EVT ShTy = Amt.getValueType();
3552 unsigned ShBits = ShTy.getScalarSizeInBits();
3553 unsigned NVTBits = NVT.getScalarSizeInBits();
3554 assert(isPowerOf2_32(NVTBits) &&
3555 "Expanded integer type size not a power of two!");
3556 SDLoc dl(N);
3557
3558 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
3559 KnownBits Known = DAG.computeKnownBits(Amt);
3560
3561 // If we don't know anything about the high bits, exit.
3562 if (((Known.Zero | Known.One) & HighBitMask) == 0)
3563 return false;
3564
3565 // Get the incoming operand to be shifted.
3566 SDValue InL, InH;
3567 GetExpandedInteger(In, InL, InH);
3568
3569 // If we know that any of the high bits of the shift amount are one, then we
3570 // can do this as a couple of simple shifts.
3571 if (Known.One.intersects(HighBitMask)) {
3572 // Mask out the high bit, which we know is set.
3573 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
3574 DAG.getConstant(~HighBitMask, dl, ShTy));
3575
3576 switch (Opc) {
3577 default: llvm_unreachable("Unknown shift");
3578 case ISD::SHL:
3579 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
3580 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
3581 return true;
3582 case ISD::SRL:
3583 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3584 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
3585 return true;
3586 case ISD::SRA:
3587 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
3588 DAG.getConstant(NVTBits - 1, dl, ShTy));
3589 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
3590 return true;
3591 }
3592 }
3593
3594 // If we know that all of the high bits of the shift amount are zero, then we
3595 // can do this as a couple of simple shifts.
3596 if (HighBitMask.isSubsetOf(Known.Zero)) {
3597 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
3598 // shift if x is zero. We can use XOR here because x is known to be smaller
3599 // than 32.
3600 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
3601 DAG.getConstant(NVTBits - 1, dl, ShTy));
3602
3603 unsigned Op1, Op2;
3604 switch (Opc) {
3605 default: llvm_unreachable("Unknown shift");
3606 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
3607 case ISD::SRL:
3608 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
3609 }
3610
3611 // When shifting right the arithmetic for Lo and Hi is swapped.
3612 if (Opc != ISD::SHL)
3613 std::swap(InL, InH);
3614
3615 // Use a little trick to get the bits that move from Lo to Hi. First
3616 // shift by one bit.
3617 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
3618 // Then compute the remaining shift with amount-1.
3619 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
3620
3621 Lo = DAG.getNode(Opc, dl, NVT, InL, Amt);
3622 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
3623
3624 if (Opc != ISD::SHL)
3625 std::swap(Hi, Lo);
3626 return true;
3627 }
3628
3629 return false;
3630}
3631
3632/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
3633/// of any size.
3634bool DAGTypeLegalizer::
3635ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3636 SDValue Amt = N->getOperand(1);
3637 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3638 EVT ShTy = Amt.getValueType();
3639 unsigned NVTBits = NVT.getSizeInBits();
3640 assert(isPowerOf2_32(NVTBits) &&
3641 "Expanded integer type size not a power of two!");
3642 SDLoc dl(N);
3643
3644 // Get the incoming operand to be shifted.
3645 SDValue InL, InH;
3646 GetExpandedInteger(N->getOperand(0), InL, InH);
3647
3648 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
3649 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
3650 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
3651 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3652 Amt, NVBitsNode, ISD::SETULT);
3653 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3654 Amt, DAG.getConstant(0, dl, ShTy),
3655 ISD::SETEQ);
3656
3657 SDValue LoS, HiS, LoL, HiL;
3658 switch (N->getOpcode()) {
3659 default: llvm_unreachable("Unknown shift");
3660 case ISD::SHL:
3661 // Short: ShAmt < NVTBits
3662 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
3663 HiS = DAG.getNode(ISD::OR, dl, NVT,
3664 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
3665 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
3666
3667 // Long: ShAmt >= NVTBits
3668 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
3669 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
3670
3671 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
3672 Hi = DAG.getSelect(dl, NVT, isZero, InH,
3673 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
3674 return true;
3675 case ISD::SRL:
3676 // Short: ShAmt < NVTBits
3677 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
3678 LoS = DAG.getNode(ISD::OR, dl, NVT,
3679 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3680 // FIXME: If Amt is zero, the following shift generates an undefined result
3681 // on some architectures.
3682 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3683
3684 // Long: ShAmt >= NVTBits
3685 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3686 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3687
3688 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3689 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3690 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3691 return true;
3692 case ISD::SRA:
3693 // Short: ShAmt < NVTBits
3694 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
3695 LoS = DAG.getNode(ISD::OR, dl, NVT,
3696 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3697 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3698
3699 // Long: ShAmt >= NVTBits
3700 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
3701 DAG.getConstant(NVTBits - 1, dl, ShTy));
3702 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3703
3704 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3705 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3706 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3707 return true;
3708 }
3709}
3710
3711static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
3712
3713 switch (Op) {
3714 default: llvm_unreachable("invalid min/max opcode");
3715 case ISD::SMAX:
3716 return std::make_pair(ISD::SETGT, ISD::UMAX);
3717 case ISD::UMAX:
3718 return std::make_pair(ISD::SETUGT, ISD::UMAX);
3719 case ISD::SMIN:
3720 return std::make_pair(ISD::SETLT, ISD::UMIN);
3721 case ISD::UMIN:
3722 return std::make_pair(ISD::SETULT, ISD::UMIN);
3723 }
3724}
3725
3726void DAGTypeLegalizer::ExpandIntRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
3727 SDLoc DL(N);
3728
3729 SDValue LHS = N->getOperand(0);
3730 SDValue RHS = N->getOperand(1);
3731 EVT NewVT = getSetCCResultType(LHS.getValueType());
3732
3733 // Taking the same approach as ScalarizeVecRes_SETCC
3734 SDValue Res = DAG.getNode(ISD::SETCC, DL, NewVT, LHS, RHS, N->getOperand(2));
3735
3736 Res = DAG.getBoolExtOrTrunc(Res, DL, N->getValueType(0), NewVT);
3737 SplitInteger(Res, Lo, Hi);
3738}
3739
3740void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
3741 SDValue &Lo, SDValue &Hi) {
3742 SDLoc DL(N);
3743
3744 SDValue LHS = N->getOperand(0);
3745 SDValue RHS = N->getOperand(1);
3746
3747 // If the upper halves are all sign bits, then we can perform the MINMAX on
3748 // the lower half and sign-extend the result to the upper half.
3749 unsigned NumBits = N->getValueType(0).getScalarSizeInBits();
3750 unsigned NumHalfBits = NumBits / 2;
3751 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits &&
3752 DAG.ComputeNumSignBits(RHS) > NumHalfBits) {
3753 SDValue LHSL, LHSH, RHSL, RHSH;
3754 GetExpandedInteger(LHS, LHSL, LHSH);
3755 GetExpandedInteger(RHS, RHSL, RHSH);
3756 EVT NVT = LHSL.getValueType();
3757
3758 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL);
3759 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo,
3760 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL));
3761 return;
3762 }
3763
3764 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1.
3765 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL.
3766 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) ||
3767 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) {
3768 SDValue LHSL, LHSH, RHSL, RHSH;
3769 GetExpandedInteger(LHS, LHSL, LHSH);
3770 GetExpandedInteger(RHS, RHSL, RHSH);
3771 EVT NVT = LHSL.getValueType();
3772 EVT CCT = getSetCCResultType(NVT);
3773
3774 SDValue HiNeg =
3775 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
3776 if (N->getOpcode() == ISD::SMIN) {
3777 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getAllOnesConstant(DL, NVT));
3778 } else {
3779 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
3780 }
3781 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3782 return;
3783 }
3784
3785 const APInt *RHSVal = nullptr;
3786 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3787 RHSVal = &RHSConst->getAPIntValue();
3788
3789 // The high half of MIN/MAX is always just the the MIN/MAX of the
3790 // high halves of the operands. Expand this way if it appears profitable.
3791 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) &&
3792 (RHSVal->countLeadingOnes() >= NumHalfBits ||
3793 RHSVal->countLeadingZeros() >= NumHalfBits)) {
3794 SDValue LHSL, LHSH, RHSL, RHSH;
3795 GetExpandedInteger(LHS, LHSL, LHSH);
3796 GetExpandedInteger(RHS, RHSL, RHSH);
3797 EVT NVT = LHSL.getValueType();
3798 EVT CCT = getSetCCResultType(NVT);
3799
3800 ISD::NodeType LoOpc;
3801 ISD::CondCode CondC;
3802 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
3803
3804 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3805 // We need to know whether to select Lo part that corresponds to 'winning'
3806 // Hi part or if Hi parts are equal.
3807 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
3808 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
3809
3810 // Lo part corresponding to the 'winning' Hi part
3811 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
3812
3813 // Recursed Lo part if Hi parts are equal, this uses unsigned version
3814 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
3815
3816 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
3817 return;
3818 }
3819
3820 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies
3821 // the compare.
3823 switch (N->getOpcode()) {
3824 default: llvm_unreachable("How did we get here?");
3825 case ISD::SMAX:
3826 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3827 Pred = ISD::SETGE;
3828 else
3829 Pred = ISD::SETGT;
3830 break;
3831 case ISD::SMIN:
3832 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3833 Pred = ISD::SETLE;
3834 else
3835 Pred = ISD::SETLT;
3836 break;
3837 case ISD::UMAX:
3838 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3839 Pred = ISD::SETUGE;
3840 else
3841 Pred = ISD::SETUGT;
3842 break;
3843 case ISD::UMIN:
3844 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3845 Pred = ISD::SETULE;
3846 else
3847 Pred = ISD::SETULT;
3848 break;
3849 }
3850 EVT VT = N->getValueType(0);
3851 EVT CCT = getSetCCResultType(VT);
3852 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred);
3853 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS);
3854 SplitInteger(Result, Lo, Hi);
3855}
3856
3857void DAGTypeLegalizer::ExpandIntRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
3858 SDValue ExpandedCMP = TLI.expandCMP(N, DAG);
3859 SplitInteger(ExpandedCMP, Lo, Hi);
3860}
3861
3862void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
3863 SDValue &Lo, SDValue &Hi) {
3864 SDLoc dl(N);
3865 // Expand the subcomponents.
3866 SDValue LHSL, LHSH, RHSL, RHSH;
3867 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3868 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3869
3870 EVT NVT = LHSL.getValueType();
3871 SDValue LoOps[2] = { LHSL, RHSL };
3872 SDValue HiOps[3] = { LHSH, RHSH };
3873
3874 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3875 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
3876 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3877 if (HasOpCarry) {
3878 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3879 if (N->getOpcode() == ISD::ADD) {
3880 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3881 HiOps[2] = Lo.getValue(1);
3882 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3883 ? DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2))
3884 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps);
3885 } else {
3886 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3887 HiOps[2] = Lo.getValue(1);
3888 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3889 ? DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2))
3890 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps);
3891 }
3892 return;
3893 }
3894
3895 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
3896 // them. TODO: Teach operation legalization how to expand unsupported
3897 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
3898 // a carry of type MVT::Glue, but there doesn't seem to be any way to
3899 // generate a value of this type in the expanded code sequence.
3900 bool hasCarry =
3901 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3903 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3904
3905 if (hasCarry) {
3906 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
3907 if (N->getOpcode() == ISD::ADD) {
3908 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3909 HiOps[2] = Lo.getValue(1);
3910 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3911 } else {
3912 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3913 HiOps[2] = Lo.getValue(1);
3914 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3915 }
3916 return;
3917 }
3918
3919 bool hasOVF =
3920 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3922 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3923 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
3924
3925 if (hasOVF) {
3926 EVT OvfVT = getSetCCResultType(NVT);
3927 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
3928 int RevOpc;
3929 if (N->getOpcode() == ISD::ADD) {
3930 RevOpc = ISD::SUB;
3931 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3932 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3933 } else {
3934 RevOpc = ISD::ADD;
3935 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3936 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3937 }
3938 SDValue OVF = Lo.getValue(1);
3939
3940 switch (BoolType) {
3942 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
3943 [[fallthrough]];
3945 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
3946 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
3947 break;
3949 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
3950 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
3951 }
3952 return;
3953 }
3954
3955 if (N->getOpcode() == ISD::ADD) {
3956 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
3957 SDValue Cmp;
3958 // Special case: X+1 has a carry out if X+1==0. This may reduce the live
3959 // range of X. We assume comparing with 0 is cheap.
3960 if (isOneConstant(LoOps[1]))
3961 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3962 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3963 else if (isAllOnesConstant(LoOps[1])) {
3964 if (isAllOnesConstant(HiOps[1]))
3965 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3966 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3967 else
3968 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3969 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3970 } else
3971 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
3972 ISD::SETULT);
3973
3974 SDValue Carry;
3976 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3977 else
3978 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3979 DAG.getConstant(0, dl, NVT));
3980
3981 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1])) {
3982 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry);
3983 } else {
3984 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3985 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
3986 }
3987 } else {
3988 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
3989 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3990 SDValue Cmp =
3991 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3992 LoOps[0], LoOps[1], ISD::SETULT);
3993
3994 SDValue Borrow;
3996 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3997 else
3998 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3999 DAG.getConstant(0, dl, NVT));
4000
4001 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
4002 }
4003}
4004
4005void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
4006 SDValue &Lo, SDValue &Hi) {
4007 // Expand the subcomponents.
4008 SDValue LHSL, LHSH, RHSL, RHSH;
4009 SDLoc dl(N);
4010 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4011 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
4012 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
4013 SDValue LoOps[2] = { LHSL, RHSL };
4014 SDValue HiOps[3] = { LHSH, RHSH };
4015
4016 if (N->getOpcode() == ISD::ADDC) {
4017 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
4018 HiOps[2] = Lo.getValue(1);
4019 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
4020 } else {
4021 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
4022 HiOps[2] = Lo.getValue(1);
4023 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
4024 }
4025
4026 // Legalized the flag result - switch anything that used the old flag to
4027 // use the new one.
4028 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
4029}
4030
4031void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
4032 SDValue &Lo, SDValue &Hi) {
4033 // Expand the subcomponents.
4034 SDValue LHSL, LHSH, RHSL, RHSH;
4035 SDLoc dl(N);
4036 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4037 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
4038 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
4039 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
4040 SDValue HiOps[3] = { LHSH, RHSH };
4041
4042 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
4043 HiOps[2] = Lo.getValue(1);
4044 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
4045
4046 // Legalized the flag result - switch anything that used the old flag to
4047 // use the new one.
4048 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
4049}
4050
4051void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
4052 SDValue &Lo, SDValue &Hi) {
4053 SDValue LHS = N->getOperand(0);
4054 SDValue RHS = N->getOperand(1);
4055 SDLoc dl(N);
4056
4057 SDValue Ovf;
4058
4059 unsigned CarryOp, NoCarryOp;
4061 switch(N->getOpcode()) {
4062 case ISD::UADDO:
4063 CarryOp = ISD::UADDO_CARRY;
4064 NoCarryOp = ISD::ADD;
4065 Cond = ISD::SETULT;
4066 break;
4067 case ISD::USUBO:
4068 CarryOp = ISD::USUBO_CARRY;
4069 NoCarryOp = ISD::SUB;
4070 Cond = ISD::SETUGT;
4071 break;
4072 default:
4073 llvm_unreachable("Node has unexpected Opcode");
4074 }
4075
4076 bool HasCarryOp = TLI.isOperationLegalOrCustom(
4077 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
4078
4079 if (HasCarryOp) {
4080 // Expand the subcomponents.
4081 SDValue LHSL, LHSH, RHSL, RHSH;
4082 GetExpandedInteger(LHS, LHSL, LHSH);
4083 GetExpandedInteger(RHS, RHSL, RHSH);
4084 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
4085 SDValue LoOps[2] = { LHSL, RHSL };
4086 SDValue HiOps[3] = { LHSH, RHSH };
4087
4088 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
4089 HiOps[2] = Lo.getValue(1);
4090 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
4091
4092 Ovf = Hi.getValue(1);
4093 } else {
4094 // Expand the result by simply replacing it with the equivalent
4095 // non-overflow-checking operation.
4096 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
4097 SplitInteger(Sum, Lo, Hi);
4098
4099 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) {
4100 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this
4101 // with (Lo | Hi) == 0.
4102 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi);
4103 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or,
4104 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ);
4105 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) {
4106 // Special case: uaddo X, -1 overflows if X == 0.
4107 Ovf =
4108 DAG.getSetCC(dl, N->getValueType(1), LHS,
4109 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE);
4110 } else {
4111 // Calculate the overflow: addition overflows iff a + b < a, and
4112 // subtraction overflows iff a - b > a.
4113 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
4114 }
4115 }
4116
4117 // Legalized the flag result - switch anything that used the old flag to
4118 // use the new one.
4119 ReplaceValueWith(SDValue(N, 1), Ovf);
4120}
4121
4122void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
4123 SDValue &Hi) {
4124 // Expand the subcomponents.
4125 SDValue LHSL, LHSH, RHSL, RHSH;
4126 SDLoc dl(N);
4127 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4128 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
4129 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
4130 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
4131 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
4132
4133 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
4134 HiOps[2] = Lo.getValue(1);
4135 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
4136
4137 // Legalized the flag result - switch anything that used the old flag to
4138 // use the new one.
4139 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
4140}
4141
4142void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
4143 SDValue &Lo, SDValue &Hi) {
4144 // Expand the subcomponents.
4145 SDValue LHSL, LHSH, RHSL, RHSH;
4146 SDLoc dl(N);
4147 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4148 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
4149 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
4150
4151 // We need to use an unsigned carry op for the lo part.
4152 unsigned CarryOp =
4154 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
4155 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
4156
4157 // Legalized the flag result - switch anything that used the old flag to
4158 // use the new one.
4159 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
4160}
4161
4162void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
4163 SDValue &Lo, SDValue &Hi) {
4164 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4165 SDLoc dl(N);
4166 SDValue Op = N->getOperand(0);
4167 if (Op.getValueType().bitsLE(NVT)) {
4168 // The low part is any extension of the input (which degenerates to a copy).
4169 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
4170 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
4171 } else {
4172 // For example, extension of an i48 to an i64. The operand type necessarily
4173 // promotes to the result type, so will end up being expanded too.
4174 assert(getTypeAction(Op.getValueType()) ==
4176 "Only know how to promote this result!");
4177 SDValue Res = GetPromotedInteger(Op);
4178 assert(Res.getValueType() == N->getValueType(0) &&
4179 "Operand over promoted?");
4180 // Split the promoted operand. This will simplify when it is expanded.
4181 SplitInteger(Res, Lo, Hi);
4182 }
4183}
4184
4185void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
4186 SDValue &Lo, SDValue &Hi) {
4187 SDLoc dl(N);
4188 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4189 EVT NVT = Lo.getValueType();
4190 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4191 unsigned NVTBits = NVT.getSizeInBits();
4192 unsigned EVTBits = EVT.getSizeInBits();
4193
4194 if (NVTBits < EVTBits) {
4195 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
4196 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4197 EVTBits - NVTBits)));
4198 } else {
4199 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
4200 // The high part replicates the sign bit of Lo, make it explicit.
4201 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
4202 DAG.getShiftAmountConstant(NVTBits - 1, NVT, dl));
4203 }
4204}
4205
4206void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
4207 SDValue &Lo, SDValue &Hi) {
4208 SDLoc dl(N);
4209 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4210 EVT NVT = Lo.getValueType();
4211 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4212 unsigned NVTBits = NVT.getSizeInBits();
4213 unsigned EVTBits = EVT.getSizeInBits();
4214
4215 if (NVTBits < EVTBits) {
4216 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
4217 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4218 EVTBits - NVTBits)));
4219 } else {
4220 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
4221 // The high part must be zero, make it explicit.
4222 Hi = DAG.getConstant(0, dl, NVT);
4223 }
4224}
4225
4226void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
4227 SDValue &Lo, SDValue &Hi) {
4228 SDLoc dl(N);
4229 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
4230 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
4231 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
4232}
4233
4234void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
4235 SDValue &Lo, SDValue &Hi) {
4236 SDLoc dl(N);
4237 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
4238 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
4239 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
4240}
4241
4242void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
4243 SDValue &Hi) {
4244 SDLoc dl(N);
4245 // parity(HiLo) -> parity(Lo^Hi)
4246 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4247 EVT NVT = Lo.getValueType();
4248 Lo =
4249 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
4250 Hi = DAG.getConstant(0, dl, NVT);
4251}
4252
4253void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
4254 SDValue &Lo, SDValue &Hi) {
4255 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4256 unsigned NBitWidth = NVT.getSizeInBits();
4258 const APInt &Cst = Constant->getAPIntValue();
4259 bool IsTarget = Constant->isTargetOpcode();
4260 bool IsOpaque = Constant->isOpaque();
4261 SDLoc dl(N);
4262 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
4263 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
4264 IsOpaque);
4265}
4266
4267void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
4268 SDLoc dl(N);
4269
4270 SDValue N0 = N->getOperand(0);
4271 GetExpandedInteger(N0, Lo, Hi);
4272 EVT NVT = Lo.getValueType();
4273
4274 // If the upper half is all sign bits, then we can perform the ABS on the
4275 // lower half and zero-extend. We could use ISD::ABS_MIN_POISON here if
4276 // DAG.ComputeNumSignBits(N0) is larger than NVT.getScalarSizeInBits() + 1.
4277 unsigned NumSignBits = DAG.ComputeNumSignBits(N0);
4278 if (NumSignBits > NVT.getScalarSizeInBits()) {
4279 unsigned AbsOpc = NumSignBits > NVT.getScalarSizeInBits() + 1
4281 : ISD::ABS;
4282 Lo = DAG.getNode(AbsOpc, dl, NVT, Lo);
4283 Hi = DAG.getConstant(0, dl, NVT);
4284 return;
4285 }
4286
4287 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
4288 // we use in LegalizeDAG. The SUB part of the expansion is based on
4289 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
4290 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
4291 // expanded if needed. Shift expansion has a special case for filling with
4292 // sign bits so that we will only end up with one SRA.
4293 bool HasSubCarry = TLI.isOperationLegalOrCustom(
4294 ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
4295 if (HasSubCarry) {
4296 SDValue Sign = DAG.getNode(
4297 ISD::SRA, dl, NVT, Hi,
4298 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl));
4299 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
4300 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
4301 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
4302 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
4303 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
4304 return;
4305 }
4306
4307 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
4308 EVT VT = N->getValueType(0);
4309 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
4310 DAG.getConstant(0, dl, VT), N0);
4311 SDValue NegLo, NegHi;
4312 SplitInteger(Neg, NegLo, NegHi);
4313
4314 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
4315 DAG.getConstant(0, dl, NVT), ISD::SETLT);
4316 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
4317 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
4318}
4319
4320void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
4321 SDValue &Lo, SDValue &Hi) {
4322 SDLoc dl(N);
4323 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
4324 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4325 EVT NVT = Lo.getValueType();
4326
4327 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
4328 DAG.getConstant(0, dl, NVT), ISD::SETNE);
4329
4330 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
4331 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, NVT, Hi);
4332
4333 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
4334 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
4335 DAG.getConstant(NVT.getSizeInBits(), dl,
4336 NVT)));
4337 Hi = DAG.getConstant(0, dl, NVT);
4338}
4339
4340void DAGTypeLegalizer::ExpandIntRes_CTLS(SDNode *N, SDValue &Lo, SDValue &Hi) {
4341 SDLoc dl(N);
4342 // ctls(HiLo) -> if (IsAllSignBits = (ctls(Hi) == BW-1)) then
4343 // BW-1 + clz(IsNegative = (Hi < 0) ? ~Lo : Lo)
4344 // else ctls(Hi)
4345 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4346 EVT NVT = Lo.getValueType();
4347 unsigned NVTBits = NVT.getScalarSizeInBits();
4348
4349 SDValue Constant0 = DAG.getConstant(0, dl, NVT);
4350 SDValue ConstantBWM1 = DAG.getConstant(NVTBits - 1, dl, NVT);
4351
4352 SDValue HiCTLS = DAG.getNode(ISD::CTLS, dl, NVT, Hi);
4353 SDValue IsAllSignBits = DAG.getSetCC(dl, getSetCCResultType(NVT), HiCTLS,
4354 ConstantBWM1, ISD::SETEQ);
4355 SDValue IsNegative =
4356 DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, Constant0, ISD::SETLT);
4357 SDValue AdjustedLo =
4358 DAG.getSelect(dl, NVT, IsNegative, DAG.getNOT(dl, Lo, NVT), Lo);
4359 SDValue LoCLZ = DAG.getNode(ISD::CTLZ, dl, NVT, AdjustedLo);
4360 Lo = DAG.getSelect(dl, NVT, IsAllSignBits,
4361 DAG.getNode(ISD::ADD, dl, NVT, LoCLZ, ConstantBWM1),
4362 HiCTLS);
4363 Hi = DAG.getConstant(0, dl, NVT);
4364}
4365
4366void DAGTypeLegalizer::ExpandIntRes_ABD(SDNode *N, SDValue &Lo, SDValue &Hi) {
4367 SDValue Result = TLI.expandABD(N, DAG);
4368 SplitInteger(Result, Lo, Hi);
4369}
4370
4371void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, SDValue &Lo, SDValue &Hi) {
4372 SDValue Op = N->getOperand(0);
4373 EVT VT = N->getValueType(0);
4374 SDLoc DL(N);
4375
4376 if (TLI.getOperationAction(ISD::CTPOP, VT) == TargetLoweringBase::LibCall) {
4377 RTLIB::Libcall LC = RTLIB::getCTPOP(VT);
4378 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4379 "LibCall explicitly requested, but not available");
4380
4381 if (RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC)) {
4382 TargetLowering::MakeLibCallOptions CallOptions;
4383 EVT IntVT =
4384 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
4385 SDValue Res =
4386 TLI.makeLibCall(DAG, LCImpl, IntVT, Op, CallOptions, DL).first;
4387 SplitInteger(DAG.getSExtOrTrunc(Res, DL, VT), Lo, Hi);
4388 return;
4389 }
4390
4391 // If the function is not available, fall back on the expansion.
4392 }
4393
4394 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
4395 GetExpandedInteger(Op, Lo, Hi);
4396 EVT NVT = Lo.getValueType();
4397 Lo = DAG.getNode(ISD::ADD, DL, NVT, DAG.getNode(ISD::CTPOP, DL, NVT, Lo),
4398 DAG.getNode(ISD::CTPOP, DL, NVT, Hi));
4399 Hi = DAG.getConstant(0, DL, NVT);
4400}
4401
4402void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
4403 SDValue &Lo, SDValue &Hi) {
4404 SDLoc dl(N);
4405 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
4406 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4407 EVT NVT = Lo.getValueType();
4408
4409 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
4410 DAG.getConstant(0, dl, NVT), ISD::SETNE);
4411
4412 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_POISON, dl, NVT, Lo);
4413 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
4414
4415 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
4416 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
4417 DAG.getConstant(NVT.getSizeInBits(), dl,
4418 NVT)));
4419 Hi = DAG.getConstant(0, dl, NVT);
4420}
4421
4422void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo,
4423 SDValue &Hi) {
4424 SDLoc dl(N);
4425 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4426 unsigned NBitWidth = NVT.getSizeInBits();
4427
4428 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0));
4429 SDValue Chain = Lo.getValue(1);
4430 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING
4431 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
4432 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
4433
4434 // Legalize the chain result - switch anything that used the old chain to
4435 // use the new one.
4436 ReplaceValueWith(SDValue(N, 1), Chain);
4437}
4438
4439// Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op.
4440static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT,
4441 SDLoc DL, SelectionDAG &DAG) {
4442 if (IsStrict) {
4443 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op});
4444 Chain = Op.getValue(1);
4445 return Op;
4446 }
4447 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Op);
4448}
4449
4450void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
4451 SDValue &Hi) {
4452 SDLoc dl(N);
4453 EVT VT = N->getValueType(0);
4454
4455 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
4456 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
4457 bool IsStrict = N->isStrictFPOpcode();
4458 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4459 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4460
4461 // If every finite input fits in the half-width type, convert to that type and
4462 // extend. A soft-promoted input qualifies when f32 is legal, since it will be
4463 // extended to f32 for the narrower conversion.
4464 EVT SrcVT = Op.getValueType();
4465 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4466 // Unsigned conversions may also use FP_TO_SINT, so require the signed width.
4467 unsigned MinIntBits = APFloat::semanticsIntSizeInBits(SrcVT.getFltSemantics(),
4468 /*isSigned=*/true);
4469 bool SoftPromoted =
4470 getTypeAction(SrcVT) == TargetLowering::TypeSoftPromoteHalf;
4471 if (MinIntBits <= NVT.getSizeInBits() &&
4472 (TLI.isTypeLegal(SrcVT) || (SoftPromoted && TLI.isTypeLegal(MVT::f32)))) {
4473 unsigned NewOpc = TLI.getPreferredFPToIntOpcode(N->getOpcode(), SrcVT, NVT);
4474 if (IsStrict) {
4475 Lo = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, {Chain, Op});
4476 ReplaceValueWith(SDValue(N, 1), Lo.getValue(1));
4477 } else {
4478 Lo = DAG.getNode(NewOpc, dl, NVT, Op);
4479 }
4480 Hi = IsSigned ? DAG.getNode(ISD::SRA, dl, NVT, Lo,
4481 DAG.getShiftAmountConstant(
4482 NVT.getSizeInBits() - 1, NVT, dl))
4483 : DAG.getConstant(0, dl, NVT);
4484 return;
4485 }
4486
4487 // If the input is bf16 or needs to be soft promoted, extend to f32.
4488 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf ||
4489 Op.getValueType() == MVT::bf16) {
4490 Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG);
4491 }
4492
4493 // NOTE: We need a variable that lives across makeLibCall so
4494 // CallOptions.setTypeListBeforeSoften can save a reference to it.
4495 EVT OpVT = Op.getValueType();
4496
4497 RTLIB::Libcall LC =
4498 IsSigned ? RTLIB::getFPTOSINT(OpVT, VT) : RTLIB::getFPTOUINT(OpVT, VT);
4499 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
4500 TargetLowering::MakeLibCallOptions CallOptions;
4501 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftenFloat)
4502 CallOptions.setTypeListBeforeSoften(OpVT, VT);
4503 else
4504 CallOptions.setIsSigned(true); // FIXME: Is this needed?
4505 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
4506 CallOptions, dl, Chain);
4507 SplitInteger(Tmp.first, Lo, Hi);
4508
4509 if (IsStrict)
4510 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4511}
4512
4513void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
4514 SDValue &Hi) {
4515 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
4516 SplitInteger(Res, Lo, Hi);
4517}
4518
4519void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
4520 SDValue &Hi) {
4521 SDLoc dl(N);
4522 bool IsStrict = N->isStrictFPOpcode();
4523 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4524 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4525
4526 EVT VT = Op.getValueType();
4527
4528 if (VT == MVT::f16) {
4529 // Extend to f32.
4530 VT = MVT::f32;
4531 Op = fpExtendHelper(Op, Chain, IsStrict, VT, dl, DAG);
4532 }
4533
4534 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4535 if (N->getOpcode() == ISD::LROUND ||
4536 N->getOpcode() == ISD::STRICT_LROUND) {
4537 LC = RTLIB::getLROUND(VT);
4538 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!");
4539 } else if (N->getOpcode() == ISD::LRINT ||
4540 N->getOpcode() == ISD::STRICT_LRINT) {
4541 LC = RTLIB::getLRINT(VT);
4542 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!");
4543 } else if (N->getOpcode() == ISD::LLROUND ||
4544 N->getOpcode() == ISD::STRICT_LLROUND) {
4545 LC = RTLIB::getLLROUND(VT);
4546 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
4547 } else if (N->getOpcode() == ISD::LLRINT ||
4548 N->getOpcode() == ISD::STRICT_LLRINT) {
4549 LC = RTLIB::getLLRINT(VT);
4550 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
4551 } else
4552 llvm_unreachable("Unexpected opcode!");
4553
4554 EVT RetVT = N->getValueType(0);
4555
4556 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
4557 if (LCImpl == RTLIB::Unsupported) {
4558 DAG.getContext()->emitError(Twine("no libcall available for ") +
4559 N->getOperationName(&DAG));
4560 SDValue Poison = DAG.getPOISON(N->getValueType(0));
4561 SplitInteger(Poison, Lo, Hi);
4562 if (N->isStrictFPOpcode())
4563 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
4564 return;
4565 }
4566
4567 TargetLowering::MakeLibCallOptions CallOptions;
4568 CallOptions.setIsSigned(true);
4569 std::pair<SDValue, SDValue> Tmp =
4570 TLI.makeLibCall(DAG, LCImpl, RetVT, Op, CallOptions, dl, Chain);
4571 SplitInteger(Tmp.first, Lo, Hi);
4572
4573 if (N->isStrictFPOpcode())
4574 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4575}
4576
4577void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
4578 SDValue &Lo, SDValue &Hi) {
4579 assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?");
4580
4581 if (ISD::isNormalLoad(N)) {
4582 ExpandRes_NormalLoad(N, Lo, Hi);
4583 return;
4584 }
4585
4586 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
4587
4588 EVT VT = N->getValueType(0);
4589 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4590 SDValue Ch = N->getChain();
4591 SDValue Ptr = N->getBasePtr();
4592 ISD::LoadExtType ExtType = N->getExtensionType();
4593 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4594 MMOMetadata Metadata = N->getMMOMetadataForSubAccess();
4595 SDLoc dl(N);
4596
4597 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4598
4599 if (N->getMemoryVT().bitsLE(NVT)) {
4600 EVT MemVT = N->getMemoryVT();
4601
4602 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
4603 N->getBaseAlign(), MMOFlags, Metadata);
4604
4605 // Remember the chain.
4606 Ch = Lo.getValue(1);
4607
4608 if (ExtType == ISD::SEXTLOAD) {
4609 // The high part is obtained by SRA'ing all but one of the bits of the
4610 // lo part.
4611 unsigned LoSize = Lo.getValueSizeInBits();
4612 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
4613 DAG.getShiftAmountConstant(LoSize - 1, NVT, dl));
4614 } else if (ExtType == ISD::ZEXTLOAD) {
4615 // The high part is just a zero.
4616 Hi = DAG.getConstant(0, dl, NVT);
4617 } else {
4618 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
4619 // The high part is undefined.
4620 Hi = DAG.getUNDEF(NVT);
4621 }
4622 } else if (DAG.getDataLayout().isLittleEndian()) {
4623 // Little-endian - low bits are at low addresses.
4624 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), N->getBaseAlign(),
4625 MMOFlags, Metadata);
4626
4627 unsigned ExcessBits =
4628 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4629 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4630
4631 // Increment the pointer to the other half.
4632 unsigned IncrementSize = NVT.getSizeInBits()/8;
4633 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
4634 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
4635 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4636 N->getBaseAlign(), MMOFlags, Metadata);
4637
4638 // Build a factor node to remember that this load is independent of the
4639 // other one.
4640 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4641 Hi.getValue(1));
4642 } else {
4643 // Big-endian - high bits are at low addresses. Favor aligned loads at
4644 // the cost of some bit-fiddling.
4645 EVT MemVT = N->getMemoryVT();
4646 unsigned EBytes = MemVT.getStoreSize();
4647 unsigned IncrementSize = NVT.getSizeInBits()/8;
4648 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4649
4650 // Load both the high bits and maybe some of the low bits.
4651 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
4652 EVT::getIntegerVT(*DAG.getContext(),
4653 MemVT.getSizeInBits() - ExcessBits),
4654 N->getBaseAlign(), MMOFlags, Metadata);
4655
4656 // Increment the pointer to the other half.
4657 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
4658 // Load the rest of the low bits.
4659 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
4660 N->getPointerInfo().getWithOffset(IncrementSize),
4661 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4662 N->getBaseAlign(), MMOFlags, Metadata);
4663
4664 // Build a factor node to remember that this load is independent of the
4665 // other one.
4666 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4667 Hi.getValue(1));
4668
4669 if (ExcessBits < NVT.getSizeInBits()) {
4670 // Transfer low bits from the bottom of Hi to the top of Lo.
4671 Lo = DAG.getNode(
4672 ISD::OR, dl, NVT, Lo,
4673 DAG.getNode(ISD::SHL, dl, NVT, Hi,
4674 DAG.getShiftAmountConstant(ExcessBits, NVT, dl)));
4675 // Move high bits to the right position in Hi.
4676 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
4677 Hi,
4678 DAG.getShiftAmountConstant(
4679 NVT.getSizeInBits() - ExcessBits, NVT, dl));
4680 }
4681 }
4682
4683 // Legalize the chain result - switch anything that used the old chain to
4684 // use the new one.
4685 ReplaceValueWith(SDValue(N, 1), Ch);
4686}
4687
4688void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
4689 SDValue &Lo, SDValue &Hi) {
4690 SDLoc dl(N);
4691 SDValue LL, LH, RL, RH;
4692 GetExpandedInteger(N->getOperand(0), LL, LH);
4693 GetExpandedInteger(N->getOperand(1), RL, RH);
4694
4695 SDNodeFlags Flags;
4696 if (N->getOpcode() == ISD::OR)
4697 Flags.setDisjoint(N->getFlags().hasDisjoint());
4698
4699 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL, Flags);
4700 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH, Flags);
4701}
4702
4703void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
4704 SDValue &Lo, SDValue &Hi) {
4705 EVT VT = N->getValueType(0);
4706 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4707 SDLoc dl(N);
4708
4709 SDValue LL, LH, RL, RH;
4710 GetExpandedInteger(N->getOperand(0), LL, LH);
4711 GetExpandedInteger(N->getOperand(1), RL, RH);
4712
4713 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
4715 LL, LH, RL, RH))
4716 return;
4717
4718 // If nothing else, we can make a libcall.
4719 RTLIB::Libcall LC = RTLIB::getMUL(VT);
4720 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
4721 if (LCImpl == RTLIB::Unsupported) {
4722 // Perform a wide multiplication where the wide type is the original VT and
4723 // the 4 parts are the split arguments.
4724 TLI.forceExpandMultiply(DAG, dl, /*Signed=*/false, Lo, Hi, LL, RL, LH, RH);
4725 return;
4726 }
4727
4728 // Note that we don't need to do a wide MUL here since we don't care about the
4729 // upper half of the result if it exceeds VT.
4730 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4731 TargetLowering::MakeLibCallOptions CallOptions;
4732 CallOptions.setIsSigned(true);
4733 SplitInteger(TLI.makeLibCall(DAG, LCImpl, VT, Ops, CallOptions, dl).first, Lo,
4734 Hi);
4735}
4736
4737void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo,
4738 SDValue &Hi) {
4739 SDLoc DL(N);
4740 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4741 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
4742 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
4743 Lo = R.getValue(0);
4744 Hi = R.getValue(1);
4745 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
4746}
4747
4748void DAGTypeLegalizer::ExpandIntRes_AVG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4749 SDValue Result = TLI.expandAVG(N, DAG);
4750 SplitInteger(Result, Lo, Hi);
4751}
4752
4753void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
4754 SDValue &Hi) {
4755 SDValue Result = TLI.expandAddSubSat(N, DAG);
4756 SplitInteger(Result, Lo, Hi);
4757}
4758
4759void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
4760 SDValue &Hi) {
4761 SDValue Result = TLI.expandShlSat(N, DAG);
4762 SplitInteger(Result, Lo, Hi);
4763}
4764
4765/// This performs an expansion of the integer result for a fixed point
4766/// multiplication. The default expansion performs rounding down towards
4767/// negative infinity, though targets that do care about rounding should specify
4768/// a target hook for rounding and provide their own expansion or lowering of
4769/// fixed point multiplication to be consistent with rounding.
4770void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
4771 SDValue &Hi) {
4772 SDLoc dl(N);
4773 EVT VT = N->getValueType(0);
4774 unsigned VTSize = VT.getScalarSizeInBits();
4775 SDValue LHS = N->getOperand(0);
4776 SDValue RHS = N->getOperand(1);
4777 uint64_t Scale = N->getConstantOperandVal(2);
4778 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
4779 N->getOpcode() == ISD::UMULFIXSAT);
4780 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
4781 N->getOpcode() == ISD::SMULFIXSAT);
4782
4783 // Handle special case when scale is equal to zero.
4784 if (!Scale) {
4786 if (!Saturating) {
4787 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
4788 } else {
4789 EVT BoolVT = getSetCCResultType(VT);
4790 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
4791 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
4792 SDValue Product = Result.getValue(0);
4793 SDValue Overflow = Result.getValue(1);
4794 if (Signed) {
4795 APInt MinVal = APInt::getSignedMinValue(VTSize);
4796 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
4797 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
4798 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4799 SDValue Zero = DAG.getConstant(0, dl, VT);
4800 // Xor the inputs, if resulting sign bit is 0 the product will be
4801 // positive, else negative.
4802 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4803 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
4804 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
4805 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
4806 } else {
4807 // For unsigned multiplication, we only need to check the max since we
4808 // can't really overflow towards zero.
4809 APInt MaxVal = APInt::getMaxValue(VTSize);
4810 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4811 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
4812 }
4813 }
4814 SplitInteger(Result, Lo, Hi);
4815 return;
4816 }
4817
4818 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
4819 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
4820 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
4821
4822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4823 SDValue LL, LH, RL, RH;
4824 GetExpandedInteger(LHS, LL, LH);
4825 GetExpandedInteger(RHS, RL, RH);
4827
4828 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
4829 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4831 LL, LH, RL, RH)) {
4832 Result.clear();
4833 Result.resize(4);
4834
4835 SDValue LoTmp, HiTmp;
4836 TLI.forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, LoTmp, HiTmp);
4837 SplitInteger(LoTmp, Result[0], Result[1]);
4838 SplitInteger(HiTmp, Result[2], Result[3]);
4839 }
4840 assert(Result.size() == 4 && "Unexpected number of partlets in the result");
4841
4842 unsigned NVTSize = NVT.getScalarSizeInBits();
4843 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
4844 "the size of the current value type");
4845
4846 // After getting the multiplication result in 4 parts, we need to perform a
4847 // shift right by the amount of the scale to get the result in that scale.
4848 //
4849 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
4850 // 128 bits that are cut into 4 32-bit parts:
4851 //
4852 // HH HL LH LL
4853 // |---32---|---32---|---32---|---32---|
4854 // 128 96 64 32 0
4855 //
4856 // |------VTSize-----|
4857 //
4858 // |NVTSize-|
4859 //
4860 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
4861 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
4862 // amount and get Lo and Hi using two funnel shifts. Or for the special case
4863 // when Scale is a multiple of NVTSize we can just pick the result without
4864 // shifting.
4865 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
4866 if (Scale % NVTSize) {
4867 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl);
4868 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
4869 ShiftAmount);
4870 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
4871 ShiftAmount);
4872 } else {
4873 Lo = Result[Part0];
4874 Hi = Result[Part0 + 1];
4875 }
4876
4877 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
4878 if (!Saturating)
4879 return;
4880
4881 // Can not overflow when there is no integer part.
4882 if (Scale == VTSize)
4883 return;
4884
4885 // To handle saturation we must check for overflow in the multiplication.
4886 //
4887 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
4888 // aren't all zeroes.
4889 //
4890 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
4891 // aren't all ones or all zeroes.
4892 //
4893 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
4894 // highest bit of HH determines saturation direction in the event of signed
4895 // saturation.
4896
4897 SDValue ResultHL = Result[2];
4898 SDValue ResultHH = Result[3];
4899
4900 SDValue SatMax, SatMin;
4901 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
4902 SDValue NVTNeg1 = DAG.getAllOnesConstant(dl, NVT);
4903 EVT BoolNVT = getSetCCResultType(NVT);
4904
4905 if (!Signed) {
4906 if (Scale < NVTSize) {
4907 // Overflow happened if ((HH | (HL >> Scale)) != 0).
4908 SDValue HLAdjusted =
4909 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4910 DAG.getShiftAmountConstant(Scale, NVT, dl));
4911 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
4912 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
4913 } else if (Scale == NVTSize) {
4914 // Overflow happened if (HH != 0).
4915 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
4916 } else if (Scale < VTSize) {
4917 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
4918 SDValue HHAdjusted =
4919 DAG.getNode(ISD::SRL, dl, NVT, ResultHH,
4920 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl));
4921 SatMax = DAG.getSetCC(dl, BoolNVT, HHAdjusted, NVTZero, ISD::SETNE);
4922 } else
4923 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
4924 "(and saturation can't happen with Scale==VTSize).");
4925
4926 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
4927 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
4928 return;
4929 }
4930
4931 if (Scale < NVTSize) {
4932 // The number of overflow bits we can check are VTSize - Scale + 1 (we
4933 // include the sign bit). If these top bits are > 0, then we overflowed past
4934 // the max value. If these top bits are < -1, then we overflowed past the
4935 // min value. Otherwise, we did not overflow.
4936 unsigned OverflowBits = VTSize - Scale + 1;
4937 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4938 "Extent of overflow bits must start within HL");
4939 SDValue HLHiMask = DAG.getConstant(
4940 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
4941 SDValue HLLoMask = DAG.getConstant(
4942 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
4943 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
4944 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4945 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4946 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
4947 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4948 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
4949 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
4950 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4951 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4952 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
4953 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4954 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
4955 } else if (Scale == NVTSize) {
4956 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
4957 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4958 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4959 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
4960 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4961 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
4962 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
4963 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4964 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4965 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
4966 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4967 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
4968 } else if (Scale < VTSize) {
4969 // This is similar to the case when we saturate if Scale < NVTSize, but we
4970 // only need to check HH.
4971 unsigned OverflowBits = VTSize - Scale + 1;
4972 SDValue HHHiMask = DAG.getConstant(
4973 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
4974 SDValue HHLoMask = DAG.getConstant(
4975 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
4976 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
4977 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
4978 } else
4979 llvm_unreachable("Illegal scale for signed fixed point mul.");
4980
4981 // Saturate to signed maximum.
4982 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
4983 APInt MaxLo = APInt::getAllOnes(NVTSize);
4984 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
4985 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
4986 // Saturate to signed minimum.
4987 APInt MinHi = APInt::getSignedMinValue(NVTSize);
4988 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
4989 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
4990}
4991
4992void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
4993 SDValue &Hi) {
4994 SDLoc dl(N);
4995 // Try expanding in the existing type first.
4996 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
4997 N->getOperand(1),
4998 N->getConstantOperandVal(2), DAG);
4999
5000 if (!Res)
5001 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
5002 N->getConstantOperandVal(2), TLI, DAG);
5003 SplitInteger(Res, Lo, Hi);
5004}
5005
5006void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
5007 SDValue &Lo, SDValue &Hi) {
5008 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
5009 "Node has unexpected Opcode");
5010 SDValue LHS = Node->getOperand(0);
5011 SDValue RHS = Node->getOperand(1);
5012 SDLoc dl(Node);
5013
5014 SDValue Ovf;
5015
5016 bool IsAdd = Node->getOpcode() == ISD::SADDO;
5017 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
5018
5019 bool HasCarryOp = TLI.isOperationLegalOrCustom(
5020 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
5021
5022 if (HasCarryOp) {
5023 // Expand the subcomponents.
5024 SDValue LHSL, LHSH, RHSL, RHSH;
5025 GetExpandedInteger(LHS, LHSL, LHSH);
5026 GetExpandedInteger(RHS, RHSL, RHSH);
5027 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
5028
5029 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
5030 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
5031
5032 Ovf = Hi.getValue(1);
5033 } else {
5034 // Expand the result by simply replacing it with the equivalent
5035 // non-overflow-checking operation.
5036 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
5037 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
5038 LHS, RHS);
5039 SplitInteger(Sum, Lo, Hi);
5040
5041 // Compute the overflow.
5042 //
5043 // LHSSign -> LHS < 0
5044 // RHSSign -> RHS < 0
5045 // SumSign -> Sum < 0
5046 //
5047 // Add:
5048 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
5049 // Sub:
5050 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
5051 //
5052 // To get better codegen we can rewrite this by doing bitwise math on
5053 // the integers and extract the final sign bit at the end. So the
5054 // above becomes:
5055 //
5056 // Add:
5057 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
5058 // Sub:
5059 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
5060 //
5061 // NOTE: This is different than the expansion we do in expandSADDSUBO
5062 // because it is more costly to implement the same overflow predicate with
5063 // SETCC nodes when the integers are split.
5064 EVT VT = LHS.getValueType();
5065 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
5066 if (IsAdd)
5067 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
5068
5069 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
5070 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
5071 EVT OType = Node->getValueType(1);
5072 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
5073 }
5074
5075 // Use the calculated overflow everywhere.
5076 ReplaceValueWith(SDValue(Node, 1), Ovf);
5077}
5078
5079void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
5080 SDValue &Lo, SDValue &Hi) {
5081 EVT VT = N->getValueType(0);
5082 SDLoc dl(N);
5083 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5084
5085 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
5086 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5087 SplitInteger(Res.getValue(0), Lo, Hi);
5088 return;
5089 }
5090
5091 RTLIB::Libcall LC = RTLIB::getSDIV(VT);
5092 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
5093
5094 TargetLowering::MakeLibCallOptions CallOptions;
5095 CallOptions.setIsSigned(true);
5096 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5097}
5098
5099void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo,
5100 SDValue &Hi) {
5101 SDLoc dl(N);
5102 SDValue Shiftee = N->getOperand(0);
5103 EVT VT = Shiftee.getValueType();
5104 SDValue ShAmt = N->getOperand(1);
5105 EVT ShAmtVT = ShAmt.getValueType();
5106
5107 EVT LoadVT = VT;
5108 do {
5109 LoadVT = TLI.getTypeToTransformTo(*DAG.getContext(), LoadVT);
5110 } while (!TLI.isTypeLegal(LoadVT));
5111
5112 const unsigned ShiftUnitInBits = LoadVT.getStoreSizeInBits();
5113 assert(ShiftUnitInBits <= VT.getScalarSizeInBits());
5114 assert(isPowerOf2_32(ShiftUnitInBits) &&
5115 "Shifting unit is not a a power of two!");
5116
5117 const bool IsOneStepShift =
5118 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >=
5119 Log2_32(ShiftUnitInBits);
5120
5121 // If we can't do it as one step, we'll have two uses of shift amount,
5122 // and thus must freeze it.
5123 if (!IsOneStepShift)
5124 ShAmt = DAG.getFreeze(ShAmt);
5125
5126 unsigned VTBitWidth = VT.getScalarSizeInBits();
5127 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?");
5128 unsigned VTByteWidth = VTBitWidth / 8;
5129 assert(isPowerOf2_32(VTByteWidth) &&
5130 "Shiftee type size is not a power of two!");
5131 unsigned StackSlotByteWidth = 2 * VTByteWidth;
5132 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
5133 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth);
5134
5135 // Get a temporary stack slot 2x the width of our VT.
5136 // FIXME: reuse stack slots?
5137 Align StackAlign = DAG.getReducedAlign(StackSlotVT, /*UseABI=*/false);
5139 DAG.CreateStackTemporary(StackSlotVT.getStoreSize(), StackAlign);
5140 EVT PtrTy = StackPtr.getValueType();
5141 SDValue Ch = DAG.getEntryNode();
5142
5143 MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack(
5144 DAG.getMachineFunction(),
5145 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex());
5146
5147 // Extend the value, that is being shifted, to the entire stack slot's width.
5148 SDValue Init;
5149 if (N->getOpcode() != ISD::SHL) {
5150 unsigned WideningOpc =
5151 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5152 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee);
5153 } else {
5154 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width.
5155 SDValue AllZeros = DAG.getConstant(0, dl, VT);
5156 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee);
5157 }
5158 // And spill it into the stack slot.
5159 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackAlign);
5160
5161 // Now, compute the full-byte offset into stack slot from where we can load.
5162 // We have shift amount, which is in bits. Offset should point to an aligned
5163 // address.
5164 SDNodeFlags Flags;
5165 Flags.setExact(IsOneStepShift);
5166 SDValue SrlTmp = DAG.getNode(
5167 ISD::SRL, dl, ShAmtVT, ShAmt,
5168 DAG.getConstant(Log2_32(ShiftUnitInBits), dl, ShAmtVT), Flags);
5169 SDValue BitOffset =
5170 DAG.getNode(ISD::SHL, dl, ShAmtVT, SrlTmp,
5171 DAG.getConstant(Log2_32(ShiftUnitInBits), dl, ShAmtVT));
5172
5173 SDValue ByteOffset =
5174 DAG.getNode(ISD::SRL, dl, ShAmtVT, BitOffset,
5175 DAG.getConstant(3, dl, ShAmtVT), SDNodeFlags::Exact);
5176 // And clamp it, because OOB load is an immediate UB,
5177 // while shift overflow would have *just* been poison.
5178 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset,
5179 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT));
5180 // We have exactly two strategies on indexing into stack slot here:
5181 // 1. upwards starting from the beginning of the slot
5182 // 2. downwards starting from the middle of the slot
5183 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift
5184 // and vice versa on big-endian machine.
5185 bool WillIndexUpwards = N->getOpcode() != ISD::SHL;
5186 if (DAG.getDataLayout().isBigEndian())
5187 WillIndexUpwards = !WillIndexUpwards;
5188
5189 SDValue AdjStackPtr;
5190 if (WillIndexUpwards) {
5191 AdjStackPtr = StackPtr;
5192 } else {
5193 AdjStackPtr = DAG.getMemBasePlusOffset(
5194 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl);
5195 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT);
5196 }
5197
5198 // Get the pointer somewhere into the stack slot from which we need to load.
5199 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy);
5200 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl);
5201
5202 // And load it! While the load is not legal, legalizing it is obvious.
5203 SDValue Res =
5204 DAG.getLoad(VT, dl, Ch, AdjStackPtr,
5205 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
5206 commonAlignment(StackAlign, LoadVT.getStoreSize()));
5207
5208 // If we may still have a remaining bits to shift by, do so now.
5209 if (!IsOneStepShift) {
5210 SDValue ShAmtRem =
5211 DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
5212 DAG.getConstant(ShiftUnitInBits - 1, dl, ShAmtVT));
5213 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem);
5214 }
5215
5216 // Finally, split the computed value.
5217 SplitInteger(Res, Lo, Hi);
5218}
5219
5220void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
5221 SDValue &Lo, SDValue &Hi) {
5222 EVT VT = N->getValueType(0);
5223 unsigned Opc = N->getOpcode();
5224 SDLoc dl(N);
5225
5226 // If we can emit an efficient shift operation, do so now. Check to see if
5227 // the RHS is a constant.
5228 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
5229 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
5230
5231 // If we can determine that the high bit of the shift is zero or one, even if
5232 // the low bits are variable, emit this shift in an optimized form.
5233 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
5234 return;
5235
5236 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
5237 unsigned PartsOpc;
5238 if (Opc == ISD::SHL) {
5239 PartsOpc = ISD::SHL_PARTS;
5240 } else if (Opc == ISD::SRL) {
5241 PartsOpc = ISD::SRL_PARTS;
5242 } else {
5243 assert(Opc == ISD::SRA && "Unknown shift!");
5244 PartsOpc = ISD::SRA_PARTS;
5245 }
5246
5247 // Next check to see if the target supports this SHL_PARTS operation or if it
5248 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
5249 // size, but create a libcall instead.
5250 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5251 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
5252 const bool LegalOrCustom =
5253 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
5254 Action == TargetLowering::Custom;
5255
5256 unsigned ExpansionFactor = 1;
5257 // That VT->NVT expansion is one step. But will we re-expand NVT?
5258 for (EVT TmpVT = NVT;;) {
5259 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT);
5260 if (NewTMPVT == TmpVT)
5261 break;
5262 TmpVT = NewTMPVT;
5263 ++ExpansionFactor;
5264 }
5265
5267 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor);
5268
5270 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi);
5271
5272 if (LegalOrCustom &&
5274 // Expand the subcomponents.
5275 SDValue LHSL, LHSH;
5276 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
5277 EVT VT = LHSL.getValueType();
5278
5279 // If the shift amount operand is coming from a vector legalization it may
5280 // have an illegal type. Fix that first by casting the operand, otherwise
5281 // the new SHL_PARTS operation would need further legalization.
5282 SDValue ShiftOp = N->getOperand(1);
5283 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5284 if (ShiftOp.getValueType() != ShiftTy)
5285 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
5286
5287 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
5288 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
5289 Hi = Lo.getValue(1);
5290 return;
5291 }
5292
5293 // Otherwise, emit a libcall.
5294 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5295 bool isSigned;
5296 if (Opc == ISD::SHL) {
5297 isSigned = false; /*sign irrelevant*/
5298 LC = RTLIB::getSHL(VT);
5299 } else if (Opc == ISD::SRL) {
5300 isSigned = false;
5301 LC = RTLIB::getSRL(VT);
5302 } else {
5303 assert(Opc == ISD::SRA && "Unknown shift!");
5304 isSigned = true;
5305 LC = RTLIB::getSRA(VT);
5306 }
5307
5308 if (RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(LC)) {
5309 EVT ShAmtTy =
5310 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
5311 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
5312 SDValue Ops[2] = {N->getOperand(0), ShAmt};
5313 TargetLowering::MakeLibCallOptions CallOptions;
5314 CallOptions.setIsSigned(isSigned);
5315 SplitInteger(
5316 TLI.makeLibCall(DAG, LibcallImpl, VT, Ops, CallOptions, dl).first, Lo,
5317 Hi);
5318 return;
5319 }
5320
5321 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
5322 llvm_unreachable("Unsupported shift!");
5323}
5324
5325void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
5326 SDValue &Lo, SDValue &Hi) {
5327 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5328 SDLoc dl(N);
5329 SDValue Op = N->getOperand(0);
5330 if (Op.getValueType().bitsLE(NVT)) {
5331 // The low part is sign extension of the input (degenerates to a copy).
5332 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
5333 // The high part is obtained by SRA'ing all but one of the bits of low part.
5334 unsigned LoSize = NVT.getSizeInBits();
5335 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
5336 DAG.getShiftAmountConstant(LoSize - 1, NVT, dl));
5337 } else {
5338 // For example, extension of an i48 to an i64. The operand type necessarily
5339 // promotes to the result type, so will end up being expanded too.
5340 assert(getTypeAction(Op.getValueType()) ==
5342 "Only know how to promote this result!");
5343 SDValue Res = GetPromotedInteger(Op);
5344 assert(Res.getValueType() == N->getValueType(0) &&
5345 "Operand over promoted?");
5346 // Split the promoted operand. This will simplify when it is expanded.
5347 SplitInteger(Res, Lo, Hi);
5348 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5349 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
5350 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
5351 ExcessBits)));
5352 }
5353}
5354
5355void DAGTypeLegalizer::
5356ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
5357 SDLoc dl(N);
5358 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5359 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
5360
5361 if (EVT.bitsLE(Lo.getValueType())) {
5362 // sext_inreg the low part if needed.
5363 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
5364 N->getOperand(1));
5365
5366 // The high part gets the sign extension from the lo-part. This handles
5367 // things like sextinreg V:i64 from i8.
5368 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
5369 DAG.getShiftAmountConstant(Hi.getValueSizeInBits() - 1,
5370 Hi.getValueType(), dl));
5371 } else {
5372 // For example, extension of an i48 to an i64. Leave the low part alone,
5373 // sext_inreg the high part.
5374 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
5375 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
5376 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
5377 ExcessBits)));
5378 }
5379}
5380
5381void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
5382 SDValue &Lo, SDValue &Hi) {
5383 EVT VT = N->getValueType(0);
5384 SDLoc dl(N);
5385 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5386
5387 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
5388 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5389 SplitInteger(Res.getValue(1), Lo, Hi);
5390 return;
5391 }
5392
5393 RTLIB::Libcall LC = RTLIB::getSREM(VT);
5394 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
5395
5396 TargetLowering::MakeLibCallOptions CallOptions;
5397 CallOptions.setIsSigned(true);
5398 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5399}
5400
5401void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
5402 SDValue &Lo, SDValue &Hi) {
5403 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5404 SDValue InOp = N->getOperand(0);
5405 EVT InVT = InOp.getValueType();
5406 SDLoc dl(N);
5407 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, InOp);
5408 Hi = DAG.getNode(ISD::SRL, dl, InVT, InOp,
5409 DAG.getShiftAmountConstant(NVT.getSizeInBits(), InVT, dl));
5410 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
5411}
5412
5413void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
5414 SDValue &Lo, SDValue &Hi) {
5415 EVT VT = N->getValueType(0);
5416 SDLoc dl(N);
5417
5418 if (N->getOpcode() == ISD::UMULO) {
5419 // This section expands the operation into the following sequence of
5420 // instructions. `iNh` here refers to a type which has half the bit width of
5421 // the type the original operation operated on.
5422 //
5423 // %0 = %LHS.HI != 0 && %RHS.HI != 0
5424 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
5425 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
5426 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
5427 // %4 = add iNh %1.0, %2.0 as iN
5428 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
5429 //
5430 // %lo = %3.LO
5431 // %hi = %5.0
5432 // %ovf = %0 || %1.1 || %2.1 || %5.1
5433 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
5434 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
5435 GetExpandedInteger(LHS, LHSLow, LHSHigh);
5436 GetExpandedInteger(RHS, RHSLow, RHSHigh);
5437 EVT HalfVT = LHSLow.getValueType();
5438 EVT BitVT = N->getValueType(1);
5439 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
5440
5441 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
5442 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
5443 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
5444 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
5445
5446 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
5447 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
5448
5449 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
5450 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
5451
5452 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
5453
5454 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
5455 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
5456 // operation recursively legalized?).
5457 //
5458 // Many backends understand this pattern and will convert into LOHI
5459 // themselves, if applicable.
5460 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
5461 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
5462 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
5463 SplitInteger(Three, Lo, Hi);
5464
5465 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
5466 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
5467 ReplaceValueWith(SDValue(N, 1), Overflow);
5468 return;
5469 }
5470
5471 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
5472 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5473 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
5474
5475 // Replace this with a libcall that will check overflow.
5476 RTLIB::Libcall LC = RTLIB::getMULO(VT);
5477 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
5478
5479 // If we don't have the libcall or if the function we are compiling is the
5480 // implementation of the expected libcall (avoid inf-loop), expand inline.
5481 if (LCImpl == RTLIB::Unsupported ||
5483 DAG.getMachineFunction().getName()) {
5484 // FIXME: This is not an optimal expansion, but better than crashing.
5485 SDValue MulLo, MulHi;
5486 TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, N->getOperand(0),
5487 N->getOperand(1), MulLo, MulHi);
5488 SDValue SRA = DAG.getNode(
5489 ISD::SRA, dl, VT, MulLo,
5490 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl));
5491 SDValue Overflow =
5492 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
5493 SplitInteger(MulLo, Lo, Hi);
5494 ReplaceValueWith(SDValue(N, 1), Overflow);
5495 return;
5496 }
5497
5498 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
5499 // Temporary for the overflow value, default it to zero.
5500 SDValue Chain =
5501 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
5502 MachinePointerInfo());
5503
5505 for (const SDValue &Op : N->op_values()) {
5506 EVT ArgVT = Op.getValueType();
5507 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
5508 TargetLowering::ArgListEntry Entry(Op, ArgTy);
5509 Entry.IsSExt = true;
5510 Entry.IsZExt = false;
5511 Args.push_back(Entry);
5512 }
5513
5514 // Also pass the address of the overflow check.
5515 TargetLowering::ArgListEntry Entry(
5516 Temp, PointerType::getUnqual(PtrTy->getContext()));
5517 Entry.IsSExt = true;
5518 Entry.IsZExt = false;
5519 Args.push_back(Entry);
5520
5521 SDValue Func = DAG.getExternalSymbol(LCImpl, PtrVT);
5522
5523 TargetLowering::CallLoweringInfo CLI(DAG);
5524 CLI.setDebugLoc(dl)
5525 .setChain(Chain)
5526 .setLibCallee(DAG.getLibcalls().getLibcallImplCallingConv(LCImpl), RetTy,
5527 Func, std::move(Args))
5528 .setSExtResult();
5529
5530 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
5531
5532 SplitInteger(CallInfo.first, Lo, Hi);
5533 SDValue Temp2 =
5534 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
5535 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
5536 DAG.getConstant(0, dl, PtrVT),
5537 ISD::SETNE);
5538 // Use the overflow from the libcall everywhere.
5539 ReplaceValueWith(SDValue(N, 1), Ofl);
5540}
5541
5542void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
5543 SDValue &Lo, SDValue &Hi) {
5544 EVT VT = N->getValueType(0);
5545 SDLoc dl(N);
5546 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5547
5548 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
5549 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5550 SplitInteger(Res.getValue(0), Lo, Hi);
5551 return;
5552 }
5553
5554 // Try to expand UDIV by constant.
5555 if (isa<ConstantSDNode>(N->getOperand(1))) {
5556 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5557 // Only if the new type is legal.
5558 if (isTypeLegal(NVT)) {
5559 SDValue InL, InH;
5560 GetExpandedInteger(N->getOperand(0), InL, InH);
5562 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
5563 Lo = Result[0];
5564 Hi = Result[1];
5565 return;
5566 }
5567 }
5568 }
5569
5570 RTLIB::Libcall LC = RTLIB::getUDIV(VT);
5571 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
5572
5573 TargetLowering::MakeLibCallOptions CallOptions;
5574 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5575}
5576
5577void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
5578 SDValue &Lo, SDValue &Hi) {
5579 EVT VT = N->getValueType(0);
5580 SDLoc dl(N);
5581 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5582
5583 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
5584 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5585 SplitInteger(Res.getValue(1), Lo, Hi);
5586 return;
5587 }
5588
5589 // Try to expand UREM by constant.
5590 if (isa<ConstantSDNode>(N->getOperand(1))) {
5591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5592 // Only if the new type is legal.
5593 if (isTypeLegal(NVT)) {
5594 SDValue InL, InH;
5595 GetExpandedInteger(N->getOperand(0), InL, InH);
5597 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
5598 Lo = Result[0];
5599 Hi = Result[1];
5600 return;
5601 }
5602 }
5603 }
5604
5605 RTLIB::Libcall LC = RTLIB::getUREM(VT);
5606 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
5607
5608 TargetLowering::MakeLibCallOptions CallOptions;
5609 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5610}
5611
5612void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
5613 SDValue &Lo, SDValue &Hi) {
5614 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5615 SDLoc dl(N);
5616 SDValue Op = N->getOperand(0);
5617 if (Op.getValueType().bitsLE(NVT)) {
5618 // The low part is zero extension of the input (degenerates to a copy).
5619 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
5620 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
5621 } else {
5622 // For example, extension of an i48 to an i64. The operand type necessarily
5623 // promotes to the result type, so will end up being expanded too.
5624 assert(getTypeAction(Op.getValueType()) ==
5626 "Only know how to promote this result!");
5627 SDValue Res = GetPromotedInteger(Op);
5628 assert(Res.getValueType() == N->getValueType(0) &&
5629 "Operand over promoted?");
5630 // Split the promoted operand. This will simplify when it is expanded.
5631 SplitInteger(Res, Lo, Hi);
5632 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5633 Hi = DAG.getZeroExtendInReg(Hi, dl,
5634 EVT::getIntegerVT(*DAG.getContext(),
5635 ExcessBits));
5636 }
5637}
5638
5639void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
5640 SDValue &Lo, SDValue &Hi) {
5641 SDLoc dl(N);
5642 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
5643 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
5644 SDValue Zero = DAG.getConstant(0, dl, VT);
5645 SDValue Swap = DAG.getAtomicCmpSwap(
5647 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
5648 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
5649
5650 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
5651 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
5652}
5653
5654void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
5655 SDValue &Lo, SDValue &Hi) {
5656 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
5657 // both halves independently.
5658 SDValue Res = TLI.expandVecReduce(N, DAG);
5659 SplitInteger(Res, Lo, Hi);
5660}
5661
5662void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
5663 SDValue &Lo, SDValue &Hi) {
5664 // Delegate to funnel-shift expansion.
5665 SDLoc DL(N);
5666 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
5667 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0),
5668 N->getOperand(0), N->getOperand(1));
5669 SplitInteger(Res, Lo, Hi);
5670}
5671
5672void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
5673 SDValue &Hi) {
5674 // Values numbered from least significant to most significant.
5675 SDValue In1, In2, In3, In4;
5676 GetExpandedInteger(N->getOperand(0), In3, In4);
5677 GetExpandedInteger(N->getOperand(1), In1, In2);
5678 EVT HalfVT = In1.getValueType();
5679
5680 SDLoc DL(N);
5681 unsigned Opc = N->getOpcode();
5682 SDValue ShAmt = N->getOperand(2);
5683 EVT ShAmtVT = ShAmt.getValueType();
5684 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5685
5686 // If the shift amount is at least half the bitwidth, swap the inputs.
5687 unsigned HalfVTBits = HalfVT.getScalarSizeInBits();
5688 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt,
5689 DAG.getConstant(HalfVTBits, DL, ShAmtVT));
5690 SDValue Cond =
5691 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT),
5693
5694 // Expand to a pair of funnel shifts.
5695 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5696 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT);
5697
5698 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2);
5699 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3);
5700 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4);
5701 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt);
5702 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
5703}
5704
5705void DAGTypeLegalizer::ExpandIntRes_CLMUL(SDNode *N, SDValue &Lo, SDValue &Hi) {
5706 if (N->getOpcode() != ISD::CLMUL) {
5707 SDValue Res = TLI.expandCLMUL(N, DAG);
5708 return SplitInteger(Res, Lo, Hi);
5709 }
5710
5711 SDValue LL, LH, RL, RH;
5712 GetExpandedInteger(N->getOperand(0), LL, LH);
5713 GetExpandedInteger(N->getOperand(1), RL, RH);
5714 EVT HalfVT = LL.getValueType();
5715 SDLoc DL(N);
5716
5717 // The low bits are a direct CLMUL of the the low bits.
5718 Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, LL, RL);
5719
5720 // We compute two Hi-Lo cross-products, XOR them, and XOR it with the overflow
5721 // of the CLMUL of the low bits (given by CLMULH of the low bits) to yield the
5722 // final high bits.
5723 SDValue LoH = DAG.getNode(ISD::CLMULH, DL, HalfVT, LL, RL);
5724 SDValue HiLoCross1 = DAG.getNode(ISD::CLMUL, DL, HalfVT, LL, RH);
5725 SDValue HiLoCross2 = DAG.getNode(ISD::CLMUL, DL, HalfVT, LH, RL);
5726 SDValue HiLoCross = DAG.getNode(ISD::XOR, DL, HalfVT, HiLoCross1, HiLoCross2);
5727 Hi = DAG.getNode(ISD::XOR, DL, HalfVT, LoH, HiLoCross);
5728}
5729
5730void DAGTypeLegalizer::ExpandIntRes_PEXT(SDNode *N, SDValue &Lo, SDValue &Hi) {
5731 SDValue Res = TLI.expandPEXT(N, DAG);
5732 SplitInteger(Res, Lo, Hi);
5733}
5734
5735void DAGTypeLegalizer::ExpandIntRes_PDEP(SDNode *N, SDValue &Lo, SDValue &Hi) {
5736 SDValue Res = TLI.expandPDEP(N, DAG);
5737 SplitInteger(Res, Lo, Hi);
5738}
5739
5740void DAGTypeLegalizer::ExpandIntRes_MULH(SDNode *N, SDValue &Lo, SDValue &Hi) {
5741 SDValue Res = TLI.expandMULH(N, DAG);
5742 SplitInteger(Res, Lo, Hi);
5743}
5744
5745void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
5746 SDValue &Hi) {
5747 EVT VT = N->getValueType(0);
5748 EVT HalfVT =
5749 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
5750 SDLoc dl(N);
5751
5752 // We assume VSCALE(1) fits into a legal integer.
5753 APInt One(HalfVT.getSizeInBits(), 1);
5754 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
5755 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
5756 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
5757 SplitInteger(Res, Lo, Hi);
5758}
5759
5760void DAGTypeLegalizer::ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo,
5761 SDValue &Hi) {
5762 const Function &Fn = DAG.getMachineFunction().getFunction();
5763 Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure(
5764 "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc()));
5765 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
5766 EVT LoVT, HiVT;
5767 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
5768 Lo = DAG.getPOISON(LoVT);
5769 Hi = DAG.getPOISON(HiVT);
5770}
5771
5772void DAGTypeLegalizer::ExpandIntRes_CTTZ_ELTS(SDNode *N, SDValue &Lo,
5773 SDValue &Hi) {
5774 // Assume that the maximum number of vector elements fits in getVectorIdxTy
5775 // and expand to that.
5776 EVT VT = N->getSimpleValueType(0);
5777 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
5778 assert(IdxVT.bitsLT(VT) &&
5779 "VectorIdxTy should be smaller than type to be expanded?");
5780
5781 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), IdxVT, N->getOperand(0));
5782 Res = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Res);
5783 SplitInteger(Res, Lo, Hi);
5784}
5785
5786//===----------------------------------------------------------------------===//
5787// Integer Operand Expansion
5788//===----------------------------------------------------------------------===//
5789
5790/// ExpandIntegerOperand - This method is called when the specified operand of
5791/// the specified node is found to need expansion. At this point, all of the
5792/// result types of the node are known to be legal, but other operands of the
5793/// node may need promotion or expansion as well as the specified one.
5794bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
5795 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG));
5796 SDValue Res = SDValue();
5797
5798 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
5799 return false;
5800
5801 switch (N->getOpcode()) {
5802 default:
5803 #ifndef NDEBUG
5804 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
5805 N->dump(&DAG); dbgs() << "\n";
5806 #endif
5807 report_fatal_error("do not know how to expand this operator's operand!");
5808
5809 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
5810 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
5811 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
5812 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
5813 case ISD::FAKE_USE:
5814 Res = ExpandOp_FAKE_USE(N);
5815 break;
5818 Res = TLI.expandLoopDependenceMask(N, DAG);
5819 break;
5820 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
5821 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
5822 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
5823 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
5824 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
5825 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
5827 case ISD::SINT_TO_FP:
5829 case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break;
5830 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
5831 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
5832
5833 case ISD::SHL:
5834 case ISD::SRA:
5835 case ISD::SRL:
5836 case ISD::ROTL:
5837 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
5838 case ISD::RETURNADDR:
5839 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
5840
5841 case ISD::SCMP:
5842 case ISD::UCMP: Res = ExpandIntOp_CMP(N); break;
5843
5844 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
5845 case ISD::STACKMAP:
5846 Res = ExpandIntOp_STACKMAP(N, OpNo);
5847 break;
5848 case ISD::PATCHPOINT:
5849 Res = ExpandIntOp_PATCHPOINT(N, OpNo);
5850 break;
5851 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5852 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5853 Res = ExpandIntOp_VP_STRIDED(N, OpNo);
5854 break;
5856 Res = ExpandIntOp_WRITE_REGISTER(N, OpNo);
5857 break;
5858 }
5859
5860 // If the result is null, the sub-method took care of registering results etc.
5861 if (!Res.getNode()) return false;
5862
5863 // If the result is N, the sub-method updated N in place. Tell the legalizer
5864 // core about this.
5865 if (Res.getNode() == N)
5866 return true;
5867
5868 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
5869 "Invalid operand expansion");
5870
5871 ReplaceValueWith(SDValue(N, 0), Res);
5872 return false;
5873}
5874
5875/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
5876/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
5877void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
5878 SDValue &NewRHS,
5879 ISD::CondCode &CCCode,
5880 const SDLoc &dl) {
5881 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5882 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5883 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5884
5885 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
5886 if (RHSLo == RHSHi && isAllOnesConstant(RHSLo)) {
5887 // Equality comparison to -1.
5888 NewLHS = DAG.getNode(ISD::AND, dl, LHSLo.getValueType(), LHSLo, LHSHi);
5889 NewRHS = RHSLo;
5890 return;
5891 }
5892
5893 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
5894 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
5895 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
5896 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
5897 return;
5898 }
5899
5900 // If this is a comparison of the sign bit, just look at the top part.
5901 // X > -1, x < 0
5902 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
5903 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
5904 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
5905 NewLHS = LHSHi;
5906 NewRHS = RHSHi;
5907 return;
5908 }
5909
5910 // FIXME: This generated code sucks.
5911 ISD::CondCode LowCC;
5912 switch (CCCode) {
5913 default: llvm_unreachable("Unknown integer setcc!");
5914 case ISD::SETLT:
5915 case ISD::SETULT: LowCC = ISD::SETULT; break;
5916 case ISD::SETGT:
5917 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
5918 case ISD::SETLE:
5919 case ISD::SETULE: LowCC = ISD::SETULE; break;
5920 case ISD::SETGE:
5921 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
5922 }
5923
5924 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
5925 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
5926 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
5927
5928 // NOTE: on targets without efficient SELECT of bools, we can always use
5929 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
5930 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
5931 nullptr);
5932 SDValue LoCmp, HiCmp;
5933 if (TLI.isTypeLegal(LHSLo.getValueType()))
5934 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
5935 RHSLo, LowCC, false, DagCombineInfo, dl);
5936 if (!LoCmp.getNode())
5937 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
5938 RHSLo, LowCC);
5939 if (TLI.isTypeLegal(LHSHi.getValueType()))
5940 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
5941 RHSHi, CCCode, false, DagCombineInfo, dl);
5942 if (!HiCmp.getNode())
5943 HiCmp =
5944 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
5945 LHSHi, RHSHi, DAG.getCondCode(CCCode));
5946
5947 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
5948 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
5949
5950 bool EqAllowed = ISD::isTrueWhenEqual(CCCode);
5951
5952 // FIXME: Is the HiCmpC->isOne() here correct for
5953 // ZeroOrNegativeOneBooleanContent.
5954 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
5955 (!EqAllowed &&
5956 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
5957 // For LE / GE, if high part is known false, ignore the low part.
5958 // For LT / GT: if low part is known false, return the high part.
5959 // if high part is known true, ignore the low part.
5960 NewLHS = HiCmp;
5961 NewRHS = SDValue();
5962 return;
5963 }
5964
5965 if (LHSHi == RHSHi) {
5966 // Comparing the low bits is enough.
5967 NewLHS = LoCmp;
5968 NewRHS = SDValue();
5969 return;
5970 }
5971
5972 // Lower with SETCCCARRY if the target supports it.
5973 EVT HiVT = LHSHi.getValueType();
5974 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
5975 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
5976
5977 // FIXME: Make all targets support this, then remove the other lowering.
5978 if (HasSETCCCARRY) {
5979 // SETCCCARRY can detect < and >= directly. For > and <=, flip
5980 // operands and condition code.
5981 bool FlipOperands = false;
5982 switch (CCCode) {
5983 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
5984 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
5985 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
5986 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
5987 default: break;
5988 }
5989 if (FlipOperands) {
5990 std::swap(LHSLo, RHSLo);
5991 std::swap(LHSHi, RHSHi);
5992 }
5993 // Perform a wide subtraction, feeding the carry from the low part into
5994 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
5995 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
5996 // zero or positive iff LHS >= RHS.
5997 EVT LoVT = LHSLo.getValueType();
5998 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
5999 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
6000 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
6001 LHSHi, RHSHi, LowCmp.getValue(1),
6002 DAG.getCondCode(CCCode));
6003 NewLHS = Res;
6004 NewRHS = SDValue();
6005 return;
6006 }
6007
6008 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
6009 false, DagCombineInfo, dl);
6010 if (!NewLHS.getNode())
6011 NewLHS =
6012 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
6013 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
6014 NewRHS = SDValue();
6015}
6016
6017SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
6018 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
6019 ISD::CondCode CCCode = N->getOperandCondCode(1);
6020 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
6021
6022 // If ExpandSetCCOperands returned a scalar, we need to compare the result
6023 // against zero to select between true and false values.
6024 if (!NewRHS.getNode()) {
6025 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
6026 CCCode = ISD::SETNE;
6027 }
6028
6029 // Update N to have the operands specified.
6030 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
6031 DAG.getCondCode(CCCode), NewLHS, NewRHS,
6032 N->getOperand(4)), 0);
6033}
6034
6035SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
6036 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
6037 ISD::CondCode CCCode = N->getOperandCondCode(4);
6038 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
6039
6040 // If ExpandSetCCOperands returned a scalar, we need to compare the result
6041 // against zero to select between true and false values.
6042 if (!NewRHS.getNode()) {
6043 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
6044 CCCode = ISD::SETNE;
6045 }
6046
6047 // Update N to have the operands specified.
6048 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
6049 N->getOperand(2), N->getOperand(3),
6050 DAG.getCondCode(CCCode)), 0);
6051}
6052
6053SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
6054 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
6055 ISD::CondCode CCCode = N->getOperandCondCode(2);
6056 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
6057
6058 // If ExpandSetCCOperands returned a scalar, use it.
6059 if (!NewRHS.getNode()) {
6060 assert(NewLHS.getValueType() == N->getValueType(0) &&
6061 "Unexpected setcc expansion!");
6062 return NewLHS;
6063 }
6064
6065 // Otherwise, update N to have the operands specified.
6066 return SDValue(
6067 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
6068}
6069
6070SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
6071 SDValue LHS = N->getOperand(0);
6072 SDValue RHS = N->getOperand(1);
6073 SDValue Carry = N->getOperand(2);
6074 SDValue Cond = N->getOperand(3);
6075 SDLoc dl = SDLoc(N);
6076
6077 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
6078 GetExpandedInteger(LHS, LHSLo, LHSHi);
6079 GetExpandedInteger(RHS, RHSLo, RHSHi);
6080
6081 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high.
6082 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
6083 SDValue LowCmp =
6084 DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry);
6085 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
6086 LowCmp.getValue(1), Cond);
6087}
6088
6089SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
6090 // Split the operand and replace with SPLAT_VECTOR_PARTS.
6091 SDValue Lo, Hi;
6092 GetExpandedInteger(N->getOperand(0), Lo, Hi);
6093 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
6094 Hi);
6095}
6096
6097SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
6098 // The value being shifted is legal, but the shift amount is too big.
6099 // It follows that either the result of the shift is undefined, or the
6100 // upper half of the shift amount is zero. Just use the lower half.
6101 SDValue Lo, Hi;
6102 GetExpandedInteger(N->getOperand(1), Lo, Hi);
6103 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
6104}
6105
6106SDValue DAGTypeLegalizer::ExpandIntOp_CMP(SDNode *N) {
6107 return TLI.expandCMP(N, DAG);
6108}
6109
6110SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
6111 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
6112 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
6113 // constant to valid type.
6114 SDValue Lo, Hi;
6115 GetExpandedInteger(N->getOperand(0), Lo, Hi);
6116 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
6117}
6118
6119SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) {
6120 bool IsStrict = N->isStrictFPOpcode();
6121 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
6122 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
6123 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
6124 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
6125 EVT DstVT = N->getValueType(0);
6126 RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(Op.getValueType(), DstVT)
6127 : RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
6128 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
6129 "Don't know how to expand this XINT_TO_FP!");
6130 TargetLowering::MakeLibCallOptions CallOptions;
6131 CallOptions.setIsSigned(true);
6132 std::pair<SDValue, SDValue> Tmp =
6133 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
6134
6135 if (!IsStrict)
6136 return Tmp.first;
6137
6138 ReplaceValueWith(SDValue(N, 1), Tmp.second);
6139 ReplaceValueWith(SDValue(N, 0), Tmp.first);
6140 return SDValue();
6141}
6142
6143SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
6144 assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?");
6145
6146 if (ISD::isNormalStore(N))
6147 return ExpandOp_NormalStore(N, OpNo);
6148
6149 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
6150 assert(OpNo == 1 && "Can only expand the stored value so far");
6151
6152 EVT VT = N->getOperand(1).getValueType();
6153 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6154 SDValue Ch = N->getChain();
6155 SDValue Ptr = N->getBasePtr();
6156 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
6157 MMOMetadata Metadata = N->getMMOMetadataForSubAccess();
6158 SDLoc dl(N);
6159 SDValue Lo, Hi;
6160
6161 assert(NVT.isByteSized() && "Expanded type not byte sized!");
6162
6163 if (N->getMemoryVT().bitsLE(NVT)) {
6164 GetExpandedInteger(N->getValue(), Lo, Hi);
6165 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
6166 N->getMemoryVT(), N->getBaseAlign(), MMOFlags,
6167 Metadata);
6168 }
6169
6170 if (DAG.getDataLayout().isLittleEndian()) {
6171 // Little-endian - low bits are at low addresses.
6172 GetExpandedInteger(N->getValue(), Lo, Hi);
6173
6174 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), N->getBaseAlign(),
6175 MMOFlags, Metadata);
6176
6177 unsigned ExcessBits =
6178 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
6179 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
6180
6181 // Increment the pointer to the other half.
6182 unsigned IncrementSize = NVT.getSizeInBits()/8;
6183 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
6184 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
6185 N->getPointerInfo().getWithOffset(IncrementSize),
6186 NEVT, N->getBaseAlign(), MMOFlags, Metadata);
6187 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
6188 }
6189
6190 // Big-endian - high bits are at low addresses. Favor aligned stores at
6191 // the cost of some bit-fiddling.
6192 GetExpandedInteger(N->getValue(), Lo, Hi);
6193
6194 EVT ExtVT = N->getMemoryVT();
6195 unsigned EBytes = ExtVT.getStoreSize();
6196 unsigned IncrementSize = NVT.getSizeInBits()/8;
6197 unsigned ExcessBits = (EBytes - IncrementSize)*8;
6198 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
6199 ExtVT.getSizeInBits() - ExcessBits);
6200
6201 if (ExcessBits < NVT.getSizeInBits()) {
6202 // Transfer high bits from the top of Lo to the bottom of Hi.
6203 Hi = DAG.getNode(
6204 ISD::SHL, dl, NVT, Hi,
6205 DAG.getShiftAmountConstant(NVT.getSizeInBits() - ExcessBits, NVT, dl));
6206 Hi = DAG.getNode(
6207 ISD::OR, dl, NVT, Hi,
6208 DAG.getNode(ISD::SRL, dl, NVT, Lo,
6209 DAG.getShiftAmountConstant(ExcessBits, NVT, dl)));
6210 }
6211
6212 // Store both the high bits and maybe some of the low bits.
6213 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
6214 N->getBaseAlign(), MMOFlags, Metadata);
6215
6216 // Increment the pointer to the other half.
6217 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
6218 // Store the lowest ExcessBits bits in the second half.
6219 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
6220 N->getPointerInfo().getWithOffset(IncrementSize),
6221 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
6222 N->getBaseAlign(), MMOFlags, Metadata);
6223 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
6224}
6225
6226SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
6227 SDValue InL, InH;
6228 GetExpandedInteger(N->getOperand(0), InL, InH);
6229 // Just truncate the low part of the source.
6230 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
6231}
6232
6233SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
6234 SDLoc dl(N);
6235 SDValue Swap =
6236 DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(),
6237 N->getOperand(0), N->getOperand(2), N->getOperand(1),
6238 cast<AtomicSDNode>(N)->getMemOperand());
6239 return Swap.getValue(1);
6240}
6241
6242SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
6243 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
6244 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
6245
6246 SDValue Hi; // The upper half is dropped out.
6247 SmallVector<SDValue, 8> NewOps(N->ops());
6248 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi);
6249
6250 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
6251}
6252
6253SDValue DAGTypeLegalizer::ExpandIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo) {
6254 const Function &Fn = DAG.getMachineFunction().getFunction();
6255 Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure(
6256 "cannot use llvm.write_register with illegal type", Fn,
6257 N->getDebugLoc()));
6258
6259 return N->getOperand(0);
6260}
6261
6262SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
6263 SDLoc dl(N);
6264
6265 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6266 SDValue V1 = GetPromotedInteger(N->getOperand(1));
6267 EVT OutVT = V0.getValueType();
6268
6269 return DAG.getNode(N->getOpcode(), dl, OutVT, V0, V1, N->getOperand(2));
6270}
6271
6272SDValue DAGTypeLegalizer::PromoteIntRes_VP_SPLICE(SDNode *N) {
6273 SDLoc dl(N);
6274
6275 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6276 SDValue V1 = GetPromotedInteger(N->getOperand(1));
6277 EVT OutVT = V0.getValueType();
6278
6279 SmallVector<SDValue, 6> NewOps(N->ops());
6280 NewOps[0] = V0;
6281 NewOps[1] = V1;
6282 return DAG.getNode(N->getOpcode(), dl, OutVT, NewOps);
6283}
6284
6285SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REPEAT(SDNode *N) {
6286 SDLoc DL(N);
6287
6288 EVT OutVT = N->getValueType(0);
6289 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6290 EVT NInVT = N->getOperand(0).getValueType().changeVectorElementType(
6291 *DAG.getContext(), NOutVT.getVectorElementType());
6292
6293 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, DL, NInVT, N->getOperand(0));
6294 return DAG.getNode(N->getOpcode(), DL, NOutVT, Op);
6295}
6296
6297SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
6298 SDLoc DL(N);
6299 unsigned Factor = N->getNumOperands();
6300
6302 for (unsigned i = 0; i != Factor; i++)
6303 Ops[i] = GetPromotedInteger(N->getOperand(i));
6304
6305 SmallVector<EVT, 8> ResVTs(Factor, Ops[0].getValueType());
6306 SDValue Res = DAG.getNode(N->getOpcode(), DL, DAG.getVTList(ResVTs), Ops);
6307
6308 for (unsigned i = 0; i != Factor; i++)
6309 SetPromotedInteger(SDValue(N, i), Res.getValue(i));
6310
6311 return SDValue();
6312}
6313
6314SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
6315
6316 EVT OutVT = N->getValueType(0);
6317 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6318 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6319 EVT NOutVTElem = NOutVT.getVectorElementType();
6320
6321 SDLoc dl(N);
6322 SDValue BaseIdx = N->getOperand(1);
6323
6324 // TODO: We may be able to use this for types other than scalable
6325 // vectors and fix those tests that expect BUILD_VECTOR to be used
6326 if (OutVT.isScalableVector()) {
6327 SDValue InOp0 = N->getOperand(0);
6328 EVT InVT = InOp0.getValueType();
6329
6330 // Try and extract from a smaller type so that it eventually falls
6331 // into the promotion code below.
6332 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
6333 getTypeAction(InVT) == TargetLowering::TypeLegal) {
6334 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
6335 unsigned NElts = NInVT.getVectorMinNumElements();
6336 uint64_t IdxVal = BaseIdx->getAsZExtVal();
6337
6338 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
6339 DAG.getConstant(alignDown(IdxVal, NElts), dl,
6340 BaseIdx.getValueType()));
6341 SDValue Step2 = DAG.getNode(
6342 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
6343 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
6344 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
6345 }
6346
6347 // Try and extract from a widened type.
6348 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
6349 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
6350 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
6351 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
6352 }
6353
6354 // Promote operands and see if this is handled by target lowering,
6355 // Otherwise, use the BUILD_VECTOR approach below
6356 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
6357 // Collect the (promoted) operands
6358 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
6359
6360 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
6361 assert(PromEltVT.bitsLE(NOutVTElem) &&
6362 "Promoted operand has an element type greater than result");
6363
6364 EVT ExtVT = NOutVT.changeVectorElementType(*DAG.getContext(), PromEltVT);
6365 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
6366 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
6367 }
6368 }
6369
6370 if (OutVT.isScalableVector())
6371 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
6372
6373 SDValue InOp0 = N->getOperand(0);
6374 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
6375 InOp0 = GetPromotedInteger(InOp0);
6376
6377 EVT InVT = InOp0.getValueType();
6378 EVT InSVT = InVT.getVectorElementType();
6379
6380 unsigned OutNumElems = OutVT.getVectorNumElements();
6382 Ops.reserve(OutNumElems);
6383 for (unsigned i = 0; i != OutNumElems; ++i) {
6384 // Extract the element from the original vector.
6385 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), BaseIdx,
6386 DAG.getConstant(i, dl, BaseIdx.getValueType()));
6387 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InSVT,
6388 N->getOperand(0), Index);
6389 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
6390 // Insert the converted element to the new vector.
6391 Ops.push_back(Op);
6392 }
6393
6394 return DAG.getBuildVector(NOutVT, dl, Ops);
6395}
6396
6397SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
6398 EVT OutVT = N->getValueType(0);
6399 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6400 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6401
6402 SDLoc dl(N);
6403 SDValue Vec = N->getOperand(0);
6404 SDValue SubVec = N->getOperand(1);
6405 SDValue Idx = N->getOperand(2);
6406
6407 EVT SubVecVT = SubVec.getValueType();
6408 EVT NSubVT =
6409 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(),
6410 SubVecVT.getVectorElementCount());
6411
6412 Vec = GetPromotedInteger(Vec);
6413 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);
6414
6415 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
6416}
6417
6418SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
6419 SDLoc dl(N);
6420
6421 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6422 EVT OutVT = V0.getValueType();
6423
6424 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
6425}
6426
6427SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
6428 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
6429 EVT VT = N->getValueType(0);
6430 SDLoc dl(N);
6431
6432 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
6433
6434 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6435 SDValue V1 = GetPromotedInteger(N->getOperand(1));
6436 EVT OutVT = V0.getValueType();
6437
6438 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
6439}
6440
6441SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
6442 EVT OutVT = N->getValueType(0);
6443 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6444 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6445 unsigned NumElems = N->getNumOperands();
6446 EVT NOutVTElem = NOutVT.getVectorElementType();
6447 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT);
6448 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType);
6449 SDLoc dl(N);
6450
6452 Ops.reserve(NumElems);
6453 for (unsigned i = 0; i != NumElems; ++i) {
6454 SDValue Op = N->getOperand(i);
6455 EVT OpVT = Op.getValueType();
6456 // BUILD_VECTOR integer operand types are allowed to be larger than the
6457 // result's element type. This may still be true after the promotion. For
6458 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
6459 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
6460 if (OpVT.bitsLT(NOutVTElem)) {
6461 unsigned ExtOpc = ISD::ANY_EXTEND;
6462 // Attempt to extend constant bool vectors to match target's BooleanContent.
6463 // While not necessary, this improves chances of the constant correctly
6464 // folding with compare results (e.g. for NOT patterns).
6465 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant)
6466 ExtOpc = NOutExtOpc;
6467 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op);
6468 }
6469 Ops.push_back(Op);
6470 }
6471
6472 return DAG.getBuildVector(NOutVT, dl, Ops);
6473}
6474
6475SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) {
6476
6477 SDLoc dl(N);
6478
6479 assert(!N->getOperand(0).getValueType().isVector() &&
6480 "Input must be a scalar");
6481
6482 EVT OutVT = N->getValueType(0);
6483 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6484 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6485 EVT NOutElemVT = NOutVT.getVectorElementType();
6486
6487 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0));
6488 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op);
6489}
6490
6491SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
6492 SDLoc dl(N);
6493 EVT OutVT = N->getValueType(0);
6494 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6495 assert(NOutVT.isScalableVector() &&
6496 "Type must be promoted to a scalable vector type");
6497 const APInt &StepVal = N->getConstantOperandAPInt(0);
6498 return DAG.getStepVector(dl, NOutVT,
6499 StepVal.sext(NOutVT.getScalarSizeInBits()));
6500}
6501
6502SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
6503 SDLoc dl(N);
6504
6505 EVT OutVT = N->getValueType(0);
6506 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6507 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6508
6509 unsigned NumOperands = N->getNumOperands();
6510 unsigned NumOutElem = NOutVT.getVectorMinNumElements();
6511 EVT OutElemTy = NOutVT.getVectorElementType();
6512 if (OutVT.isScalableVector()) {
6513 EVT OpVT = N->getOperand(0).getValueType();
6514 TargetLowering::LegalizeTypeAction OpAction = getTypeAction(OpVT);
6515 assert((OpAction == TargetLowering::TypeLegal ||
6517 OpAction == TargetLowering::TypeWidenVector) &&
6518 "Unhandled legalization type");
6519
6520 EVT ExtendedOpVT =
6521 OpVT.changeVectorElementType(*DAG.getContext(), OutElemTy);
6522
6524 for (unsigned I = 0; I < NumOperands; ++I) {
6525 SDValue Op = N->getOperand(I);
6526 if (OpAction == TargetLowering::TypePromoteInteger)
6527 Op = GetPromotedInteger(Op);
6528 else if (OpAction == TargetLowering::TypeWidenVector)
6529 Op = DAG.getNode(ISD::ANY_EXTEND, dl, ExtendedOpVT, Op);
6530 Ops.push_back(Op);
6531 }
6532
6533 // Do the CONCAT on the legalized operands' element type, then extend
6534 // or truncate to the promoted result type.
6535 EVT ConcatVT = OutVT.changeVectorElementType(
6536 *DAG.getContext(), Ops[0].getValueType().getVectorElementType());
6537 return DAG.getAnyExtOrTrunc(
6538 DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, Ops), dl, NOutVT);
6539 }
6540
6541 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
6542 assert(NumElem * NumOperands == NumOutElem &&
6543 "Unexpected number of elements");
6544
6545 // Take the elements from the first vector.
6546 SmallVector<SDValue, 8> Ops(NumOutElem);
6547 for (unsigned i = 0; i < NumOperands; ++i) {
6548 SDValue Op = N->getOperand(i);
6549 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
6550 Op = GetPromotedInteger(Op);
6551 EVT SclrTy = Op.getValueType().getVectorElementType();
6552 assert(NumElem == Op.getValueType().getVectorNumElements() &&
6553 "Unexpected number of elements");
6554
6555 for (unsigned j = 0; j < NumElem; ++j) {
6556 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
6557 DAG.getVectorIdxConstant(j, dl));
6558 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
6559 }
6560 }
6561
6562 return DAG.getBuildVector(NOutVT, dl, Ops);
6563}
6564
6565SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
6566 EVT VT = N->getValueType(0);
6567 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6568 assert(NVT.isVector() && "This type must be promoted to a vector type");
6569
6570 SDLoc dl(N);
6571
6572 // For operands whose TypeAction is to promote, extend the promoted node
6573 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
6574 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
6575 // type..
6576 if (getTypeAction(N->getOperand(0).getValueType())
6578 SDValue Promoted;
6579
6580 switch(N->getOpcode()) {
6582 Promoted = SExtPromotedInteger(N->getOperand(0));
6583 break;
6585 Promoted = ZExtPromotedInteger(N->getOperand(0));
6586 break;
6588 Promoted = GetPromotedInteger(N->getOperand(0));
6589 break;
6590 default:
6591 llvm_unreachable("Node has unexpected Opcode");
6592 }
6593 unsigned NewSize = NVT.getSizeInBits();
6594 if (Promoted.getValueType().getSizeInBits() > NewSize) {
6595 EVT ExtractVT = EVT::getVectorVT(
6596 *DAG.getContext(), Promoted.getValueType().getVectorElementType(),
6597 NewSize / Promoted.getScalarValueSizeInBits());
6598
6599 Promoted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ExtractVT, Promoted,
6600 DAG.getVectorIdxConstant(0, dl));
6601 }
6602 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
6603 }
6604
6605 // Directly extend to the appropriate transform-to type.
6606 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
6607}
6608
6609SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
6610 EVT VT = N->getValueType(0);
6611 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6612 return DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, SDLoc(N), NVT, N->ops());
6613}
6614
6615SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
6616 EVT VT = N->getValueType(0);
6617 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6618 return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
6619}
6620
6621SDValue DAGTypeLegalizer::PromoteIntRes_MASK_BEFOREFIRST(SDNode *N) {
6622 EVT VT = N->getValueType(0);
6623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6624 SDValue Op = PromoteTargetBoolean(N->getOperand(0), NVT);
6625 return DAG.getNode(ISD::MASK_BEFOREFIRST, SDLoc(N), NVT, Op);
6626}
6627
6628SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_MATCH(SDNode *N) {
6629 EVT VT = N->getValueType(0);
6630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6631 SmallVector<SDValue, 3> NewOps(N->ops());
6632 NewOps[2] = PromoteTargetBoolean(N->getOperand(2), NVT);
6633 return DAG.getNode(ISD::VECTOR_MATCH, SDLoc(N), NVT, NewOps, N->getFlags());
6634}
6635
6636SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
6637 SDLoc DL(N);
6638 EVT VT = N->getValueType(0);
6639 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6640 SDValue ExtAcc = GetPromotedInteger(N->getOperand(0));
6641 return DAG.getNode(N->getOpcode(), DL, NVT, ExtAcc, N->getOperand(1),
6642 N->getOperand(2));
6643}
6644
6645SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
6646 EVT OutVT = N->getValueType(0);
6647 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6648 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6649
6650 EVT NOutVTElem = NOutVT.getVectorElementType();
6651
6652 SDLoc dl(N);
6653 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6654
6655 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
6656 NOutVTElem, N->getOperand(1));
6657 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
6658 V0, ConvElem, N->getOperand(2));
6659}
6660
6661SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
6662 // The VECREDUCE result size may be larger than the element size, so
6663 // we can simply change the result type.
6664 SDLoc dl(N);
6665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6666 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
6667}
6668
6669SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
6670 // The VP_REDUCE result size may be larger than the element size, so we can
6671 // simply change the result type. However the start value and result must be
6672 // the same.
6673 SDLoc DL(N);
6674 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
6675 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
6676 N->getOperand(1), N->getOperand(2), N->getOperand(3));
6677}
6678
6679SDValue DAGTypeLegalizer::PromoteIntRes_PATCHPOINT(SDNode *N) {
6680 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6681 SDLoc dl(N);
6682
6683 assert(N->getNumValues() == 3 && "Expected 3 values for PATCHPOINT");
6684 SDVTList VTList = DAG.getVTList({NVT, MVT::Other, MVT::Glue});
6685
6686 SmallVector<SDValue> Ops(N->ops());
6687 SDValue Res = DAG.getNode(ISD::PATCHPOINT, dl, VTList, Ops);
6688
6689 // Replace chain and glue uses with the new patchpoint.
6690 SDValue From[] = {SDValue(N, 1), SDValue(N, 2)};
6691 SDValue To[] = {Res.getValue(1), Res.getValue(2)};
6692 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
6693
6694 return Res.getValue(0);
6695}
6696
6697SDValue DAGTypeLegalizer::PromoteIntRes_READ_REGISTER(SDNode *N) {
6698 const Function &Fn = DAG.getMachineFunction().getFunction();
6699 Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure(
6700 "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc()));
6701
6702 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6703 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
6704 return DAG.getPOISON(NVT);
6705}
6706
6707SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
6708 SDLoc dl(N);
6709 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6710 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
6711 TLI.getVectorIdxTy(DAG.getDataLayout()));
6713 V0->getValueType(0).getScalarType(), V0, V1);
6714
6715 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
6716 // element types. If this is the case then we need to expand the outgoing
6717 // value and not truncate it.
6718 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
6719}
6720
6721SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
6722 SDLoc dl(N);
6723 // The result type is equal to the first input operand's type, so the
6724 // type that needs promoting must be the second source vector.
6725 SDValue V0 = N->getOperand(0);
6726 SDValue V1 = GetPromotedInteger(N->getOperand(1));
6727 SDValue Idx = N->getOperand(2);
6728 EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
6729 V1.getValueType().getVectorElementType(),
6730 V0.getValueType().getVectorElementCount());
6731 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
6732 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
6733 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
6734}
6735
6736// FIXME: We wouldn't need this if clang could promote short integers
6737// that are arguments to FAKE_USE.
6738SDValue DAGTypeLegalizer::PromoteIntOp_FAKE_USE(SDNode *N) {
6739 SDLoc dl(N);
6740 SDValue V0 = N->getOperand(0);
6741 SDValue V1 = N->getOperand(1);
6742 EVT InVT1 = V1.getValueType();
6743 SDValue VPromoted =
6744 DAG.getNode(ISD::ANY_EXTEND, dl,
6745 TLI.getTypeToTransformTo(*DAG.getContext(), InVT1), V1);
6746 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), V0, VPromoted);
6747}
6748
6749SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
6750 SDLoc dl(N);
6751 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6752 MVT InVT = V0.getValueType().getSimpleVT();
6753 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
6754 N->getValueType(0).getVectorNumElements());
6755 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
6756 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
6757}
6758
6759SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
6760 SDLoc dl(N);
6761
6762 EVT ResVT = N->getValueType(0);
6763 unsigned NumElems = N->getNumOperands();
6764
6765 // New legal concat from promoted operands.
6766 EVT PromotedOpTy = TLI.getTypeToTransformTo(*DAG.getContext(),
6767 N->getOperand(0).getValueType());
6768 EVT WideVT = PromotedOpTy.changeVectorElementCount(
6769 *DAG.getContext(), ResVT.getVectorElementCount());
6770 if (isTypeLegal(WideVT)) {
6771 SmallVector<SDValue> PromotedOps;
6772 for (const SDValue &Op : N->ops())
6773 PromotedOps.push_back(GetPromotedInteger(Op));
6774
6775 SDValue WideConcat =
6776 DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, PromotedOps);
6777 return DAG.getAnyExtOrTrunc(WideConcat, dl, ResVT);
6778 }
6779
6780 if (ResVT.isScalableVector()) {
6781 SDValue ResVec = DAG.getPOISON(ResVT);
6782
6783 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6784 SDValue Op = N->getOperand(OpIdx);
6785 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
6786 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
6787 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
6788 }
6789
6790 return ResVec;
6791 }
6792
6793 SmallVector<SDValue> NewOps;
6794 NewOps.reserve(ResVT.getVectorMinNumElements());
6795 for (const SDValue &Op : N->ops())
6796 DAG.ExtractVectorElements(GetPromotedInteger(Op), NewOps);
6797 return DAG.getBuildVector(ResVT, dl, NewOps);
6798}
6799
6800SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
6801 assert(OpNo > 1);
6802 SDValue Op = N->getOperand(OpNo);
6803
6804 // FIXME: Non-constant operands are not yet handled:
6805 // - https://github.com/llvm/llvm-project/issues/26431
6806 // - https://github.com/llvm/llvm-project/issues/55957
6807 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
6808 if (!CN)
6809 return SDValue();
6810
6811 // Copy operands before the one being expanded.
6812 SmallVector<SDValue> NewOps;
6813 for (unsigned I = 0; I < OpNo; I++)
6814 NewOps.push_back(N->getOperand(I));
6815
6816 EVT Ty = Op.getValueType();
6817 SDLoc DL = SDLoc(N);
6818 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6819 NewOps.push_back(
6820 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6821 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6822 } else {
6823 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6824 return SDValue();
6825 }
6826
6827 // Copy remaining operands.
6828 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6829 NewOps.push_back(N->getOperand(I));
6830
6831 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6832
6833 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6834 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6835
6836 return SDValue(); // Signal that we have replaced the node already.
6837}
6838
6839SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
6840 assert(OpNo >= 7);
6841 SDValue Op = N->getOperand(OpNo);
6842
6843 // FIXME: Non-constant operands are not yet handled:
6844 // - https://github.com/llvm/llvm-project/issues/26431
6845 // - https://github.com/llvm/llvm-project/issues/55957
6846 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
6847 if (!CN)
6848 return SDValue();
6849
6850 // Copy operands before the one being expanded.
6851 SmallVector<SDValue> NewOps;
6852 for (unsigned I = 0; I < OpNo; I++)
6853 NewOps.push_back(N->getOperand(I));
6854
6855 EVT Ty = Op.getValueType();
6856 SDLoc DL = SDLoc(N);
6857 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6858 NewOps.push_back(
6859 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6860 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6861 } else {
6862 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6863 return SDValue();
6864 }
6865
6866 // Copy remaining operands.
6867 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6868 NewOps.push_back(N->getOperand(I));
6869
6870 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6871
6872 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6873 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6874
6875 return SDValue(); // Signal that we have replaced the node already.
6876}
return SDValue()
const TargetLowering & TLI
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
dxil translate DXIL Translate Metadata
static bool isSigned(unsigned Opcode)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define I(x, y, z)
Definition MD5.cpp:57
SmallVector< MachineOperand, 4 > Cond
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
static LLVM_ABI unsigned int semanticsIntSizeInBits(const fltSemantics &, bool)
Definition APFloat.cpp:349
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
unsigned countLeadingOnes() const
Definition APInt.h:1644
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
unsigned countTrailingZeros() const
Definition APInt.h:1667
unsigned countLeadingZeros() const
Definition APInt.h:1626
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
unsigned countTrailingOnes() const
Definition APInt.h:1682
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
constexpr bool isPowerOf2() const
Definition TypeSize.h:324
const Function & getFunction() const
Definition Function.h:167
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
ISD::CondCode getOperandCondCode(unsigned Num) const
Helper method returns the CondCode of a CondCodeSDNode.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
void reserve(size_type N)
void push_back(const T &Elt)
This class is used to represent ISD::STORE nodes.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
BooleanContent
Enum that describes how the target represents true/false values.
std::vector< ArgListEntry > ArgListTy
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
Definition TypeSize.h:266
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS*X will result in a value whose quantity matches our own.
Definition TypeSize.h:274
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Entry
Definition COFF.h:862
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:296
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
Definition ISDOpcodes.h:665
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:942
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ SET_ROUNDING
Set rounding mode.
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ STRICT_UINT_TO_FP
Definition ISDOpcodes.h:490
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition ISDOpcodes.h:859
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition ISDOpcodes.h:707
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ CTLS
Count leading redundant sign bits.
Definition ISDOpcodes.h:824
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ STRICT_FP_TO_FP16
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ STRICT_FP16_TO_FP
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition ISDOpcodes.h:931
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition ISDOpcodes.h:650
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition ISDOpcodes.h:489
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ STRICT_FP_TO_UINT
Definition ISDOpcodes.h:483
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:482
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition ISDOpcodes.h:510
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ STRICT_FP_TO_BF16
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:306
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition ISDOpcodes.h:697
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:715
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:953
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ PARTIAL_REDUCE_SUMLA
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
Definition ISDOpcodes.h:645
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ CTTZ_ELTS_ZERO_POISON
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:341
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUREM(EVT VT)
LLVM_ABI Libcall getSHL(EVT VT)
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSDIV(EVT VT)
LLVM_ABI Libcall getSRL(EVT VT)
LLVM_ABI Libcall getSRA(EVT VT)
LLVM_ABI Libcall getUDIV(EVT VT)
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSREM(EVT VT)
LLVM_ABI Libcall getMUL(EVT VT)
LLVM_ABI AtomicLibcall selectAtomicLibcall(const LibcallLoweringInfo &Libcalls, unsigned Opc, AtomicOrdering Order, MVT VT)
Lower to outline atomic libcall if outline atomics enabled, or to sync libcall otherwise.
LLVM_ABI Libcall getCTPOP(EVT VT)
LLVM_ABI Libcall getMULO(EVT VT)
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
@ Success
The lock was released successfully.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ AfterLegalizeTypes
Definition DAGCombine.h:17
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition ValueTypes.h:450
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
EVT changeVectorElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element coun...
Definition ValueTypes.h:109
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition ValueTypes.h:294
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition ValueTypes.h:346
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:499
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)