LLVM 22.0.0git
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type. For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type. For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//
19
20#include "LegalizeTypes.h"
29#include <algorithm>
30using namespace llvm;
31
32#define DEBUG_TYPE "legalize-types"
33
34//===----------------------------------------------------------------------===//
35// Integer Result Promotion
36//===----------------------------------------------------------------------===//
37
38/// PromoteIntegerResult - This method is called when a result of a node is
39/// found to be in need of promotion to a larger type. At this point, the node
40/// may also have invalid operands or may have other results that need
41/// expansion, we just know that (at least) one result needs promotion.
42void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
43 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
44 SDValue Res = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
48 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
49 return;
50 }
51
52 switch (N->getOpcode()) {
53 default:
54#ifndef NDEBUG
55 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
56 N->dump(&DAG); dbgs() << "\n";
57#endif
58 report_fatal_error("Do not know how to promote this operator!");
59 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
60 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
61 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
62 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
63 case ISD::VP_BITREVERSE:
64 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
65 case ISD::VP_BSWAP:
66 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
67 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
68 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
69 case ISD::VP_CTLZ_ZERO_UNDEF:
70 case ISD::VP_CTLZ:
72 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
73 case ISD::PARITY:
74 case ISD::VP_CTPOP:
75 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
76 case ISD::VP_CTTZ_ZERO_UNDEF:
77 case ISD::VP_CTTZ:
79 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
80 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
81 case ISD::VP_CTTZ_ELTS:
82 Res = PromoteIntRes_VP_CttzElements(N);
83 break;
85 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
86 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
87 case ISD::VP_LOAD:
88 Res = PromoteIntRes_VP_LOAD(cast<VPLoadSDNode>(N));
89 break;
90 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
91 break;
92 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
93 break;
95 Res = PromoteIntRes_VECTOR_COMPRESS(N);
96 break;
97 case ISD::SELECT:
98 case ISD::VSELECT:
99 case ISD::VP_SELECT:
100 case ISD::VP_MERGE:
101 Res = PromoteIntRes_Select(N);
102 break;
103 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
106 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
107 case ISD::SMIN:
108 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
109 case ISD::UMIN:
110 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
111
112 case ISD::SHL:
113 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
115 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
116 case ISD::SRA:
117 case ISD::VP_SRA: Res = PromoteIntRes_SRA(N); break;
118 case ISD::SRL:
119 case ISD::VP_SRL: Res = PromoteIntRes_SRL(N); break;
120 case ISD::VP_TRUNCATE:
121 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
122 case ISD::POISON:
123 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
124 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
125 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
126
128 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
130 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
132 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
134 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
136 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
139 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
140 return;
142 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
144 Res = PromoteIntRes_BUILD_VECTOR(N);
145 break;
148 case ISD::EXPERIMENTAL_VP_SPLAT:
149 Res = PromoteIntRes_ScalarOp(N);
150 break;
151 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
153 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
154
158 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
159
161 Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(N);
162 break;
163
165 Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
166 break;
167
171 Res = PromoteIntRes_PARTIAL_REDUCE_MLA(N);
172 break;
173
174 case ISD::SIGN_EXTEND:
175 case ISD::VP_SIGN_EXTEND:
176 case ISD::ZERO_EXTEND:
177 case ISD::VP_ZERO_EXTEND:
178 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
179
180 case ISD::VP_FP_TO_SINT:
181 case ISD::VP_FP_TO_UINT:
184 case ISD::FP_TO_SINT:
185 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
186
189 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
190
191 case ISD::FP_TO_BF16:
192 case ISD::FP_TO_FP16:
193 Res = PromoteIntRes_FP_TO_FP16_BF16(N);
194 break;
197 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
198 break;
199 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
200
201 case ISD::AND:
202 case ISD::OR:
203 case ISD::XOR:
204 case ISD::ADD:
205 case ISD::SUB:
206 case ISD::MUL:
207 case ISD::VP_AND:
208 case ISD::VP_OR:
209 case ISD::VP_XOR:
210 case ISD::VP_ADD:
211 case ISD::VP_SUB:
212 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
213
214 case ISD::ABDS:
215 case ISD::AVGCEILS:
216 case ISD::AVGFLOORS:
217 case ISD::VP_SMIN:
218 case ISD::VP_SMAX:
219 case ISD::SDIV:
220 case ISD::SREM:
221 case ISD::VP_SDIV:
222 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
223
224 case ISD::ABDU:
225 case ISD::AVGCEILU:
226 case ISD::AVGFLOORU:
227 case ISD::VP_UMIN:
228 case ISD::VP_UMAX:
229 case ISD::UDIV:
230 case ISD::UREM:
231 case ISD::VP_UDIV:
232 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
233
234 case ISD::SADDO:
235 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
236 case ISD::UADDO:
237 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
238 case ISD::SMULO:
239 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
240
241 case ISD::ADDE:
242 case ISD::SUBE:
243 case ISD::UADDO_CARRY:
244 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
245
246 case ISD::SADDO_CARRY:
247 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
248
249 case ISD::SADDSAT:
250 case ISD::UADDSAT:
251 case ISD::SSUBSAT:
252 case ISD::USUBSAT:
253 case ISD::SSHLSAT:
254 case ISD::USHLSAT:
255 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(N);
256 break;
257 case ISD::VP_SADDSAT:
258 case ISD::VP_UADDSAT:
259 case ISD::VP_SSUBSAT:
260 case ISD::VP_USUBSAT:
261 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(N);
262 break;
263
264 case ISD::SCMP:
265 case ISD::UCMP:
266 Res = PromoteIntRes_CMP(N);
267 break;
268
269 case ISD::SMULFIX:
270 case ISD::SMULFIXSAT:
271 case ISD::UMULFIX:
272 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
273
274 case ISD::SDIVFIX:
275 case ISD::SDIVFIXSAT:
276 case ISD::UDIVFIX:
277 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
278
279 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
280
281 case ISD::ATOMIC_LOAD:
282 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
283
295 case ISD::ATOMIC_SWAP:
296 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
297
300 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
301 break;
302
312 Res = PromoteIntRes_VECREDUCE(N);
313 break;
314
315 case ISD::VP_REDUCE_ADD:
316 case ISD::VP_REDUCE_MUL:
317 case ISD::VP_REDUCE_AND:
318 case ISD::VP_REDUCE_OR:
319 case ISD::VP_REDUCE_XOR:
320 case ISD::VP_REDUCE_SMAX:
321 case ISD::VP_REDUCE_SMIN:
322 case ISD::VP_REDUCE_UMAX:
323 case ISD::VP_REDUCE_UMIN:
324 Res = PromoteIntRes_VP_REDUCE(N);
325 break;
326
329 Res = PromoteIntRes_LOOP_DEPENDENCE_MASK(N);
330 break;
331
332 case ISD::FREEZE:
333 Res = PromoteIntRes_FREEZE(N);
334 break;
335
336 case ISD::ROTL:
337 case ISD::ROTR:
338 Res = PromoteIntRes_Rotate(N);
339 break;
340
341 case ISD::FSHL:
342 case ISD::FSHR:
343 Res = PromoteIntRes_FunnelShift(N);
344 break;
345
346 case ISD::VP_FSHL:
347 case ISD::VP_FSHR:
348 Res = PromoteIntRes_VPFunnelShift(N);
349 break;
350
351 case ISD::IS_FPCLASS:
352 Res = PromoteIntRes_IS_FPCLASS(N);
353 break;
354 case ISD::FFREXP:
355 Res = PromoteIntRes_FFREXP(N);
356 break;
357
358 case ISD::LRINT:
359 case ISD::LLRINT:
360 Res = PromoteIntRes_XRINT(N);
361 break;
362
363 case ISD::PATCHPOINT:
364 Res = PromoteIntRes_PATCHPOINT(N);
365 break;
367 Res = PromoteIntRes_READ_REGISTER(N);
368 break;
369 }
370
371 // If the result is null then the sub-method took care of registering it.
372 if (Res.getNode())
373 SetPromotedInteger(SDValue(N, ResNo), Res);
374}
375
376SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
377 unsigned ResNo) {
378 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
379 return GetPromotedInteger(Op);
380}
381
382SDValue DAGTypeLegalizer::PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
383 EVT VT = N->getValueType(0);
384 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
385 return DAG.getNode(N->getOpcode(), SDLoc(N), NewVT, N->ops());
386}
387
388SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
389 // Sign-extend the new bits, and continue the assertion.
390 SDValue Op = SExtPromotedInteger(N->getOperand(0));
391 return DAG.getNode(ISD::AssertSext, SDLoc(N),
392 Op.getValueType(), Op, N->getOperand(1));
393}
394
395SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
396 // Zero the new bits, and continue the assertion.
397 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
398 return DAG.getNode(ISD::AssertZext, SDLoc(N),
399 Op.getValueType(), Op, N->getOperand(1));
400}
401
402SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
403 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
404 ISD::LoadExtType ExtType = N->getExtensionType();
405 if (ExtType == ISD::NON_EXTLOAD) {
406 switch (TLI.getExtendForAtomicOps()) {
407 case ISD::SIGN_EXTEND:
408 ExtType = ISD::SEXTLOAD;
409 break;
410 case ISD::ZERO_EXTEND:
411 ExtType = ISD::ZEXTLOAD;
412 break;
413 case ISD::ANY_EXTEND:
414 ExtType = ISD::EXTLOAD;
415 break;
416 default:
417 llvm_unreachable("Invalid atomic op extension");
418 }
419 }
420
421 SDValue Res =
422 DAG.getAtomicLoad(ExtType, SDLoc(N), N->getMemoryVT(), ResVT,
423 N->getChain(), N->getBasePtr(), N->getMemOperand());
424
425 // Legalize the chain result - switch anything that used the old chain to
426 // use the new one.
427 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
428 return Res;
429}
430
431SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
432 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
433 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
434 N->getMemoryVT(),
435 N->getChain(), N->getBasePtr(),
436 Op2, N->getMemOperand());
437 // Legalize the chain result - switch anything that used the old chain to
438 // use the new one.
439 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
440 return Res;
441}
442
443SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
444 unsigned ResNo) {
445 if (ResNo == 1) {
447 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
448 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
449
450 // Only use the result of getSetCCResultType if it is legal,
451 // otherwise just use the promoted result type (NVT).
452 if (!TLI.isTypeLegal(SVT))
453 SVT = NVT;
454
455 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
456 SDValue Res = DAG.getAtomicCmpSwap(
457 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
458 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
459 N->getMemOperand());
460 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
461 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
462 return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT);
463 }
464
465 // Op2 is used for the comparison and thus must be extended according to the
466 // target's atomic operations. Op3 is merely stored and so can be left alone.
467 SDValue Op2 = N->getOperand(2);
468 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
469 switch (TLI.getExtendForAtomicCmpSwapArg()) {
470 case ISD::SIGN_EXTEND:
471 Op2 = SExtPromotedInteger(Op2);
472 break;
473 case ISD::ZERO_EXTEND:
474 Op2 = ZExtPromotedInteger(Op2);
475 break;
476 case ISD::ANY_EXTEND:
477 Op2 = GetPromotedInteger(Op2);
478 break;
479 default:
480 llvm_unreachable("Invalid atomic op extension");
481 }
482
483 SDVTList VTs =
484 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
485 SDValue Res = DAG.getAtomicCmpSwap(
486 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
487 N->getBasePtr(), Op2, Op3, N->getMemOperand());
488 // Update the use to N with the newly created Res.
489 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
490 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
491 return Res;
492}
493
494SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
495 SDValue InOp = N->getOperand(0);
496 EVT InVT = InOp.getValueType();
497 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
498 EVT OutVT = N->getValueType(0);
499 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
500 SDLoc dl(N);
501
502 switch (getTypeAction(InVT)) {
504 break;
506 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
507 // The input promotes to the same size. Convert the promoted value.
508 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
509 break;
511 // Promote the integer operand by hand.
512 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
514 // Promote the integer operand by hand.
515 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
517 // Convert the promoted float by hand.
518 if (!NOutVT.isVector())
519 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
520 break;
521 }
524 break;
526 // Convert the element to an integer and promote it by hand.
527 if (!NOutVT.isVector())
528 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
529 BitConvertToInteger(GetScalarizedVector(InOp)));
530 break;
532 report_fatal_error("Scalarization of scalable vectors is not supported.");
534 if (!NOutVT.isVector()) {
535 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
536 // pieces of the input into integers and reassemble in the final type.
537 SDValue Lo, Hi;
538 GetSplitVector(N->getOperand(0), Lo, Hi);
539 Lo = BitConvertToInteger(Lo);
540 Hi = BitConvertToInteger(Hi);
541
542 if (DAG.getDataLayout().isBigEndian())
543 std::swap(Lo, Hi);
544
545 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
547 NOutVT.getSizeInBits()),
548 JoinIntegers(Lo, Hi));
549 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
550 }
551 break;
552 }
554 // The input is widened to the same size. Convert to the widened value.
555 // Make sure that the outgoing value is not a vector, because this would
556 // make us bitcast between two vectors which are legalized in different ways.
557 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
558 SDValue Res =
559 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
560
561 // For big endian targets we need to shift the casted value or the
562 // interesting bits will end up at the wrong place.
563 if (DAG.getDataLayout().isBigEndian()) {
564 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
565 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
566 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
567 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
568 }
569 return Res;
570 }
571 // If the output type is also a vector and widening it to the same size
572 // as the widened input type would be a legal type, we can widen the bitcast
573 // and handle the promotion after.
574 if (NOutVT.isVector()) {
575 TypeSize WidenInSize = NInVT.getSizeInBits();
576 TypeSize OutSize = OutVT.getSizeInBits();
577 if (WidenInSize.hasKnownScalarFactor(OutSize)) {
578 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
579 EVT WideOutVT =
581 OutVT.getVectorElementCount() * Scale);
582 if (isTypeLegal(WideOutVT)) {
583 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
584 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
585 DAG.getVectorIdxConstant(0, dl));
586 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
587 }
588 }
589 }
590 }
591
592 // TODO: Handle big endian
593 if (!NOutVT.isVector() && InOp.getValueType().isVector() &&
595 // Pad the vector operand with undef and cast to a wider integer.
596 EVT EltVT = InOp.getValueType().getVectorElementType();
597 TypeSize EltSize = EltVT.getSizeInBits();
598 TypeSize OutSize = NOutVT.getSizeInBits();
599
600 if (OutSize.hasKnownScalarFactor(EltSize)) {
601 unsigned NumEltsWithPadding = OutSize.getKnownScalarFactor(EltSize);
602 EVT WideVecVT =
603 EVT::getVectorVT(*DAG.getContext(), EltVT, NumEltsWithPadding);
604
605 if (isTypeLegal(WideVecVT)) {
606 SDValue Inserted = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT,
607 DAG.getUNDEF(WideVecVT), InOp,
608 DAG.getVectorIdxConstant(0, dl));
609
610 return DAG.getNode(ISD::BITCAST, dl, NOutVT, Inserted);
611 }
612 }
613 }
614
615 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
616 CreateStackStoreLoad(InOp, OutVT));
617}
618
619SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
620 SDValue V = GetPromotedInteger(N->getOperand(0));
621 return DAG.getNode(ISD::FREEZE, SDLoc(N),
622 V.getValueType(), V);
623}
624
625SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
626 SDValue Op = GetPromotedInteger(N->getOperand(0));
627 EVT OVT = N->getValueType(0);
628 EVT NVT = Op.getValueType();
629 SDLoc dl(N);
630
631 // If the larger BSWAP isn't supported by the target, try to expand now.
632 // If we expand later we'll end up with more operations since we lost the
633 // original type. We only do this for scalars since we have a shuffle
634 // based lowering for vectors in LegalizeVectorOps.
635 if (!OVT.isVector() &&
637 if (SDValue Res = TLI.expandBSWAP(N, DAG))
638 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
639 }
640
641 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
642 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
643 if (N->getOpcode() == ISD::BSWAP)
644 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
645 ShAmt);
646 SDValue Mask = N->getOperand(1);
647 SDValue EVL = N->getOperand(2);
648 return DAG.getNode(ISD::VP_SRL, dl, NVT,
649 DAG.getNode(ISD::VP_BSWAP, dl, NVT, Op, Mask, EVL), ShAmt,
650 Mask, EVL);
651}
652
653SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
654 SDValue Op = GetPromotedInteger(N->getOperand(0));
655 EVT OVT = N->getValueType(0);
656 EVT NVT = Op.getValueType();
657 SDLoc dl(N);
658
659 // If the larger BITREVERSE isn't supported by the target, try to expand now.
660 // If we expand later we'll end up with more operations since we lost the
661 // original type. We only do this for scalars since we have a shuffle
662 // based lowering for vectors in LegalizeVectorOps.
663 if (!OVT.isVector() && OVT.isSimple() &&
665 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
666 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
667 }
668
669 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
670 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
671 if (N->getOpcode() == ISD::BITREVERSE)
672 return DAG.getNode(ISD::SRL, dl, NVT,
673 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt);
674 SDValue Mask = N->getOperand(1);
675 SDValue EVL = N->getOperand(2);
676 return DAG.getNode(ISD::VP_SRL, dl, NVT,
677 DAG.getNode(ISD::VP_BITREVERSE, dl, NVT, Op, Mask, EVL),
678 ShAmt, Mask, EVL);
679}
680
681SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
682 // The pair element type may be legal, or may not promote to the same type as
683 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
684 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
686 N->getValueType(0)), JoinIntegers(N->getOperand(0),
687 N->getOperand(1)));
688}
689
690SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
691 EVT VT = N->getValueType(0);
692 // FIXME there is no actual debug info here
693 SDLoc dl(N);
694 // Zero extend things like i1, sign extend everything else. It shouldn't
695 // matter in theory which one we pick, but this tends to give better code?
697 SDValue Result = DAG.getNode(Opc, dl,
698 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
699 SDValue(N, 0));
700 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
701 return Result;
702}
703
704SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
705 EVT OVT = N->getValueType(0);
706 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
707 SDLoc dl(N);
708
709 // If the larger CTLZ isn't supported by the target, try to expand now.
710 // If we expand later we'll end up with more operations since we lost the
711 // original type.
712 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
715 if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
716 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
717 return Result;
718 }
719 }
720
721 unsigned CtlzOpcode = N->getOpcode();
722 if (CtlzOpcode == ISD::CTLZ || CtlzOpcode == ISD::VP_CTLZ) {
723 // Subtract off the extra leading bits in the bigger type.
724 SDValue ExtractLeadingBits = DAG.getConstant(
725 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
726
727 if (!N->isVPOpcode()) {
728 // Zero extend to the promoted type and do the count there.
729 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
730 return DAG.getNode(ISD::SUB, dl, NVT,
731 DAG.getNode(N->getOpcode(), dl, NVT, Op),
732 ExtractLeadingBits);
733 }
734 SDValue Mask = N->getOperand(1);
735 SDValue EVL = N->getOperand(2);
736 // Zero extend to the promoted type and do the count there.
737 SDValue Op = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
738 return DAG.getNode(ISD::VP_SUB, dl, NVT,
739 DAG.getNode(N->getOpcode(), dl, NVT, Op, Mask, EVL),
740 ExtractLeadingBits, Mask, EVL);
741 }
742 if (CtlzOpcode == ISD::CTLZ_ZERO_UNDEF ||
743 CtlzOpcode == ISD::VP_CTLZ_ZERO_UNDEF) {
744 // Any Extend the argument
745 SDValue Op = GetPromotedInteger(N->getOperand(0));
746 // Op = Op << (sizeinbits(NVT) - sizeinbits(Old VT))
747 unsigned SHLAmount = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
748 auto ShiftConst =
749 DAG.getShiftAmountConstant(SHLAmount, Op.getValueType(), dl);
750 if (!N->isVPOpcode()) {
751 Op = DAG.getNode(ISD::SHL, dl, NVT, Op, ShiftConst);
752 return DAG.getNode(CtlzOpcode, dl, NVT, Op);
753 }
754
755 SDValue Mask = N->getOperand(1);
756 SDValue EVL = N->getOperand(2);
757 Op = DAG.getNode(ISD::VP_SHL, dl, NVT, Op, ShiftConst, Mask, EVL);
758 return DAG.getNode(CtlzOpcode, dl, NVT, Op, Mask, EVL);
759 }
760 llvm_unreachable("Invalid CTLZ Opcode");
761}
762
763SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
764 EVT OVT = N->getValueType(0);
765 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
766
767 // If the larger CTPOP isn't supported by the target, try to expand now.
768 // If we expand later we'll end up with more operations since we lost the
769 // original type.
770 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
771 // TargetLowering.
772 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
774 if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
775 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
776 return Result;
777 }
778 }
779
780 // Zero extend to the promoted type and do the count or parity there.
781 if (!N->isVPOpcode()) {
782 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
783 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
784 }
785
786 SDValue Mask = N->getOperand(1);
787 SDValue EVL = N->getOperand(2);
788 SDValue Op = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
789 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op, Mask,
790 EVL);
791}
792
793SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
794 SDValue Op = GetPromotedInteger(N->getOperand(0));
795 EVT OVT = N->getValueType(0);
796 EVT NVT = Op.getValueType();
797 SDLoc dl(N);
798
799 // If the larger CTTZ isn't supported by the target, try to expand now.
800 // If we expand later we'll end up with more operations since we lost the
801 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
802 // larger type.
803 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
806 !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
807 !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
808 if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
809 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
810 return Result;
811 }
812 }
813
814 unsigned NewOpc = N->getOpcode();
815 if (NewOpc == ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
816 // The count is the same in the promoted type except if the original
817 // value was zero. This can be handled by setting the bit just off
818 // the top of the original type.
819 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
820 OVT.getScalarSizeInBits());
821 if (NewOpc == ISD::CTTZ) {
822 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
823 NewOpc = ISD::CTTZ_ZERO_UNDEF;
824 } else {
825 Op =
826 DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT),
827 N->getOperand(1), N->getOperand(2));
828 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
829 }
830 }
831 if (!N->isVPOpcode())
832 return DAG.getNode(NewOpc, dl, NVT, Op);
833 return DAG.getNode(NewOpc, dl, NVT, Op, N->getOperand(1), N->getOperand(2));
834}
835
836SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(SDNode *N) {
837 SDLoc DL(N);
838 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
839 return DAG.getNode(N->getOpcode(), DL, NewVT, N->ops());
840}
841
842SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
843 SDLoc dl(N);
844 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
845
846 SDValue Op0 = N->getOperand(0);
847 SDValue Op1 = N->getOperand(1);
848
849 // If the input also needs to be promoted, do that first so we can get a
850 // get a good idea for the output type.
851 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
853 SDValue In = GetPromotedInteger(Op0);
854
855 // If the new type is larger than NVT, use it. We probably won't need to
856 // promote it again.
857 EVT SVT = In.getValueType().getScalarType();
858 if (SVT.bitsGE(NVT)) {
859 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
860 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
861 }
862 }
863
864 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
865}
866
867SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
868 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
869 unsigned NewOpc =
870 TLI.getPreferredFPToIntOpcode(N->getOpcode(), N->getValueType(0), NVT);
871 SDLoc dl(N);
872
873 SDValue Res;
874 if (N->isStrictFPOpcode()) {
875 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
876 {N->getOperand(0), N->getOperand(1)});
877 // Legalize the chain result - switch anything that used the old chain to
878 // use the new one.
879 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
880 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
881 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1),
882 N->getOperand(2)});
883 } else {
884 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
885 }
886
887 // Assert that the converted value fits in the original type. If it doesn't
888 // (eg: because the value being converted is too big), then the result of the
889 // original operation was undefined anyway, so the assert is still correct.
890 //
891 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
892 // before legalization: fp-to-uint16, 65534. -> 0xfffe
893 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
894 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
895 N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
896 N->getOpcode() == ISD::VP_FP_TO_UINT)
899 dl, NVT, Res,
900 DAG.getValueType(N->getValueType(0).getScalarType()));
901}
902
903SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
904 // Promote the result type, while keeping the original width in Op1.
905 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
906 SDLoc dl(N);
907 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
908 N->getOperand(1));
909}
910
911SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
912 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
913 SDLoc dl(N);
914
915 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
916}
917
918SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
920 SDLoc dl(N);
921
922 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
923 N->getOperand(0), N->getOperand(1));
924 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
925 return Res;
926}
927
928SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
929 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
930 SDLoc dl(N);
931 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
932}
933
934SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
935 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
936 SDLoc dl(N);
937
938 SDValue Res =
939 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
940
941 // Legalize the chain result - switch anything that used the old chain to
942 // use the new one.
943 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
944 return Res;
945}
946
947SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
948 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
949 SDLoc dl(N);
950
951 if (getTypeAction(N->getOperand(0).getValueType())
953 SDValue Res = GetPromotedInteger(N->getOperand(0));
954 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
955
956 // If the result and operand types are the same after promotion, simplify
957 // to an in-register extension. Unless this is a VP_*_EXTEND.
958 if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
959 // The high bits are not guaranteed to be anything. Insert an extend.
960 if (N->getOpcode() == ISD::SIGN_EXTEND)
961 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
962 DAG.getValueType(N->getOperand(0).getValueType()));
963 if (N->getOpcode() == ISD::ZERO_EXTEND)
964 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
965 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
966 return Res;
967 }
968 }
969
970 // Otherwise, just extend the original operand all the way to the larger type.
971 if (N->getNumOperands() != 1) {
972 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
973 assert(N->isVPOpcode() && "Expected VP opcode");
974 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
975 N->getOperand(1), N->getOperand(2));
976 }
977 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
978}
979
980SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
981 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
982 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
983 ISD::LoadExtType ExtType =
984 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
985 SDLoc dl(N);
986 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
987 N->getMemoryVT(), N->getMemOperand());
988
989 // Legalize the chain result - switch anything that used the old chain to
990 // use the new one.
991 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
992 return Res;
993}
994
995SDValue DAGTypeLegalizer::PromoteIntRes_VP_LOAD(VPLoadSDNode *N) {
996 assert(!N->isIndexed() && "Indexed vp_load during type legalization!");
997 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
998 ISD::LoadExtType ExtType = (N->getExtensionType() == ISD::NON_EXTLOAD)
1000 : N->getExtensionType();
1001 SDLoc dl(N);
1002 SDValue Res =
1003 DAG.getExtLoadVP(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
1004 N->getMask(), N->getVectorLength(), N->getMemoryVT(),
1005 N->getMemOperand(), N->isExpandingLoad());
1006 // Legalize the chain result - switch anything that used the old chain to
1007 // use the new one.
1008 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1009 return Res;
1010}
1011
1012SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
1013 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1014 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
1015
1016 ISD::LoadExtType ExtType = N->getExtensionType();
1017 if (ExtType == ISD::NON_EXTLOAD)
1018 ExtType = ISD::EXTLOAD;
1019
1020 SDLoc dl(N);
1021 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
1022 N->getOffset(), N->getMask(), ExtPassThru,
1023 N->getMemoryVT(), N->getMemOperand(),
1024 N->getAddressingMode(), ExtType,
1025 N->isExpandingLoad());
1026 // Legalize the chain result - switch anything that used the old chain to
1027 // use the new one.
1028 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1029 return Res;
1030}
1031
1032SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
1033 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1034 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
1035 assert(NVT == ExtPassThru.getValueType() &&
1036 "Gather result type and the passThru argument type should be the same");
1037
1038 ISD::LoadExtType ExtType = N->getExtensionType();
1039 if (ExtType == ISD::NON_EXTLOAD)
1040 ExtType = ISD::EXTLOAD;
1041
1042 SDLoc dl(N);
1043 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
1044 N->getIndex(), N->getScale() };
1045 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
1046 N->getMemoryVT(), dl, Ops,
1047 N->getMemOperand(), N->getIndexType(),
1048 ExtType);
1049 // Legalize the chain result - switch anything that used the old chain to
1050 // use the new one.
1051 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1052 return Res;
1053}
1054
1055SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(SDNode *N) {
1056 SDValue Vec = GetPromotedInteger(N->getOperand(0));
1057 SDValue Passthru = GetPromotedInteger(N->getOperand(2));
1058 return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), Vec.getValueType(), Vec,
1059 N->getOperand(1), Passthru);
1060}
1061
1062/// Promote the overflow flag of an overflowing arithmetic node.
1063SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
1064 // Change the return type of the boolean result while obeying
1065 // getSetCCResultType.
1066 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1067 EVT VT = N->getValueType(0);
1068 EVT SVT = getSetCCResultType(VT);
1069 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
1070 unsigned NumOps = N->getNumOperands();
1071 assert(NumOps <= 3 && "Too many operands");
1072 if (NumOps == 3)
1073 Ops[2] = PromoteTargetBoolean(N->getOperand(2), VT);
1074
1075 SDLoc dl(N);
1076 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
1077 ArrayRef(Ops, NumOps));
1078
1079 // Modified the sum result - switch anything that used the old sum to use
1080 // the new one.
1081 ReplaceValueWith(SDValue(N, 0), Res);
1082
1083 // Convert to the expected type.
1084 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
1085}
1086
1087template <class MatchContextClass>
1088SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
1089 // If the promoted type is legal, we can convert this to:
1090 // 1. ANY_EXTEND iN to iM
1091 // 2. SHL by M-N
1092 // 3. [US][ADD|SUB|SHL]SAT
1093 // 4. L/ASHR by M-N
1094 // Else it is more efficient to convert this to a min and a max
1095 // operation in the higher precision arithmetic.
1096 SDLoc dl(N);
1097 SDValue Op1 = N->getOperand(0);
1098 SDValue Op2 = N->getOperand(1);
1099 MatchContextClass matcher(DAG, TLI, N);
1100
1101 unsigned Opcode = matcher.getRootBaseOpcode();
1102 unsigned OldBits = Op1.getScalarValueSizeInBits();
1103
1104 // USUBSAT can always be promoted as long as we have zero/sign-extended the
1105 // args.
1106 if (Opcode == ISD::USUBSAT) {
1107 SExtOrZExtPromotedOperands(Op1, Op2);
1108 return matcher.getNode(ISD::USUBSAT, dl, Op1.getValueType(), Op1, Op2);
1109 }
1110
1111 if (Opcode == ISD::UADDSAT) {
1112 EVT OVT = Op1.getValueType();
1113 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1114 // We can promote if we use sign-extend. Do this if the target prefers.
1115 if (TLI.isSExtCheaperThanZExt(OVT, NVT)) {
1116 Op1 = SExtPromotedInteger(Op1);
1117 Op2 = SExtPromotedInteger(Op2);
1118 return matcher.getNode(ISD::UADDSAT, dl, NVT, Op1, Op2);
1119 }
1120
1121 Op1 = ZExtPromotedInteger(Op1);
1122 Op2 = ZExtPromotedInteger(Op2);
1123 unsigned NewBits = NVT.getScalarSizeInBits();
1124 APInt MaxVal = APInt::getLowBitsSet(NewBits, OldBits);
1125 SDValue SatMax = DAG.getConstant(MaxVal, dl, NVT);
1126 SDValue Add = matcher.getNode(ISD::ADD, dl, NVT, Op1, Op2);
1127 return matcher.getNode(ISD::UMIN, dl, NVT, Add, SatMax);
1128 }
1129
1130 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
1131
1132 // FIXME: We need vp-aware PromotedInteger functions.
1133 if (IsShift) {
1134 Op1 = GetPromotedInteger(Op1);
1135 Op2 = ZExtPromotedInteger(Op2);
1136 } else {
1137 Op1 = SExtPromotedInteger(Op1);
1138 Op2 = SExtPromotedInteger(Op2);
1139 }
1140 EVT PromotedType = Op1.getValueType();
1141 unsigned NewBits = PromotedType.getScalarSizeInBits();
1142
1143 // Shift cannot use a min/max expansion, we can't detect overflow if all of
1144 // the bits have been shifted out.
1145 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1146 unsigned ShiftOp;
1147 switch (Opcode) {
1148 case ISD::SADDSAT:
1149 case ISD::SSUBSAT:
1150 case ISD::SSHLSAT:
1151 ShiftOp = ISD::SRA;
1152 break;
1153 case ISD::USHLSAT:
1154 ShiftOp = ISD::SRL;
1155 break;
1156 default:
1157 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
1158 "addition, subtraction or left shift");
1159 }
1160
1161 unsigned SHLAmount = NewBits - OldBits;
1162 SDValue ShiftAmount =
1163 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
1164 Op1 = DAG.getNode(ISD::SHL, dl, PromotedType, Op1, ShiftAmount);
1165 if (!IsShift)
1166 Op2 = matcher.getNode(ISD::SHL, dl, PromotedType, Op2, ShiftAmount);
1167
1168 SDValue Result = matcher.getNode(Opcode, dl, PromotedType, Op1, Op2);
1169 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1170 }
1171
1172 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
1173 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
1174 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
1175 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
1176 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1177 SDValue Result = matcher.getNode(AddOp, dl, PromotedType, Op1, Op2);
1178 Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
1179 Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
1180 return Result;
1181}
1182
1183SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
1184 // Can just promote the operands then continue with operation.
1185 SDLoc dl(N);
1186 SDValue Op1Promoted, Op2Promoted;
1187 bool Signed =
1188 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
1189 bool Saturating =
1190 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
1191 if (Signed) {
1192 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1193 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1194 } else {
1195 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1196 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1197 }
1198 EVT OldType = N->getOperand(0).getValueType();
1199 EVT PromotedType = Op1Promoted.getValueType();
1200 unsigned DiffSize =
1201 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
1202
1203 if (Saturating) {
1204 // Promoting the operand and result values changes the saturation width,
1205 // which is extends the values that we clamp to on saturation. This could be
1206 // resolved by shifting one of the operands the same amount, which would
1207 // also shift the result we compare against, then shifting back.
1208 Op1Promoted =
1209 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1210 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1211 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1212 Op2Promoted, N->getOperand(2));
1213 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
1214 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
1215 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1216 }
1217 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1218 N->getOperand(2));
1219}
1220
1222 unsigned SatW, bool Signed,
1223 const TargetLowering &TLI,
1224 SelectionDAG &DAG) {
1225 EVT VT = V.getValueType();
1226 unsigned VTW = VT.getScalarSizeInBits();
1227
1228 if (!Signed) {
1229 // Saturate to the unsigned maximum by getting the minimum of V and the
1230 // maximum.
1231 return DAG.getNode(ISD::UMIN, dl, VT, V,
1232 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
1233 dl, VT));
1234 }
1235
1236 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
1237 // signed minimum of it and V.
1238 V = DAG.getNode(ISD::SMIN, dl, VT, V,
1239 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
1240 dl, VT));
1241 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
1242 // signed maximum of it and V.
1243 V = DAG.getNode(ISD::SMAX, dl, VT, V,
1244 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
1245 dl, VT));
1246 return V;
1247}
1248
1250 unsigned Scale, const TargetLowering &TLI,
1251 SelectionDAG &DAG, unsigned SatW = 0) {
1252 EVT VT = LHS.getValueType();
1253 unsigned VTSize = VT.getScalarSizeInBits();
1254 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1255 N->getOpcode() == ISD::SDIVFIXSAT;
1256 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1257 N->getOpcode() == ISD::UDIVFIXSAT;
1258
1259 SDLoc dl(N);
1260 // Widen the types by a factor of two. This is guaranteed to expand, since it
1261 // will always have enough high bits in the LHS to shift into.
1262 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
1263 if (VT.isVector())
1264 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
1266 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT);
1267 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT);
1268 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
1269 DAG);
1270 assert(Res && "Expanding DIVFIX with wide type failed?");
1271 if (Saturating) {
1272 // If the caller has told us to saturate at something less, use that width
1273 // instead of the type before doubling. However, it cannot be more than
1274 // what we just widened!
1275 assert(SatW <= VTSize &&
1276 "Tried to saturate to more than the original type?");
1277 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1278 TLI, DAG);
1279 }
1280 return DAG.getZExtOrTrunc(Res, dl, VT);
1281}
1282
1283SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1284 SDLoc dl(N);
1285 SDValue Op1Promoted, Op2Promoted;
1286 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1287 N->getOpcode() == ISD::SDIVFIXSAT;
1288 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1289 N->getOpcode() == ISD::UDIVFIXSAT;
1290 if (Signed) {
1291 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1292 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1293 } else {
1294 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1295 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1296 }
1297 EVT PromotedType = Op1Promoted.getValueType();
1298 unsigned Scale = N->getConstantOperandVal(2);
1299
1300 // If the type is already legal and the operation is legal in that type, we
1301 // should not early expand.
1302 if (TLI.isTypeLegal(PromotedType)) {
1304 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1305 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1306 unsigned Diff = PromotedType.getScalarSizeInBits() -
1307 N->getValueType(0).getScalarSizeInBits();
1308 if (Saturating)
1309 Op1Promoted =
1310 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1311 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1312 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1313 Op2Promoted, N->getOperand(2));
1314 if (Saturating)
1315 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1316 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1317 return Res;
1318 }
1319 }
1320
1321 // See if we can perform the division in this type without expanding.
1322 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1323 Op2Promoted, Scale, DAG)) {
1324 if (Saturating)
1325 Res = SaturateWidenedDIVFIX(Res, dl,
1326 N->getValueType(0).getScalarSizeInBits(),
1327 Signed, TLI, DAG);
1328 return Res;
1329 }
1330 // If we cannot, expand it to twice the type width. If we are saturating, give
1331 // it the original width as a saturating width so we don't need to emit
1332 // two saturations.
1333 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1334 N->getValueType(0).getScalarSizeInBits());
1335}
1336
1337SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1338 if (ResNo == 1)
1339 return PromoteIntRes_Overflow(N);
1340
1341 // The operation overflowed iff the result in the larger type is not the
1342 // sign extension of its truncation to the original type.
1343 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1344 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1345 EVT OVT = N->getOperand(0).getValueType();
1346 EVT NVT = LHS.getValueType();
1347 SDLoc dl(N);
1348
1349 // Do the arithmetic in the larger type.
1350 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1351 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1352
1353 // Calculate the overflow flag: sign extend the arithmetic result from
1354 // the original type.
1355 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1356 DAG.getValueType(OVT));
1357 // Overflowed if and only if this is not equal to Res.
1358 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1359
1360 // Use the calculated overflow everywhere.
1361 ReplaceValueWith(SDValue(N, 1), Ofl);
1362
1363 return Res;
1364}
1365
1366SDValue DAGTypeLegalizer::PromoteIntRes_CMP(SDNode *N) {
1367 EVT PromotedResultTy =
1368 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1369 return DAG.getNode(N->getOpcode(), SDLoc(N), PromotedResultTy,
1370 N->getOperand(0), N->getOperand(1));
1371}
1372
1373SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1374 SDValue Mask = N->getOperand(0);
1375
1376 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1377 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1378
1379 unsigned Opcode = N->getOpcode();
1380 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1381 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
1382 N->getOperand(3));
1383 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
1384}
1385
1386SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1387 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1388 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1389 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1390 LHS.getValueType(), N->getOperand(0),
1391 N->getOperand(1), LHS, RHS, N->getOperand(4));
1392}
1393
1394SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1395 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1396 EVT InVT = N->getOperand(OpNo).getValueType();
1397 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1398
1399 EVT SVT = getSetCCResultType(InVT);
1400
1401 // If we got back a type that needs to be promoted, this likely means the
1402 // the input type also needs to be promoted. So get the promoted type for
1403 // the input and try the query again.
1404 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1405 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1406 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1407 SVT = getSetCCResultType(InVT);
1408 } else {
1409 // Input type isn't promoted, just use the default promoted type.
1410 SVT = NVT;
1411 }
1412 }
1413
1414 SDLoc dl(N);
1415 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1416 "Vector compare must return a vector result!");
1417
1418 // Get the SETCC result using the canonical SETCC type.
1419 SDValue SetCC;
1420 if (N->isStrictFPOpcode()) {
1421 SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
1422 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1423 N->getOperand(2), N->getOperand(3)};
1424 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
1425 // Legalize the chain result - switch anything that used the old chain to
1426 // use the new one.
1427 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1428 } else
1429 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1430 N->getOperand(1), N->getOperand(2), N->getFlags());
1431
1432 // Convert to the expected type.
1433 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1434}
1435
1436SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1437 SDLoc DL(N);
1438 SDValue Arg = N->getOperand(0);
1439 SDValue Test = N->getOperand(1);
1440 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1441 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
1442}
1443
1444SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
1445 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1446 EVT VT = N->getValueType(0);
1447
1448 SDLoc dl(N);
1449 SDValue Res =
1450 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0));
1451
1452 ReplaceValueWith(SDValue(N, 0), Res);
1453 return Res.getValue(1);
1454}
1455
1456SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1457 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1458 SDValue RHS = N->getOperand(1);
1459 if (N->getOpcode() != ISD::VP_SHL) {
1460 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1461 RHS = ZExtPromotedInteger(RHS);
1462
1463 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1464 }
1465
1466 SDValue Mask = N->getOperand(2);
1467 SDValue EVL = N->getOperand(3);
1468 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1469 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1470 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1471 Mask, EVL);
1472}
1473
1474SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1475 SDValue Op = GetPromotedInteger(N->getOperand(0));
1477 Op.getValueType(), Op, N->getOperand(1));
1478}
1479
1480SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1481 // The input may have strange things in the top bits of the registers, but
1482 // these operations don't care. They may have weird bits going out, but
1483 // that too is okay if they are integer operations.
1484 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1485 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1486 if (N->getNumOperands() == 2)
1487 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1488 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1489 assert(N->isVPOpcode() && "Expected VP opcode");
1490 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1491 N->getOperand(2), N->getOperand(3));
1492}
1493
1494SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1495 if (N->getNumOperands() == 2) {
1496 // Sign extend the input.
1497 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1498 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1499 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1500 }
1501 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1502 assert(N->isVPOpcode() && "Expected VP opcode");
1503 SDValue Mask = N->getOperand(2);
1504 SDValue EVL = N->getOperand(3);
1505 // Sign extend the input.
1506 SDValue LHS = VPSExtPromotedInteger(N->getOperand(0), Mask, EVL);
1507 SDValue RHS = VPSExtPromotedInteger(N->getOperand(1), Mask, EVL);
1508 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1509 Mask, EVL);
1510}
1511
1512SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1513 if (N->getNumOperands() == 2) {
1514 // Zero extend the input.
1515 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1516 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1517 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1518 }
1519 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1520 assert(N->isVPOpcode() && "Expected VP opcode");
1521 // Zero extend the input.
1522 SDValue Mask = N->getOperand(2);
1523 SDValue EVL = N->getOperand(3);
1524 SDValue LHS = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
1525 SDValue RHS = VPZExtPromotedInteger(N->getOperand(1), Mask, EVL);
1526 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1527 Mask, EVL);
1528}
1529
1530SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1531 SDValue LHS = N->getOperand(0);
1532 SDValue RHS = N->getOperand(1);
1533
1534 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1535 // whatever is best for the target and the promoted operands.
1536 SExtOrZExtPromotedOperands(LHS, RHS);
1537
1538 return DAG.getNode(N->getOpcode(), SDLoc(N),
1539 LHS.getValueType(), LHS, RHS);
1540}
1541
1542SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1543 SDValue RHS = N->getOperand(1);
1544 if (N->getOpcode() != ISD::VP_SRA) {
1545 // The input value must be properly sign extended.
1546 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1547 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1548 RHS = ZExtPromotedInteger(RHS);
1549 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1550 }
1551
1552 SDValue Mask = N->getOperand(2);
1553 SDValue EVL = N->getOperand(3);
1554 // The input value must be properly sign extended.
1555 SDValue LHS = VPSExtPromotedInteger(N->getOperand(0), Mask, EVL);
1556 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1557 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1558 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1559 Mask, EVL);
1560}
1561
1562SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1563 SDValue RHS = N->getOperand(1);
1564 if (N->getOpcode() != ISD::VP_SRL) {
1565 // The input value must be properly zero extended.
1566 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1567 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1568 RHS = ZExtPromotedInteger(RHS);
1569 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1570 }
1571
1572 SDValue Mask = N->getOperand(2);
1573 SDValue EVL = N->getOperand(3);
1574 // The input value must be properly zero extended.
1575 SDValue LHS = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
1576 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1577 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1578 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1579 Mask, EVL);
1580}
1581
1582SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1583 // Lower the rotate to shifts and ORs which can be promoted.
1584 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
1585 ReplaceValueWith(SDValue(N, 0), Res);
1586 return SDValue();
1587}
1588
1589SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1590 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1591 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1592 SDValue Amt = N->getOperand(2);
1593 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1594 Amt = ZExtPromotedInteger(Amt);
1595 EVT AmtVT = Amt.getValueType();
1596
1597 SDLoc DL(N);
1598 EVT OldVT = N->getOperand(0).getValueType();
1599 EVT VT = Lo.getValueType();
1600 unsigned Opcode = N->getOpcode();
1601 bool IsFSHR = Opcode == ISD::FSHR;
1602 unsigned OldBits = OldVT.getScalarSizeInBits();
1603 unsigned NewBits = VT.getScalarSizeInBits();
1604
1605 // Amount has to be interpreted modulo the old bit width.
1606 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1607 DAG.getConstant(OldBits, DL, AmtVT));
1608
1609 // If the promoted type is twice the size (or more), then we use the
1610 // traditional funnel 'double' shift codegen. This isn't necessary if the
1611 // shift amount is constant.
1612 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1613 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1614 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1615 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1616 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1617 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1618 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1619 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1620 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1621 if (!IsFSHR)
1622 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1623 return Res;
1624 }
1625
1626 // Shift Lo up to occupy the upper bits of the promoted type.
1627 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1628 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1629
1630 // Increase Amount to shift the result into the lower bits of the promoted
1631 // type.
1632 if (IsFSHR)
1633 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset);
1634
1635 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1636}
1637
1638// A vp version of PromoteIntRes_FunnelShift.
1639SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) {
1640 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1641 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1642 SDValue Amt = N->getOperand(2);
1643 SDValue Mask = N->getOperand(3);
1644 SDValue EVL = N->getOperand(4);
1645 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1646 Amt = VPZExtPromotedInteger(Amt, Mask, EVL);
1647 EVT AmtVT = Amt.getValueType();
1648
1649 SDLoc DL(N);
1650 EVT OldVT = N->getOperand(0).getValueType();
1651 EVT VT = Lo.getValueType();
1652 unsigned Opcode = N->getOpcode();
1653 bool IsFSHR = Opcode == ISD::VP_FSHR;
1654 unsigned OldBits = OldVT.getScalarSizeInBits();
1655 unsigned NewBits = VT.getScalarSizeInBits();
1656
1657 // Amount has to be interpreted modulo the old bit width.
1658 Amt = DAG.getNode(ISD::VP_UREM, DL, AmtVT, Amt,
1659 DAG.getConstant(OldBits, DL, AmtVT), Mask, EVL);
1660
1661 // If the promoted type is twice the size (or more), then we use the
1662 // traditional funnel 'double' shift codegen. This isn't necessary if the
1663 // shift amount is constant.
1664 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1665 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1666 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1667 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1668 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1669 Hi = DAG.getNode(ISD::VP_SHL, DL, VT, Hi, HiShift, Mask, EVL);
1670 Lo = DAG.getVPZeroExtendInReg(Lo, Mask, EVL, DL, OldVT);
1671 SDValue Res = DAG.getNode(ISD::VP_OR, DL, VT, Hi, Lo, Mask, EVL);
1672 Res = DAG.getNode(IsFSHR ? ISD::VP_SRL : ISD::VP_SHL, DL, VT, Res, Amt,
1673 Mask, EVL);
1674 if (!IsFSHR)
1675 Res = DAG.getNode(ISD::VP_SRL, DL, VT, Res, HiShift, Mask, EVL);
1676 return Res;
1677 }
1678
1679 // Shift Lo up to occupy the upper bits of the promoted type.
1680 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1681 Lo = DAG.getNode(ISD::VP_SHL, DL, VT, Lo, ShiftOffset, Mask, EVL);
1682
1683 // Increase Amount to shift the result into the lower bits of the promoted
1684 // type.
1685 if (IsFSHR)
1686 Amt = DAG.getNode(ISD::VP_ADD, DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1687
1688 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt, Mask, EVL);
1689}
1690
1691SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1692 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1693 SDValue Res;
1694 SDValue InOp = N->getOperand(0);
1695 SDLoc dl(N);
1696
1697 switch (getTypeAction(InOp.getValueType())) {
1698 default: llvm_unreachable("Unknown type action!");
1701 Res = InOp;
1702 break;
1704 Res = GetPromotedInteger(InOp);
1705 break;
1707 EVT InVT = InOp.getValueType();
1708 assert(InVT.isVector() && "Cannot split scalar types");
1709 ElementCount NumElts = InVT.getVectorElementCount();
1710 assert(NumElts == NVT.getVectorElementCount() &&
1711 "Dst and Src must have the same number of elements");
1713 "Promoted vector type must be a power of two");
1714
1715 SDValue EOp1, EOp2;
1716 GetSplitVector(InOp, EOp1, EOp2);
1717
1718 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1719 NumElts.divideCoefficientBy(2));
1720 if (N->getOpcode() == ISD::TRUNCATE) {
1721 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1722 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1723 } else {
1724 assert(N->getOpcode() == ISD::VP_TRUNCATE &&
1725 "Expected VP_TRUNCATE opcode");
1726 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1727 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
1728 std::tie(EVLLo, EVLHi) =
1729 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
1730 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1731 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1732 }
1733 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1734 }
1735 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
1736 // targets.
1738 SDValue WideInOp = GetWidenedVector(InOp);
1739
1740 // Truncate widened InOp.
1741 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1742 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1743 N->getValueType(0).getScalarType(), NumElem);
1744 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1745
1746 // Zero extend so that the elements are of same type as those of NVT
1748 NumElem);
1749 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1750
1751 // Extract the low NVT subvector.
1752 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1753 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1754 }
1755 }
1756
1757 // Truncate to NVT instead of VT
1758 if (N->getOpcode() == ISD::VP_TRUNCATE)
1759 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1),
1760 N->getOperand(2));
1761 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1762}
1763
1764SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1765 if (ResNo == 1)
1766 return PromoteIntRes_Overflow(N);
1767
1768 // The operation overflowed iff the result in the larger type is not the
1769 // zero extension of its truncation to the original type.
1770 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1771 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1772 EVT OVT = N->getOperand(0).getValueType();
1773 EVT NVT = LHS.getValueType();
1774 SDLoc dl(N);
1775
1776 // Do the arithmetic in the larger type.
1777 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1778 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1779
1780 // Calculate the overflow flag: zero extend the arithmetic result from
1781 // the original type.
1782 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1783 // Overflowed if and only if this is not equal to Res.
1784 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1785
1786 // Use the calculated overflow everywhere.
1787 ReplaceValueWith(SDValue(N, 1), Ofl);
1788
1789 return Res;
1790}
1791
1792// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
1793// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1794// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
1795SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
1796 unsigned ResNo) {
1797 if (ResNo == 1)
1798 return PromoteIntRes_Overflow(N);
1799
1800 // We need to sign-extend the operands so the carry value computed by the
1801 // wide operation will be equivalent to the carry value computed by the
1802 // narrow operation.
1803 // An UADDO_CARRY can generate carry only if any of the operands has its
1804 // most significant bit set. Sign extension propagates the most significant
1805 // bit into the higher bits which means the extra bit that the narrow
1806 // addition would need (i.e. the carry) will be propagated through the higher
1807 // bits of the wide addition.
1808 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
1809 // be preserved by sign extension.
1810 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1811 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1812
1813 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1814
1815 // Do the arithmetic in the wide type.
1816 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1817 LHS, RHS, N->getOperand(2));
1818
1819 // Update the users of the original carry/borrow value.
1820 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1821
1822 return SDValue(Res.getNode(), 0);
1823}
1824
1825SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1826 unsigned ResNo) {
1827 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1828 return PromoteIntRes_Overflow(N);
1829}
1830
1831SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1832 EVT OVT = N->getValueType(0);
1833 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1834
1835 // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1836 // If we expand later we'll end up sign extending more than just the sra input
1837 // in sra+xor+sub expansion.
1838 if (!OVT.isVector() &&
1840 !TLI.isOperationLegal(ISD::SMAX, NVT)) {
1841 if (SDValue Res = TLI.expandABS(N, DAG))
1842 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
1843 }
1844
1845 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1846 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1847}
1848
1849SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1850 // Promote the overflow bit trivially.
1851 if (ResNo == 1)
1852 return PromoteIntRes_Overflow(N);
1853
1854 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1855 SDLoc DL(N);
1856 EVT SmallVT = LHS.getValueType();
1857
1858 // To determine if the result overflowed in a larger type, we extend the
1859 // input to the larger type, do the multiply (checking if it overflows),
1860 // then also check the high bits of the result to see if overflow happened
1861 // there.
1862 if (N->getOpcode() == ISD::SMULO) {
1863 LHS = SExtPromotedInteger(LHS);
1864 RHS = SExtPromotedInteger(RHS);
1865 } else {
1866 LHS = ZExtPromotedInteger(LHS);
1867 RHS = ZExtPromotedInteger(RHS);
1868 }
1869 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1870 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1871
1872 // Overflow occurred if it occurred in the larger type, or if the high part
1873 // of the result does not zero/sign-extend the low part. Check this second
1874 // possibility first.
1875 SDValue Overflow;
1876 if (N->getOpcode() == ISD::UMULO) {
1877 // Unsigned overflow occurred if the high part is non-zero.
1878 unsigned Shift = SmallVT.getScalarSizeInBits();
1879 SDValue Hi =
1880 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1881 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
1882 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1883 DAG.getConstant(0, DL, Hi.getValueType()),
1884 ISD::SETNE);
1885 } else {
1886 // Signed overflow occurred if the high part does not sign extend the low.
1887 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1888 Mul, DAG.getValueType(SmallVT));
1889 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1890 }
1891
1892 // The only other way for overflow to occur is if the multiplication in the
1893 // larger type itself overflowed.
1894 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1895 SDValue(Mul.getNode(), 1));
1896
1897 // Use the calculated overflow everywhere.
1898 ReplaceValueWith(SDValue(N, 1), Overflow);
1899 return Mul;
1900}
1901
1902SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1903 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1904 N->getValueType(0)));
1905}
1906
1907SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1908 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1909
1910 const APInt &MulImm = N->getConstantOperandAPInt(0);
1911 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
1912}
1913
1914SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1915 SDValue Chain = N->getOperand(0); // Get the chain.
1916 SDValue Ptr = N->getOperand(1); // Get the pointer.
1917 EVT VT = N->getValueType(0);
1918 SDLoc dl(N);
1919
1920 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1921 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1922 // The argument is passed as NumRegs registers of type RegVT.
1923
1924 SmallVector<SDValue, 8> Parts(NumRegs);
1925 for (unsigned i = 0; i < NumRegs; ++i) {
1926 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1927 N->getConstantOperandVal(3));
1928 Chain = Parts[i].getValue(1);
1929 }
1930
1931 // Handle endianness of the load.
1932 if (DAG.getDataLayout().isBigEndian())
1933 std::reverse(Parts.begin(), Parts.end());
1934
1935 // Assemble the parts in the promoted type.
1936 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1937 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1938 for (unsigned i = 1; i < NumRegs; ++i) {
1939 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1940 // Shift it to the right position and "or" it in.
1941 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1942 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1943 TLI.getPointerTy(DAG.getDataLayout())));
1944 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1945 }
1946
1947 // Modified the chain result - switch anything that used the old chain to
1948 // use the new one.
1949 ReplaceValueWith(SDValue(N, 1), Chain);
1950
1951 return Res;
1952}
1953
1954//===----------------------------------------------------------------------===//
1955// Integer Operand Promotion
1956//===----------------------------------------------------------------------===//
1957
1958/// PromoteIntegerOperand - This method is called when the specified operand of
1959/// the specified node is found to need promotion. At this point, all of the
1960/// result types of the node are known to be legal, but other operands of the
1961/// node may need promotion or expansion as well as the specified one.
1962bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1963 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
1964 SDValue Res = SDValue();
1965 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1966 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1967 return false;
1968 }
1969
1970 switch (N->getOpcode()) {
1971 default:
1972 #ifndef NDEBUG
1973 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1974 N->dump(&DAG); dbgs() << "\n";
1975 #endif
1976 report_fatal_error("Do not know how to promote this operator's operand!");
1977
1978 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1979 case ISD::ATOMIC_STORE:
1980 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1981 break;
1982 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1983 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1984 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1985 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1986 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1987 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1988 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1989 case ISD::FAKE_USE:
1990 Res = PromoteIntOp_FAKE_USE(N);
1991 break;
1993 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
1994 break;
1995 case ISD::SPLAT_VECTOR:
1997 case ISD::EXPERIMENTAL_VP_SPLAT:
1998 Res = PromoteIntOp_ScalarOp(N);
1999 break;
2000 case ISD::VSELECT:
2001 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
2002 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
2003 case ISD::VP_SETCC:
2004 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
2005 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
2006 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break;
2007 case ISD::VP_SINT_TO_FP:
2008 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
2009 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
2010 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
2011 OpNo); break;
2012 case ISD::VP_STORE:
2013 Res = PromoteIntOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2014 break;
2015 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
2016 OpNo); break;
2017 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
2018 OpNo); break;
2019 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
2020 OpNo); break;
2021 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
2022 OpNo); break;
2024 Res = PromoteIntOp_VECTOR_COMPRESS(N, OpNo);
2025 break;
2026 case ISD::VP_TRUNCATE:
2027 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
2028 case ISD::BF16_TO_FP:
2029 case ISD::FP16_TO_FP:
2030 case ISD::VP_UINT_TO_FP:
2031 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
2033 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
2034 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
2035 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break;
2036 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
2037 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
2038
2039 case ISD::SHL:
2040 case ISD::SRA:
2041 case ISD::SRL:
2042 case ISD::ROTL:
2043 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
2044
2045 case ISD::SCMP:
2046 case ISD::UCMP: Res = PromoteIntOp_CMP(N); break;
2047
2048 case ISD::FSHL:
2049 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
2050
2051 case ISD::FRAMEADDR:
2052 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
2053
2054 case ISD::SMULFIX:
2055 case ISD::SMULFIXSAT:
2056 case ISD::UMULFIX:
2057 case ISD::UMULFIXSAT:
2058 case ISD::SDIVFIX:
2059 case ISD::SDIVFIXSAT:
2060 case ISD::UDIVFIX:
2061 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
2062 case ISD::FPOWI:
2063 case ISD::STRICT_FPOWI:
2064 case ISD::FLDEXP:
2065 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
2066 case ISD::VECREDUCE_ADD:
2067 case ISD::VECREDUCE_MUL:
2068 case ISD::VECREDUCE_AND:
2069 case ISD::VECREDUCE_OR:
2070 case ISD::VECREDUCE_XOR:
2074 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
2075 case ISD::VP_REDUCE_ADD:
2076 case ISD::VP_REDUCE_MUL:
2077 case ISD::VP_REDUCE_AND:
2078 case ISD::VP_REDUCE_OR:
2079 case ISD::VP_REDUCE_XOR:
2080 case ISD::VP_REDUCE_SMAX:
2081 case ISD::VP_REDUCE_SMIN:
2082 case ISD::VP_REDUCE_UMAX:
2083 case ISD::VP_REDUCE_UMIN:
2084 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
2085 break;
2086
2087 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
2088 case ISD::STACKMAP:
2089 Res = PromoteIntOp_STACKMAP(N, OpNo);
2090 break;
2091 case ISD::PATCHPOINT:
2092 Res = PromoteIntOp_PATCHPOINT(N, OpNo);
2093 break;
2095 Res = PromoteIntOp_WRITE_REGISTER(N, OpNo);
2096 break;
2097 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2098 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2099 Res = PromoteIntOp_VP_STRIDED(N, OpNo);
2100 break;
2101 case ISD::EXPERIMENTAL_VP_SPLICE:
2102 Res = PromoteIntOp_VP_SPLICE(N, OpNo);
2103 break;
2105 Res = PromoteIntOp_VECTOR_HISTOGRAM(N, OpNo);
2106 break;
2108 Res = PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(N, OpNo);
2109 break;
2111 Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N);
2112 break;
2116 Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N);
2117 break;
2118 }
2119
2120 // If the result is null, the sub-method took care of registering results etc.
2121 if (!Res.getNode()) return false;
2122
2123 // If the result is N, the sub-method updated N in place. Tell the legalizer
2124 // core about this.
2125 if (Res.getNode() == N)
2126 return true;
2127
2128 const bool IsStrictFp = N->isStrictFPOpcode();
2129 assert(Res.getValueType() == N->getValueType(0) &&
2130 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2131 "Invalid operand expansion");
2132 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
2133 Res.dump());
2134
2135 ReplaceValueWith(SDValue(N, 0), Res);
2136 if (IsStrictFp)
2137 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2138
2139 return false;
2140}
2141
2142// These operands can be either sign extended or zero extended as long as we
2143// treat them the same. If an extension is free, choose that. Otherwise, follow
2144// target preference.
2145void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) {
2146 SDValue OpL = GetPromotedInteger(LHS);
2147 SDValue OpR = GetPromotedInteger(RHS);
2148
2149 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
2150 // The target would prefer to promote the comparison operand with sign
2151 // extension. Honor that unless the promoted values are already zero
2152 // extended.
2153 unsigned OpLEffectiveBits =
2155 unsigned OpREffectiveBits =
2157 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
2158 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
2159 LHS = OpL;
2160 RHS = OpR;
2161 return;
2162 }
2163
2164 // The promoted values aren't zero extended, use a sext_inreg.
2165 LHS = SExtPromotedInteger(LHS);
2166 RHS = SExtPromotedInteger(RHS);
2167 return;
2168 }
2169
2170 // Prefer to promote the comparison operand with zero extension.
2171
2172 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
2173 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
2174 // that we might not be able to remove.
2175 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
2176 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
2177 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
2178 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
2179 LHS = OpL;
2180 RHS = OpR;
2181 return;
2182 }
2183
2184 // Otherwise, use zext_inreg.
2185 LHS = ZExtPromotedInteger(LHS);
2186 RHS = ZExtPromotedInteger(RHS);
2187}
2188
2189/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
2190/// shared among BR_CC, SELECT_CC, and SETCC handlers.
2191void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
2192 ISD::CondCode CCCode) {
2193 // We have to insert explicit sign or zero extends. Note that we could
2194 // insert sign extends for ALL conditions. For those operations where either
2195 // zero or sign extension would be valid, we ask the target which extension
2196 // it would prefer.
2197
2198 // Signed comparisons always require sign extension.
2199 if (ISD::isSignedIntSetCC(CCCode)) {
2200 LHS = SExtPromotedInteger(LHS);
2201 RHS = SExtPromotedInteger(RHS);
2202 return;
2203 }
2204
2206 "Unknown integer comparison!");
2207
2208 SExtOrZExtPromotedOperands(LHS, RHS);
2209}
2210
2211SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
2212 SDValue Op = GetPromotedInteger(N->getOperand(0));
2213 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
2214}
2215
2216SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
2217 SDValue Op1 = GetPromotedInteger(N->getOperand(1));
2218 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
2219 N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
2220}
2221
2222SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
2223 EVT OutVT = N->getValueType(0);
2224 SDValue InOp = N->getOperand(0);
2225 EVT InVT = InOp.getValueType();
2226 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
2227 SDLoc dl(N);
2228
2229 switch (getTypeAction(InVT)) {
2231 // TODO: Handle big endian & vector input type.
2232 if (OutVT.isVector() && !InVT.isVector() &&
2233 DAG.getDataLayout().isLittleEndian()) {
2234 EVT EltVT = OutVT.getVectorElementType();
2235 TypeSize EltSize = EltVT.getSizeInBits();
2236 TypeSize NInSize = NInVT.getSizeInBits();
2237
2238 if (NInSize.hasKnownScalarFactor(EltSize)) {
2239 unsigned NumEltsWithPadding = NInSize.getKnownScalarFactor(EltSize);
2240 EVT WideVecVT =
2241 EVT::getVectorVT(*DAG.getContext(), EltVT, NumEltsWithPadding);
2242
2243 if (isTypeLegal(WideVecVT)) {
2244 SDValue Promoted = GetPromotedInteger(InOp);
2245 SDValue Cast = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Promoted);
2246 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, Cast,
2247 DAG.getVectorIdxConstant(0, dl));
2248 }
2249 }
2250 }
2251
2252 break;
2253 }
2254 default:
2255 break;
2256 }
2257
2258 // This should only occur in unusual situations like bitcasting to an
2259 // x86_fp80, so just turn it into a store+load
2260 return CreateStackStoreLoad(InOp, OutVT);
2261}
2262
2263SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
2264 assert(OpNo == 2 && "Don't know how to promote this operand!");
2265
2266 SDValue LHS = N->getOperand(2);
2267 SDValue RHS = N->getOperand(3);
2268 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
2269
2270 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
2271 // legal types.
2272 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2273 N->getOperand(1), LHS, RHS, N->getOperand(4)),
2274 0);
2275}
2276
2277SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
2278 assert(OpNo == 1 && "only know how to promote condition");
2279
2280 // Promote all the way up to the canonical SetCC type.
2281 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
2282
2283 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
2284 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
2285 N->getOperand(2)), 0);
2286}
2287
2288SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
2289 // Since the result type is legal, the operands must promote to it.
2290 EVT OVT = N->getOperand(0).getValueType();
2291 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
2292 SDValue Hi = GetPromotedInteger(N->getOperand(1));
2293 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
2294 SDLoc dl(N);
2295
2296 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
2297 DAG.getConstant(OVT.getSizeInBits(), dl,
2298 TLI.getPointerTy(DAG.getDataLayout())));
2299 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
2300}
2301
2302SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
2303 // The vector type is legal but the element type is not. This implies
2304 // that the vector is a power-of-two in length and that the element
2305 // type does not have a strange size (eg: it is not i1).
2306 EVT VecVT = N->getValueType(0);
2307 unsigned NumElts = VecVT.getVectorNumElements();
2308 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2309 "Legal vector of one illegal element?");
2310
2311 // Promote the inserted value. The type does not need to match the
2312 // vector element type. Check that any extra bits introduced will be
2313 // truncated away.
2314 assert(N->getOperand(0).getValueSizeInBits() >=
2315 N->getValueType(0).getScalarSizeInBits() &&
2316 "Type of inserted value narrower than vector element type!");
2317
2319 for (unsigned i = 0; i < NumElts; ++i)
2320 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
2321
2322 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2323}
2324
2325SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
2326 unsigned OpNo) {
2327 if (OpNo == 1) {
2328 // Promote the inserted value. This is valid because the type does not
2329 // have to match the vector element type.
2330
2331 // Check that any extra bits introduced will be truncated away.
2332 assert(N->getOperand(1).getValueSizeInBits() >=
2333 N->getValueType(0).getScalarSizeInBits() &&
2334 "Type of inserted value narrower than vector element type!");
2335 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2336 GetPromotedInteger(N->getOperand(1)),
2337 N->getOperand(2)),
2338 0);
2339 }
2340
2341 assert(OpNo == 2 && "Different operand and result vector types?");
2342
2343 // Promote the index.
2344 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
2345 TLI.getVectorIdxTy(DAG.getDataLayout()));
2346 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2347 N->getOperand(1), Idx), 0);
2348}
2349
2350SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2351 SDValue Op = GetPromotedInteger(N->getOperand(0));
2352 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_SPLAT)
2353 return SDValue(
2354 DAG.UpdateNodeOperands(N, Op, N->getOperand(1), N->getOperand(2)), 0);
2355
2356 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
2357 // so just promote the operand in place.
2358 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2359}
2360
2361SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
2362 assert(OpNo == 0 && "Only know how to promote the condition!");
2363 SDValue Cond = N->getOperand(0);
2364 EVT OpTy = N->getOperand(1).getValueType();
2365
2366 if (N->getOpcode() == ISD::VSELECT)
2367 if (SDValue Res = WidenVSELECTMask(N))
2368 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2369 Res, N->getOperand(1), N->getOperand(2));
2370
2371 // Promote all the way up to the canonical SetCC type.
2372 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
2373 Cond = PromoteTargetBoolean(Cond, OpVT);
2374
2375 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
2376 N->getOperand(2)), 0);
2377}
2378
2379SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2380 assert(OpNo == 0 && "Don't know how to promote this operand!");
2381
2382 SDValue LHS = N->getOperand(0);
2383 SDValue RHS = N->getOperand(1);
2384 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
2385
2386 // The CC (#4) and the possible return values (#2 and #3) have legal types.
2387 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2388 N->getOperand(3), N->getOperand(4)), 0);
2389}
2390
2391SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
2392 assert(OpNo == 0 && "Don't know how to promote this operand!");
2393
2394 SDValue LHS = N->getOperand(0);
2395 SDValue RHS = N->getOperand(1);
2396 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
2397
2398 // The CC (#2) is always legal.
2399 if (N->getOpcode() == ISD::SETCC)
2400 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
2401
2402 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2403
2404 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2405 N->getOperand(3), N->getOperand(4)),
2406 0);
2407}
2408
2409SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
2410 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2411 ZExtPromotedInteger(N->getOperand(1))), 0);
2412}
2413
2414SDValue DAGTypeLegalizer::PromoteIntOp_CMP(SDNode *N) {
2415 SDValue LHS = N->getOperand(0);
2416 SDValue RHS = N->getOperand(1);
2417
2418 if (N->getOpcode() == ISD::SCMP) {
2419 LHS = SExtPromotedInteger(LHS);
2420 RHS = SExtPromotedInteger(RHS);
2421 } else {
2422 SExtOrZExtPromotedOperands(LHS, RHS);
2423 }
2424
2425 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS), 0);
2426}
2427
2428SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
2429 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2430 ZExtPromotedInteger(N->getOperand(2))), 0);
2431}
2432
2433SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
2434 SDValue Op = GetPromotedInteger(N->getOperand(0));
2435 SDLoc dl(N);
2436 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2437 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
2438 Op, DAG.getValueType(N->getOperand(0).getValueType()));
2439}
2440
2441SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) {
2442 SDLoc dl(N);
2443 EVT VT = N->getValueType(0);
2444 SDValue Op = GetPromotedInteger(N->getOperand(0));
2445 // FIXME: There is no VP_ANY_EXTEND yet.
2446 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2447 N->getOperand(2));
2448 unsigned Diff =
2449 VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits();
2450 SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl);
2451 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts.
2452 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1),
2453 N->getOperand(2));
2454 return DAG.getNode(ISD::VP_SRA, dl, VT, Shl, ShAmt, N->getOperand(1),
2455 N->getOperand(2));
2456}
2457
2458SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2459 if (N->getOpcode() == ISD::VP_SINT_TO_FP)
2460 return SDValue(DAG.UpdateNodeOperands(N,
2461 SExtPromotedInteger(N->getOperand(0)),
2462 N->getOperand(1), N->getOperand(2)),
2463 0);
2464 return SDValue(DAG.UpdateNodeOperands(N,
2465 SExtPromotedInteger(N->getOperand(0))), 0);
2466}
2467
2468SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2469 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2470 SExtPromotedInteger(N->getOperand(1))), 0);
2471}
2472
2473SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2474 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2475 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2476 SDLoc dl(N);
2477
2478 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
2479
2480 // Truncate the value and store the result.
2481 return DAG.getTruncStore(Ch, dl, Val, Ptr,
2482 N->getMemoryVT(), N->getMemOperand());
2483}
2484
2485SDValue DAGTypeLegalizer::PromoteIntOp_VP_STORE(VPStoreSDNode *N,
2486 unsigned OpNo) {
2487
2488 assert(OpNo == 1 && "Unexpected operand for promotion");
2489 assert(!N->isIndexed() && "expecting unindexed vp_store!");
2490
2491 SDValue DataOp = GetPromotedInteger(N->getValue());
2492 return DAG.getTruncStoreVP(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2493 N->getMask(), N->getVectorLength(),
2494 N->getMemoryVT(), N->getMemOperand(),
2495 N->isCompressingStore());
2496}
2497
2498SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2499 unsigned OpNo) {
2500 SDValue DataOp = N->getValue();
2501 SDValue Mask = N->getMask();
2502
2503 if (OpNo == 4) {
2504 // The Mask. Update in place.
2505 EVT DataVT = DataOp.getValueType();
2506 Mask = PromoteTargetBoolean(Mask, DataVT);
2507 SmallVector<SDValue, 4> NewOps(N->ops());
2508 NewOps[4] = Mask;
2509 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2510 }
2511
2512 assert(OpNo == 1 && "Unexpected operand for promotion");
2513 DataOp = GetPromotedInteger(DataOp);
2514
2515 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2516 N->getOffset(), Mask, N->getMemoryVT(),
2517 N->getMemOperand(), N->getAddressingMode(),
2518 /*IsTruncating*/ true, N->isCompressingStore());
2519}
2520
2521SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2522 unsigned OpNo) {
2523 assert(OpNo == 3 && "Only know how to promote the mask!");
2524 EVT DataVT = N->getValueType(0);
2525 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2526 SmallVector<SDValue, 4> NewOps(N->ops());
2527 NewOps[OpNo] = Mask;
2528 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2529 if (Res == N)
2530 return SDValue(Res, 0);
2531
2532 // Update triggered CSE, do our own replacement since caller can't.
2533 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2534 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2535 return SDValue();
2536}
2537
2538SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2539 unsigned OpNo) {
2540 SmallVector<SDValue, 5> NewOps(N->ops());
2541
2542 if (OpNo == 2) {
2543 // The Mask
2544 EVT DataVT = N->getValueType(0);
2545 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2546 } else if (OpNo == 4) {
2547 // The Index
2548 if (N->isIndexSigned())
2549 // Need to sign extend the index since the bits will likely be used.
2550 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2551 else
2552 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2553 } else
2554 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2555
2556 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2557 if (Res == N)
2558 return SDValue(Res, 0);
2559
2560 // Update triggered CSE, do our own replacement since caller can't.
2561 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2562 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2563 return SDValue();
2564}
2565
2566SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2567 unsigned OpNo) {
2568 bool TruncateStore = N->isTruncatingStore();
2569 SmallVector<SDValue, 5> NewOps(N->ops());
2570
2571 if (OpNo == 2) {
2572 // The Mask
2573 EVT DataVT = N->getValue().getValueType();
2574 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2575 } else if (OpNo == 4) {
2576 // The Index
2577 if (N->isIndexSigned())
2578 // Need to sign extend the index since the bits will likely be used.
2579 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2580 else
2581 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2582 } else {
2583 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2584 TruncateStore = true;
2585 }
2586
2587 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2588 SDLoc(N), NewOps, N->getMemOperand(),
2589 N->getIndexType(), TruncateStore);
2590}
2591
2592SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_COMPRESS(SDNode *N,
2593 unsigned OpNo) {
2594 assert(OpNo == 1 && "Can only promote VECTOR_COMPRESS mask.");
2595 SDValue Vec = N->getOperand(0);
2596 EVT VT = Vec.getValueType();
2597 SDValue Passthru = N->getOperand(2);
2598 SDValue Mask = PromoteTargetBoolean(N->getOperand(1), VT);
2599 return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), VT, Vec, Mask, Passthru);
2600}
2601
2602SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2603 SDValue Op = GetPromotedInteger(N->getOperand(0));
2604 if (N->getOpcode() == ISD::VP_TRUNCATE)
2605 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op,
2606 N->getOperand(1), N->getOperand(2));
2607 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2608}
2609
2610SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2611 if (N->getOpcode() == ISD::VP_UINT_TO_FP)
2612 return SDValue(DAG.UpdateNodeOperands(N,
2613 ZExtPromotedInteger(N->getOperand(0)),
2614 N->getOperand(1), N->getOperand(2)),
2615 0);
2616 return SDValue(DAG.UpdateNodeOperands(N,
2617 ZExtPromotedInteger(N->getOperand(0))), 0);
2618}
2619
2620SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2621 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2622 ZExtPromotedInteger(N->getOperand(1))), 0);
2623}
2624
2625SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2626 SDLoc dl(N);
2627 SDValue Src = N->getOperand(0);
2628 SDValue Op = GetPromotedInteger(Src);
2629 EVT VT = N->getValueType(0);
2630
2631 // If this zext has the nneg flag and the target prefers sext, see if the
2632 // promoted input is already sign extended.
2633 // TODO: Should we have some way to set nneg on ISD::AND instead?
2634 if (N->getFlags().hasNonNeg() && Op.getValueType() == VT &&
2635 TLI.isSExtCheaperThanZExt(Src.getValueType(), VT)) {
2636 unsigned OpEffectiveBits = DAG.ComputeMaxSignificantBits(Op);
2637 if (OpEffectiveBits <= Src.getScalarValueSizeInBits())
2638 return Op;
2639 }
2640
2641 Op = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op);
2642 return DAG.getZeroExtendInReg(Op, dl, Src.getValueType());
2643}
2644
2645SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) {
2646 SDLoc dl(N);
2647 EVT VT = N->getValueType(0);
2648 SDValue Op = GetPromotedInteger(N->getOperand(0));
2649 // FIXME: There is no VP_ANY_EXTEND yet.
2650 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2651 N->getOperand(2));
2652 return DAG.getVPZeroExtendInReg(Op, N->getOperand(1), N->getOperand(2), dl,
2653 N->getOperand(0).getValueType());
2654}
2655
2656SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2657 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2658 return SDValue(
2659 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2660}
2661
2662SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2663 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2664 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2665 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2666}
2667
2668SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
2669 bool IsStrict = N->isStrictFPOpcode();
2670 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2671
2672 bool IsPowI =
2673 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
2674 unsigned OpOffset = IsStrict ? 1 : 0;
2675
2676 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result
2677 // and floating point operand is already type legalized).
2678 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
2679 : RTLIB::getLDEXP(N->getValueType(0));
2680
2681 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2682 // Scalarize vector FPOWI instead of promoting the type. This allows the
2683 // scalar FPOWIs to be visited and converted to libcalls before promoting
2684 // the type.
2685 // FIXME: This should be done in LegalizeVectorOps/LegalizeDAG, but call
2686 // lowering needs the unpromoted EVT.
2687 if (IsPowI && N->getValueType(0).isVector())
2688 return DAG.UnrollVectorOp(N);
2689 SmallVector<SDValue, 3> NewOps(N->ops());
2690 NewOps[1 + OpOffset] = SExtPromotedInteger(N->getOperand(1 + OpOffset));
2691 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2692 }
2693
2694 // We can't just promote the exponent type in FPOWI, since we want to lower
2695 // the node to a libcall and we if we promote to a type larger than
2696 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2697 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2698 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2699
2700 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2701 assert(DAG.getLibInfo().getIntSize() ==
2702 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2703 "POWI exponent should match with sizeof(int) when doing the libcall.");
2705 CallOptions.setIsSigned(true);
2706 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2707 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2708 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
2709 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2710 if (IsStrict)
2711 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2712 return SDValue();
2713}
2714
2716 switch (N->getOpcode()) {
2717 default:
2718 llvm_unreachable("Expected integer vector reduction");
2719 case ISD::VECREDUCE_ADD:
2720 case ISD::VECREDUCE_MUL:
2721 case ISD::VECREDUCE_AND:
2722 case ISD::VECREDUCE_OR:
2723 case ISD::VECREDUCE_XOR:
2724 case ISD::VP_REDUCE_ADD:
2725 case ISD::VP_REDUCE_MUL:
2726 case ISD::VP_REDUCE_AND:
2727 case ISD::VP_REDUCE_OR:
2728 case ISD::VP_REDUCE_XOR:
2729 return ISD::ANY_EXTEND;
2732 case ISD::VP_REDUCE_SMAX:
2733 case ISD::VP_REDUCE_SMIN:
2734 return ISD::SIGN_EXTEND;
2737 case ISD::VP_REDUCE_UMAX:
2738 case ISD::VP_REDUCE_UMIN:
2739 return ISD::ZERO_EXTEND;
2740 }
2741}
2742
2743SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2744 switch (getExtendForIntVecReduction(N)) {
2745 default:
2746 llvm_unreachable("Impossible extension kind for integer reduction");
2747 case ISD::ANY_EXTEND:
2748 return GetPromotedInteger(V);
2749 case ISD::SIGN_EXTEND:
2750 return SExtPromotedInteger(V);
2751 case ISD::ZERO_EXTEND:
2752 return ZExtPromotedInteger(V);
2753 }
2754}
2755
2756SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2757 SDLoc dl(N);
2758 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2759
2760 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
2761 EVT InVT = Op.getValueType();
2762 EVT EltVT = InVT.getVectorElementType();
2763 EVT ResVT = N->getValueType(0);
2764 unsigned Opcode = N->getOpcode();
2765
2766 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2767 // vecreduce_xor is not legal
2768 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2771 Opcode = ISD::VECREDUCE_ADD;
2772
2773 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2774 // vecreduce_or is not legal
2775 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2778 Opcode = ISD::VECREDUCE_UMAX;
2779 // Can't use promoteTargetBoolean here because we still need
2780 // to either sign_ext or zero_ext in the undefined case.
2781 switch (TLI.getBooleanContents(InVT)) {
2784 Op = ZExtPromotedInteger(N->getOperand(0));
2785 break;
2787 Op = SExtPromotedInteger(N->getOperand(0));
2788 break;
2789 }
2790 }
2791
2792 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2793 // vecreduce_and is not legal
2794 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2797 Opcode = ISD::VECREDUCE_UMIN;
2798 // Can't use promoteTargetBoolean here because we still need
2799 // to either sign_ext or zero_ext in the undefined case.
2800 switch (TLI.getBooleanContents(InVT)) {
2803 Op = ZExtPromotedInteger(N->getOperand(0));
2804 break;
2806 Op = SExtPromotedInteger(N->getOperand(0));
2807 break;
2808 }
2809 }
2810
2811 if (ResVT.bitsGE(EltVT))
2812 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
2813
2814 // Result size must be >= element size. If this is not the case after
2815 // promotion, also promote the result type and then truncate.
2816 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
2817 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
2818}
2819
2820SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2821 SDLoc DL(N);
2822 SDValue Op = N->getOperand(OpNo);
2823 SmallVector<SDValue, 4> NewOps(N->ops());
2824
2825 if (OpNo == 2) { // Mask
2826 // Update in place.
2827 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2828 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2829 }
2830
2831 assert(OpNo == 1 && "Unexpected operand for promotion");
2832
2833 Op = PromoteIntOpVectorReduction(N, Op);
2834
2835 NewOps[OpNo] = Op;
2836
2837 EVT VT = N->getValueType(0);
2838 EVT EltVT = Op.getValueType().getScalarType();
2839
2840 if (VT.bitsGE(EltVT))
2841 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2842
2843 // Result size must be >= element/start-value size. If this is not the case
2844 // after promotion, also promote both the start value and result type and
2845 // then truncate.
2846 NewOps[0] =
2847 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2848 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2849 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2850}
2851
2852SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2853 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2854 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2855}
2856
2857SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
2858 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
2859 SmallVector<SDValue> NewOps(N->ops());
2860 SDValue Operand = N->getOperand(OpNo);
2861 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2862 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2863 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2864}
2865
2866SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
2867 assert(OpNo >= 7);
2868 SmallVector<SDValue> NewOps(N->ops());
2869 SDValue Operand = N->getOperand(OpNo);
2870 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2871 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2872 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2873}
2874
2875SDValue DAGTypeLegalizer::PromoteIntOp_WRITE_REGISTER(SDNode *N,
2876 unsigned OpNo) {
2877 const Function &Fn = DAG.getMachineFunction().getFunction();
2879 "cannot use llvm.write_register with illegal type", Fn,
2880 N->getDebugLoc()));
2881 return N->getOperand(0);
2882}
2883
2884SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
2885 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2886 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2887
2888 SmallVector<SDValue, 8> NewOps(N->ops());
2889 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2890
2891 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2892}
2893
2894SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) {
2895 SmallVector<SDValue, 6> NewOps(N->ops());
2896
2897 if (OpNo == 2) { // Offset operand
2898 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2899 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2900 }
2901
2902 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion");
2903
2904 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2905 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2906}
2907
2908SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N,
2909 unsigned OpNo) {
2910 assert(OpNo == 1 && "Unexpected operand for promotion");
2911 SmallVector<SDValue, 7> NewOps(N->ops());
2912 NewOps[1] = GetPromotedInteger(N->getOperand(1));
2913 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2914}
2915
2916SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N,
2917 unsigned OpNo) {
2918 SmallVector<SDValue, 1> NewOps(N->ops());
2919 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2920 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2921}
2922
2923SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
2924 SmallVector<SDValue, 1> NewOps(N->ops());
2925 NewOps[0] = ZExtPromotedInteger(N->getOperand(0));
2926 NewOps[1] = ZExtPromotedInteger(N->getOperand(1));
2927 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2928}
2929
2930SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N) {
2931 SmallVector<SDValue, 1> NewOps(N->ops());
2932 switch (N->getOpcode()) {
2934 NewOps[1] = SExtPromotedInteger(N->getOperand(1));
2935 NewOps[2] = SExtPromotedInteger(N->getOperand(2));
2936 break;
2938 NewOps[1] = ZExtPromotedInteger(N->getOperand(1));
2939 NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
2940 break;
2942 NewOps[1] = SExtPromotedInteger(N->getOperand(1));
2943 NewOps[2] = ZExtPromotedInteger(N->getOperand(2));
2944 break;
2945 default:
2946 llvm_unreachable("unexpected opcode");
2947 }
2948 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2949}
2950
2951//===----------------------------------------------------------------------===//
2952// Integer Result Expansion
2953//===----------------------------------------------------------------------===//
2954
2955/// ExpandIntegerResult - This method is called when the specified result of the
2956/// specified node is found to need expansion. At this point, the node may also
2957/// have invalid operands or may have other results that need promotion, we just
2958/// know that (at least) one result needs expansion.
2959void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2960 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG));
2961 SDValue Lo, Hi;
2962 Lo = Hi = SDValue();
2963
2964 // See if the target wants to custom expand this node.
2965 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2966 return;
2967
2968 switch (N->getOpcode()) {
2969 default:
2970#ifndef NDEBUG
2971 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2972 N->dump(&DAG); dbgs() << "\n";
2973#endif
2974 report_fatal_error("Do not know how to expand the result of this "
2975 "operator!");
2976
2977 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
2978 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2979 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
2980 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2981 case ISD::POISON:
2982 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2983 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2984 case ISD::SETCC: ExpandIntRes_SETCC(N, Lo, Hi); break;
2985
2986 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2987 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2988 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2989 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2990 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2991
2992 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2993 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2994 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2995 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2996 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2997 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2998 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2999 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
3000 case ISD::ABDS:
3001 case ISD::ABDU: ExpandIntRes_ABD(N, Lo, Hi); break;
3003 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
3004 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
3006 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
3007 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break;
3009 case ISD::FP_TO_SINT:
3011 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break;
3013 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
3014 case ISD::STRICT_LROUND:
3015 case ISD::STRICT_LRINT:
3016 case ISD::LROUND:
3017 case ISD::LRINT:
3019 case ISD::STRICT_LLRINT:
3020 case ISD::LLROUND:
3021 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break;
3022 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
3023 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
3025 case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break;
3026 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
3027 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
3028 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
3029 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
3030 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
3031 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
3032 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
3033 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
3034 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
3035
3047 case ISD::ATOMIC_SWAP:
3048 case ISD::ATOMIC_CMP_SWAP: {
3049 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
3050 SplitInteger(Tmp.first, Lo, Hi);
3051 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3052 break;
3053 }
3055 AtomicSDNode *AN = cast<AtomicSDNode>(N);
3056 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
3057 SDValue Tmp = DAG.getAtomicCmpSwap(
3059 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
3060 AN->getMemOperand());
3061
3062 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
3063 // success simply by comparing the loaded value against the ingoing
3064 // comparison.
3065 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
3066 N->getOperand(2), ISD::SETEQ);
3067
3068 SplitInteger(Tmp, Lo, Hi);
3069 ReplaceValueWith(SDValue(N, 1), Success);
3070 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
3071 break;
3072 }
3073
3074 case ISD::AND:
3075 case ISD::OR:
3076 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
3077
3078 case ISD::UMAX:
3079 case ISD::SMAX:
3080 case ISD::UMIN:
3081 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
3082
3083 case ISD::SCMP:
3084 case ISD::UCMP: ExpandIntRes_CMP(N, Lo, Hi); break;
3085
3086 case ISD::ADD:
3087 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
3088
3089 case ISD::ADDC:
3090 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
3091
3092 case ISD::ADDE:
3093 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
3094
3095 case ISD::UADDO_CARRY:
3096 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
3097
3098 case ISD::SADDO_CARRY:
3099 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
3100
3101 case ISD::SHL:
3102 case ISD::SRA:
3103 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
3104
3105 case ISD::SADDO:
3106 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
3107 case ISD::UADDO:
3108 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
3109 case ISD::UMULO:
3110 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
3111
3112 case ISD::SADDSAT:
3113 case ISD::UADDSAT:
3114 case ISD::SSUBSAT:
3115 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
3116
3117 case ISD::SSHLSAT:
3118 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
3119
3120 case ISD::AVGCEILS:
3121 case ISD::AVGCEILU:
3122 case ISD::AVGFLOORS:
3123 case ISD::AVGFLOORU: ExpandIntRes_AVG(N, Lo, Hi); break;
3124
3125 case ISD::SMULFIX:
3126 case ISD::SMULFIXSAT:
3127 case ISD::UMULFIX:
3128 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
3129
3130 case ISD::SDIVFIX:
3131 case ISD::SDIVFIXSAT:
3132 case ISD::UDIVFIX:
3133 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
3134
3135 case ISD::VECREDUCE_ADD:
3136 case ISD::VECREDUCE_MUL:
3137 case ISD::VECREDUCE_AND:
3138 case ISD::VECREDUCE_OR:
3139 case ISD::VECREDUCE_XOR:
3143 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
3144
3145 case ISD::ROTL:
3146 case ISD::ROTR:
3147 ExpandIntRes_Rotate(N, Lo, Hi);
3148 break;
3149
3150 case ISD::FSHL:
3151 case ISD::FSHR:
3152 ExpandIntRes_FunnelShift(N, Lo, Hi);
3153 break;
3154
3155 case ISD::VSCALE:
3156 ExpandIntRes_VSCALE(N, Lo, Hi);
3157 break;
3158
3159 case ISD::READ_REGISTER:
3160 ExpandIntRes_READ_REGISTER(N, Lo, Hi);
3161 break;
3162 }
3163
3164 // If Lo/Hi is null, the sub-method took care of registering results etc.
3165 if (Lo.getNode())
3166 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
3167}
3168
3169/// Lower an atomic node to the appropriate builtin call.
3170std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
3171 unsigned Opc = Node->getOpcode();
3172 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3173 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3174 // Lower to outline atomic libcall if outline atomics enabled,
3175 // or to sync libcall otherwise
3176 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
3177 EVT RetVT = Node->getValueType(0);
3180 if (TLI.getLibcallName(LC)) {
3181 Ops.append(Node->op_begin() + 2, Node->op_end());
3182 Ops.push_back(Node->getOperand(1));
3183 } else {
3184 LC = RTLIB::getSYNC(Opc, VT);
3185 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3186 "Unexpected atomic op or value type!");
3187 Ops.append(Node->op_begin() + 1, Node->op_end());
3188 }
3189 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
3190 Node->getOperand(0));
3191}
3192
3193/// N is a shift by a value that needs to be expanded,
3194/// and the shift amount is a constant 'Amt'. Expand the operation.
3195void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
3196 SDValue &Lo, SDValue &Hi) {
3197 SDLoc DL(N);
3198 // Expand the incoming operand to be shifted, so that we have its parts
3199 SDValue InL, InH;
3200 GetExpandedInteger(N->getOperand(0), InL, InH);
3201
3202 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
3203 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
3204 if (!Amt) {
3205 Lo = InL;
3206 Hi = InH;
3207 return;
3208 }
3209
3210 EVT NVT = InL.getValueType();
3211 unsigned VTBits = N->getValueType(0).getSizeInBits();
3212 unsigned NVTBits = NVT.getSizeInBits();
3213
3214 if (N->getOpcode() == ISD::SHL) {
3215 if (Amt.uge(VTBits)) {
3216 Lo = Hi = DAG.getConstant(0, DL, NVT);
3217 } else if (Amt.ugt(NVTBits)) {
3218 Lo = DAG.getConstant(0, DL, NVT);
3219 Hi = DAG.getNode(ISD::SHL, DL, NVT, InL,
3220 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
3221 } else if (Amt == NVTBits) {
3222 Lo = DAG.getConstant(0, DL, NVT);
3223 Hi = InL;
3224 } else {
3225 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL,
3226 DAG.getShiftAmountConstant(Amt, NVT, DL));
3227 Hi = DAG.getNode(
3228 ISD::OR, DL, NVT,
3229 DAG.getNode(ISD::SHL, DL, NVT, InH,
3230 DAG.getShiftAmountConstant(Amt, NVT, DL)),
3231 DAG.getNode(ISD::SRL, DL, NVT, InL,
3232 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
3233 }
3234 return;
3235 }
3236
3237 if (N->getOpcode() == ISD::SRL) {
3238 if (Amt.uge(VTBits)) {
3239 Lo = Hi = DAG.getConstant(0, DL, NVT);
3240 } else if (Amt.ugt(NVTBits)) {
3241 Lo = DAG.getNode(ISD::SRL, DL, NVT, InH,
3242 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
3243 Hi = DAG.getConstant(0, DL, NVT);
3244 } else if (Amt == NVTBits) {
3245 Lo = InH;
3246 Hi = DAG.getConstant(0, DL, NVT);
3247 } else {
3248 Lo = DAG.getNode(
3249 ISD::OR, DL, NVT,
3250 DAG.getNode(ISD::SRL, DL, NVT, InL,
3251 DAG.getShiftAmountConstant(Amt, NVT, DL)),
3252 DAG.getNode(ISD::SHL, DL, NVT, InH,
3253 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
3254 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH,
3255 DAG.getShiftAmountConstant(Amt, NVT, DL));
3256 }
3257 return;
3258 }
3259
3260 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3261 if (Amt.uge(VTBits)) {
3262 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
3263 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
3264 } else if (Amt.ugt(NVTBits)) {
3265 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
3266 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
3267 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
3268 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
3269 } else if (Amt == NVTBits) {
3270 Lo = InH;
3271 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
3272 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
3273 } else {
3274 Lo = DAG.getNode(
3275 ISD::OR, DL, NVT,
3276 DAG.getNode(ISD::SRL, DL, NVT, InL,
3277 DAG.getShiftAmountConstant(Amt, NVT, DL)),
3278 DAG.getNode(ISD::SHL, DL, NVT, InH,
3279 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
3280 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
3281 DAG.getShiftAmountConstant(Amt, NVT, DL));
3282 }
3283}
3284
3285/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
3286/// this shift based on knowledge of the high bit of the shift amount. If we
3287/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
3288/// shift amount.
3289bool DAGTypeLegalizer::
3290ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3291 unsigned Opc = N->getOpcode();
3292 SDValue In = N->getOperand(0);
3293 SDValue Amt = N->getOperand(1);
3294 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3295 EVT ShTy = Amt.getValueType();
3296 unsigned ShBits = ShTy.getScalarSizeInBits();
3297 unsigned NVTBits = NVT.getScalarSizeInBits();
3298 assert(isPowerOf2_32(NVTBits) &&
3299 "Expanded integer type size not a power of two!");
3300 SDLoc dl(N);
3301
3302 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
3303 KnownBits Known = DAG.computeKnownBits(Amt);
3304
3305 // If we don't know anything about the high bits, exit.
3306 if (((Known.Zero | Known.One) & HighBitMask) == 0)
3307 return false;
3308
3309 // Get the incoming operand to be shifted.
3310 SDValue InL, InH;
3311 GetExpandedInteger(In, InL, InH);
3312
3313 // If we know that any of the high bits of the shift amount are one, then we
3314 // can do this as a couple of simple shifts.
3315 if (Known.One.intersects(HighBitMask)) {
3316 // Mask out the high bit, which we know is set.
3317 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
3318 DAG.getConstant(~HighBitMask, dl, ShTy));
3319
3320 switch (Opc) {
3321 default: llvm_unreachable("Unknown shift");
3322 case ISD::SHL:
3323 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
3324 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
3325 return true;
3326 case ISD::SRL:
3327 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3328 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
3329 return true;
3330 case ISD::SRA:
3331 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
3332 DAG.getConstant(NVTBits - 1, dl, ShTy));
3333 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
3334 return true;
3335 }
3336 }
3337
3338 // If we know that all of the high bits of the shift amount are zero, then we
3339 // can do this as a couple of simple shifts.
3340 if (HighBitMask.isSubsetOf(Known.Zero)) {
3341 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
3342 // shift if x is zero. We can use XOR here because x is known to be smaller
3343 // than 32.
3344 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
3345 DAG.getConstant(NVTBits - 1, dl, ShTy));
3346
3347 unsigned Op1, Op2;
3348 switch (Opc) {
3349 default: llvm_unreachable("Unknown shift");
3350 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
3351 case ISD::SRL:
3352 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
3353 }
3354
3355 // When shifting right the arithmetic for Lo and Hi is swapped.
3356 if (Opc != ISD::SHL)
3357 std::swap(InL, InH);
3358
3359 // Use a little trick to get the bits that move from Lo to Hi. First
3360 // shift by one bit.
3361 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
3362 // Then compute the remaining shift with amount-1.
3363 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
3364
3365 Lo = DAG.getNode(Opc, dl, NVT, InL, Amt);
3366 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
3367
3368 if (Opc != ISD::SHL)
3369 std::swap(Hi, Lo);
3370 return true;
3371 }
3372
3373 return false;
3374}
3375
3376/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
3377/// of any size.
3378bool DAGTypeLegalizer::
3379ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3380 SDValue Amt = N->getOperand(1);
3381 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3382 EVT ShTy = Amt.getValueType();
3383 unsigned NVTBits = NVT.getSizeInBits();
3384 assert(isPowerOf2_32(NVTBits) &&
3385 "Expanded integer type size not a power of two!");
3386 SDLoc dl(N);
3387
3388 // Get the incoming operand to be shifted.
3389 SDValue InL, InH;
3390 GetExpandedInteger(N->getOperand(0), InL, InH);
3391
3392 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
3393 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
3394 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
3395 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3396 Amt, NVBitsNode, ISD::SETULT);
3397 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3398 Amt, DAG.getConstant(0, dl, ShTy),
3399 ISD::SETEQ);
3400
3401 SDValue LoS, HiS, LoL, HiL;
3402 switch (N->getOpcode()) {
3403 default: llvm_unreachable("Unknown shift");
3404 case ISD::SHL:
3405 // Short: ShAmt < NVTBits
3406 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
3407 HiS = DAG.getNode(ISD::OR, dl, NVT,
3408 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
3409 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
3410
3411 // Long: ShAmt >= NVTBits
3412 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
3413 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
3414
3415 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
3416 Hi = DAG.getSelect(dl, NVT, isZero, InH,
3417 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
3418 return true;
3419 case ISD::SRL:
3420 // Short: ShAmt < NVTBits
3421 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
3422 LoS = DAG.getNode(ISD::OR, dl, NVT,
3423 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3424 // FIXME: If Amt is zero, the following shift generates an undefined result
3425 // on some architectures.
3426 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3427
3428 // Long: ShAmt >= NVTBits
3429 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3430 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3431
3432 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3433 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3434 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3435 return true;
3436 case ISD::SRA:
3437 // Short: ShAmt < NVTBits
3438 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
3439 LoS = DAG.getNode(ISD::OR, dl, NVT,
3440 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3441 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3442
3443 // Long: ShAmt >= NVTBits
3444 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
3445 DAG.getConstant(NVTBits - 1, dl, ShTy));
3446 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3447
3448 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3449 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3450 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3451 return true;
3452 }
3453}
3454
3455static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
3456
3457 switch (Op) {
3458 default: llvm_unreachable("invalid min/max opcode");
3459 case ISD::SMAX:
3460 return std::make_pair(ISD::SETGT, ISD::UMAX);
3461 case ISD::UMAX:
3462 return std::make_pair(ISD::SETUGT, ISD::UMAX);
3463 case ISD::SMIN:
3464 return std::make_pair(ISD::SETLT, ISD::UMIN);
3465 case ISD::UMIN:
3466 return std::make_pair(ISD::SETULT, ISD::UMIN);
3467 }
3468}
3469
3470void DAGTypeLegalizer::ExpandIntRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
3471 SDLoc DL(N);
3472
3473 SDValue LHS = N->getOperand(0);
3474 SDValue RHS = N->getOperand(1);
3475 EVT NewVT = getSetCCResultType(LHS.getValueType());
3476
3477 // Taking the same approach as ScalarizeVecRes_SETCC
3478 SDValue Res = DAG.getNode(ISD::SETCC, DL, NewVT, LHS, RHS, N->getOperand(2));
3479
3480 Res = DAG.getBoolExtOrTrunc(Res, DL, N->getValueType(0), NewVT);
3481 SplitInteger(Res, Lo, Hi);
3482}
3483
3484void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
3485 SDValue &Lo, SDValue &Hi) {
3486 SDLoc DL(N);
3487
3488 SDValue LHS = N->getOperand(0);
3489 SDValue RHS = N->getOperand(1);
3490
3491 // If the upper halves are all sign bits, then we can perform the MINMAX on
3492 // the lower half and sign-extend the result to the upper half.
3493 unsigned NumBits = N->getValueType(0).getScalarSizeInBits();
3494 unsigned NumHalfBits = NumBits / 2;
3495 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits &&
3496 DAG.ComputeNumSignBits(RHS) > NumHalfBits) {
3497 SDValue LHSL, LHSH, RHSL, RHSH;
3498 GetExpandedInteger(LHS, LHSL, LHSH);
3499 GetExpandedInteger(RHS, RHSL, RHSH);
3500 EVT NVT = LHSL.getValueType();
3501
3502 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL);
3503 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo,
3504 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL));
3505 return;
3506 }
3507
3508 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1.
3509 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL.
3510 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) ||
3511 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) {
3512 SDValue LHSL, LHSH, RHSL, RHSH;
3513 GetExpandedInteger(LHS, LHSL, LHSH);
3514 GetExpandedInteger(RHS, RHSL, RHSH);
3515 EVT NVT = LHSL.getValueType();
3516 EVT CCT = getSetCCResultType(NVT);
3517
3518 SDValue HiNeg =
3519 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
3520 if (N->getOpcode() == ISD::SMIN) {
3521 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getAllOnesConstant(DL, NVT));
3522 } else {
3523 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
3524 }
3525 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3526 return;
3527 }
3528
3529 const APInt *RHSVal = nullptr;
3530 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3531 RHSVal = &RHSConst->getAPIntValue();
3532
3533 // The high half of MIN/MAX is always just the the MIN/MAX of the
3534 // high halves of the operands. Expand this way if it appears profitable.
3535 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) &&
3536 (RHSVal->countLeadingOnes() >= NumHalfBits ||
3537 RHSVal->countLeadingZeros() >= NumHalfBits)) {
3538 SDValue LHSL, LHSH, RHSL, RHSH;
3539 GetExpandedInteger(LHS, LHSL, LHSH);
3540 GetExpandedInteger(RHS, RHSL, RHSH);
3541 EVT NVT = LHSL.getValueType();
3542 EVT CCT = getSetCCResultType(NVT);
3543
3544 ISD::NodeType LoOpc;
3545 ISD::CondCode CondC;
3546 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
3547
3548 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3549 // We need to know whether to select Lo part that corresponds to 'winning'
3550 // Hi part or if Hi parts are equal.
3551 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
3552 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
3553
3554 // Lo part corresponding to the 'winning' Hi part
3555 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
3556
3557 // Recursed Lo part if Hi parts are equal, this uses unsigned version
3558 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
3559
3560 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
3561 return;
3562 }
3563
3564 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies
3565 // the compare.
3566 ISD::CondCode Pred;
3567 switch (N->getOpcode()) {
3568 default: llvm_unreachable("How did we get here?");
3569 case ISD::SMAX:
3570 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3571 Pred = ISD::SETGE;
3572 else
3573 Pred = ISD::SETGT;
3574 break;
3575 case ISD::SMIN:
3576 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3577 Pred = ISD::SETLE;
3578 else
3579 Pred = ISD::SETLT;
3580 break;
3581 case ISD::UMAX:
3582 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3583 Pred = ISD::SETUGE;
3584 else
3585 Pred = ISD::SETUGT;
3586 break;
3587 case ISD::UMIN:
3588 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3589 Pred = ISD::SETULE;
3590 else
3591 Pred = ISD::SETULT;
3592 break;
3593 }
3594 EVT VT = N->getValueType(0);
3595 EVT CCT = getSetCCResultType(VT);
3596 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred);
3597 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS);
3598 SplitInteger(Result, Lo, Hi);
3599}
3600
3601void DAGTypeLegalizer::ExpandIntRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
3602 SDValue ExpandedCMP = TLI.expandCMP(N, DAG);
3603 SplitInteger(ExpandedCMP, Lo, Hi);
3604}
3605
3606void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
3607 SDValue &Lo, SDValue &Hi) {
3608 SDLoc dl(N);
3609 // Expand the subcomponents.
3610 SDValue LHSL, LHSH, RHSL, RHSH;
3611 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3612 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3613
3614 EVT NVT = LHSL.getValueType();
3615 SDValue LoOps[2] = { LHSL, RHSL };
3616 SDValue HiOps[3] = { LHSH, RHSH };
3617
3618 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3619 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
3620 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3621 if (HasOpCarry) {
3622 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3623 if (N->getOpcode() == ISD::ADD) {
3624 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3625 HiOps[2] = Lo.getValue(1);
3626 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3627 ? DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2))
3628 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps);
3629 } else {
3630 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3631 HiOps[2] = Lo.getValue(1);
3632 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3633 ? DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2))
3634 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps);
3635 }
3636 return;
3637 }
3638
3639 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
3640 // them. TODO: Teach operation legalization how to expand unsupported
3641 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
3642 // a carry of type MVT::Glue, but there doesn't seem to be any way to
3643 // generate a value of this type in the expanded code sequence.
3644 bool hasCarry =
3645 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3647 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3648
3649 if (hasCarry) {
3650 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
3651 if (N->getOpcode() == ISD::ADD) {
3652 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3653 HiOps[2] = Lo.getValue(1);
3654 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3655 } else {
3656 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3657 HiOps[2] = Lo.getValue(1);
3658 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3659 }
3660 return;
3661 }
3662
3663 bool hasOVF =
3664 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3666 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3668
3669 if (hasOVF) {
3670 EVT OvfVT = getSetCCResultType(NVT);
3671 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
3672 int RevOpc;
3673 if (N->getOpcode() == ISD::ADD) {
3674 RevOpc = ISD::SUB;
3675 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3676 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3677 } else {
3678 RevOpc = ISD::ADD;
3679 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3680 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3681 }
3682 SDValue OVF = Lo.getValue(1);
3683
3684 switch (BoolType) {
3686 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
3687 [[fallthrough]];
3689 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
3690 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
3691 break;
3693 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
3694 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
3695 }
3696 return;
3697 }
3698
3699 if (N->getOpcode() == ISD::ADD) {
3700 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
3701 SDValue Cmp;
3702 // Special case: X+1 has a carry out if X+1==0. This may reduce the live
3703 // range of X. We assume comparing with 0 is cheap.
3704 if (isOneConstant(LoOps[1]))
3705 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3706 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3707 else if (isAllOnesConstant(LoOps[1])) {
3708 if (isAllOnesConstant(HiOps[1]))
3709 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3710 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3711 else
3712 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3713 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3714 } else
3715 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
3716 ISD::SETULT);
3717
3718 SDValue Carry;
3720 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3721 else
3722 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3723 DAG.getConstant(0, dl, NVT));
3724
3725 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1])) {
3726 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry);
3727 } else {
3728 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3729 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
3730 }
3731 } else {
3732 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
3733 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3734 SDValue Cmp =
3735 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3736 LoOps[0], LoOps[1], ISD::SETULT);
3737
3738 SDValue Borrow;
3740 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3741 else
3742 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3743 DAG.getConstant(0, dl, NVT));
3744
3745 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
3746 }
3747}
3748
3749void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
3750 SDValue &Lo, SDValue &Hi) {
3751 // Expand the subcomponents.
3752 SDValue LHSL, LHSH, RHSL, RHSH;
3753 SDLoc dl(N);
3754 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3755 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3756 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3757 SDValue LoOps[2] = { LHSL, RHSL };
3758 SDValue HiOps[3] = { LHSH, RHSH };
3759
3760 if (N->getOpcode() == ISD::ADDC) {
3761 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3762 HiOps[2] = Lo.getValue(1);
3763 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3764 } else {
3765 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3766 HiOps[2] = Lo.getValue(1);
3767 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3768 }
3769
3770 // Legalized the flag result - switch anything that used the old flag to
3771 // use the new one.
3772 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3773}
3774
3775void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
3776 SDValue &Lo, SDValue &Hi) {
3777 // Expand the subcomponents.
3778 SDValue LHSL, LHSH, RHSL, RHSH;
3779 SDLoc dl(N);
3780 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3781 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3782 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3783 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3784 SDValue HiOps[3] = { LHSH, RHSH };
3785
3786 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3787 HiOps[2] = Lo.getValue(1);
3788 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3789
3790 // Legalized the flag result - switch anything that used the old flag to
3791 // use the new one.
3792 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3793}
3794
3795void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
3796 SDValue &Lo, SDValue &Hi) {
3797 SDValue LHS = N->getOperand(0);
3798 SDValue RHS = N->getOperand(1);
3799 SDLoc dl(N);
3800
3801 SDValue Ovf;
3802
3803 unsigned CarryOp, NoCarryOp;
3805 switch(N->getOpcode()) {
3806 case ISD::UADDO:
3807 CarryOp = ISD::UADDO_CARRY;
3808 NoCarryOp = ISD::ADD;
3809 Cond = ISD::SETULT;
3810 break;
3811 case ISD::USUBO:
3812 CarryOp = ISD::USUBO_CARRY;
3813 NoCarryOp = ISD::SUB;
3814 Cond = ISD::SETUGT;
3815 break;
3816 default:
3817 llvm_unreachable("Node has unexpected Opcode");
3818 }
3819
3820 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3821 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3822
3823 if (HasCarryOp) {
3824 // Expand the subcomponents.
3825 SDValue LHSL, LHSH, RHSL, RHSH;
3826 GetExpandedInteger(LHS, LHSL, LHSH);
3827 GetExpandedInteger(RHS, RHSL, RHSH);
3828 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3829 SDValue LoOps[2] = { LHSL, RHSL };
3830 SDValue HiOps[3] = { LHSH, RHSH };
3831
3832 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3833 HiOps[2] = Lo.getValue(1);
3834 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
3835
3836 Ovf = Hi.getValue(1);
3837 } else {
3838 // Expand the result by simply replacing it with the equivalent
3839 // non-overflow-checking operation.
3840 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
3841 SplitInteger(Sum, Lo, Hi);
3842
3843 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) {
3844 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this
3845 // with (Lo | Hi) == 0.
3846 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi);
3847 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or,
3848 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ);
3849 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) {
3850 // Special case: uaddo X, -1 overflows if X == 0.
3851 Ovf =
3852 DAG.getSetCC(dl, N->getValueType(1), LHS,
3853 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE);
3854 } else {
3855 // Calculate the overflow: addition overflows iff a + b < a, and
3856 // subtraction overflows iff a - b > a.
3857 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
3858 }
3859 }
3860
3861 // Legalized the flag result - switch anything that used the old flag to
3862 // use the new one.
3863 ReplaceValueWith(SDValue(N, 1), Ovf);
3864}
3865
3866void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
3867 SDValue &Hi) {
3868 // Expand the subcomponents.
3869 SDValue LHSL, LHSH, RHSL, RHSH;
3870 SDLoc dl(N);
3871 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3872 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3873 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3874 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3875 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
3876
3877 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3878 HiOps[2] = Lo.getValue(1);
3879 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3880
3881 // Legalized the flag result - switch anything that used the old flag to
3882 // use the new one.
3883 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3884}
3885
3886void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
3887 SDValue &Lo, SDValue &Hi) {
3888 // Expand the subcomponents.
3889 SDValue LHSL, LHSH, RHSL, RHSH;
3890 SDLoc dl(N);
3891 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3892 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3893 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3894
3895 // We need to use an unsigned carry op for the lo part.
3896 unsigned CarryOp =
3898 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
3899 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3900
3901 // Legalized the flag result - switch anything that used the old flag to
3902 // use the new one.
3903 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3904}
3905
3906void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
3907 SDValue &Lo, SDValue &Hi) {
3908 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3909 SDLoc dl(N);
3910 SDValue Op = N->getOperand(0);
3911 if (Op.getValueType().bitsLE(NVT)) {
3912 // The low part is any extension of the input (which degenerates to a copy).
3913 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
3914 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
3915 } else {
3916 // For example, extension of an i48 to an i64. The operand type necessarily
3917 // promotes to the result type, so will end up being expanded too.
3918 assert(getTypeAction(Op.getValueType()) ==
3920 "Only know how to promote this result!");
3921 SDValue Res = GetPromotedInteger(Op);
3922 assert(Res.getValueType() == N->getValueType(0) &&
3923 "Operand over promoted?");
3924 // Split the promoted operand. This will simplify when it is expanded.
3925 SplitInteger(Res, Lo, Hi);
3926 }
3927}
3928
3929void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3930 SDValue &Lo, SDValue &Hi) {
3931 SDLoc dl(N);
3932 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3933 EVT NVT = Lo.getValueType();
3934 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3935 unsigned NVTBits = NVT.getSizeInBits();
3936 unsigned EVTBits = EVT.getSizeInBits();
3937
3938 if (NVTBits < EVTBits) {
3939 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3941 EVTBits - NVTBits)));
3942 } else {
3943 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3944 // The high part replicates the sign bit of Lo, make it explicit.
3945 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3946 DAG.getConstant(NVTBits - 1, dl,
3947 TLI.getPointerTy(DAG.getDataLayout())));
3948 }
3949}
3950
3951void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3952 SDValue &Lo, SDValue &Hi) {
3953 SDLoc dl(N);
3954 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3955 EVT NVT = Lo.getValueType();
3956 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3957 unsigned NVTBits = NVT.getSizeInBits();
3958 unsigned EVTBits = EVT.getSizeInBits();
3959
3960 if (NVTBits < EVTBits) {
3961 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3963 EVTBits - NVTBits)));
3964 } else {
3965 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3966 // The high part must be zero, make it explicit.
3967 Hi = DAG.getConstant(0, dl, NVT);
3968 }
3969}
3970
3971void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3972 SDValue &Lo, SDValue &Hi) {
3973 SDLoc dl(N);
3974 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3975 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3976 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3977}
3978
3979void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3980 SDValue &Lo, SDValue &Hi) {
3981 SDLoc dl(N);
3982 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3983 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3984 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3985}
3986
3987void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3988 SDValue &Hi) {
3989 SDLoc dl(N);
3990 // parity(HiLo) -> parity(Lo^Hi)
3991 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3992 EVT NVT = Lo.getValueType();
3993 Lo =
3994 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3995 Hi = DAG.getConstant(0, dl, NVT);
3996}
3997
3998void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3999 SDValue &Lo, SDValue &Hi) {
4000 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4001 unsigned NBitWidth = NVT.getSizeInBits();
4002 auto Constant = cast<ConstantSDNode>(N);
4003 const APInt &Cst = Constant->getAPIntValue();
4004 bool IsTarget = Constant->isTargetOpcode();
4005 bool IsOpaque = Constant->isOpaque();
4006 SDLoc dl(N);
4007 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
4008 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
4009 IsOpaque);
4010}
4011
4012void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
4013 SDLoc dl(N);
4014
4015 SDValue N0 = N->getOperand(0);
4016 GetExpandedInteger(N0, Lo, Hi);
4017 EVT NVT = Lo.getValueType();
4018
4019 // If the upper half is all sign bits, then we can perform the ABS on the
4020 // lower half and zero-extend.
4021 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) {
4022 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo);
4023 Hi = DAG.getConstant(0, dl, NVT);
4024 return;
4025 }
4026
4027 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
4028 // we use in LegalizeDAG. The SUB part of the expansion is based on
4029 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
4030 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
4031 // expanded if needed. Shift expansion has a special case for filling with
4032 // sign bits so that we will only end up with one SRA.
4033 bool HasSubCarry = TLI.isOperationLegalOrCustom(
4035 if (HasSubCarry) {
4036 SDValue Sign = DAG.getNode(
4037 ISD::SRA, dl, NVT, Hi,
4038 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl));
4039 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
4040 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
4041 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
4042 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
4043 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
4044 return;
4045 }
4046
4047 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
4048 EVT VT = N->getValueType(0);
4049 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
4050 DAG.getConstant(0, dl, VT), N0);
4051 SDValue NegLo, NegHi;
4052 SplitInteger(Neg, NegLo, NegHi);
4053
4054 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
4055 DAG.getConstant(0, dl, NVT), ISD::SETLT);
4056 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
4057 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
4058}
4059
4060void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
4061 SDValue &Lo, SDValue &Hi) {
4062 SDLoc dl(N);
4063 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
4064 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4065 EVT NVT = Lo.getValueType();
4066
4067 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
4068 DAG.getConstant(0, dl, NVT), ISD::SETNE);
4069
4070 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
4071 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
4072
4073 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
4074 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
4075 DAG.getConstant(NVT.getSizeInBits(), dl,
4076 NVT)));
4077 Hi = DAG.getConstant(0, dl, NVT);
4078}
4079
4080void DAGTypeLegalizer::ExpandIntRes_ABD(SDNode *N, SDValue &Lo, SDValue &Hi) {
4081 SDValue Result = TLI.expandABD(N, DAG);
4082 SplitInteger(Result, Lo, Hi);
4083}
4084
4085void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, SDValue &Lo, SDValue &Hi) {
4086 SDValue Op = N->getOperand(0);
4087 EVT VT = N->getValueType(0);
4088 SDLoc DL(N);
4089
4091 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4092 if (VT == MVT::i32)
4093 LC = RTLIB::CTPOP_I32;
4094 else if (VT == MVT::i64)
4095 LC = RTLIB::CTPOP_I64;
4096 else if (VT == MVT::i128)
4097 LC = RTLIB::CTPOP_I128;
4098 assert(LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC) &&
4099 "LibCall explicitly requested, but not available");
4101 EVT IntVT =
4103 SDValue Res = TLI.makeLibCall(DAG, LC, IntVT, Op, CallOptions, DL).first;
4104 SplitInteger(DAG.getSExtOrTrunc(Res, DL, VT), Lo, Hi);
4105 return;
4106 }
4107
4108 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
4109 GetExpandedInteger(Op, Lo, Hi);
4110 EVT NVT = Lo.getValueType();
4111 Lo = DAG.getNode(ISD::ADD, DL, NVT, DAG.getNode(ISD::CTPOP, DL, NVT, Lo),
4112 DAG.getNode(ISD::CTPOP, DL, NVT, Hi));
4113 Hi = DAG.getConstant(0, DL, NVT);
4114}
4115
4116void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
4117 SDValue &Lo, SDValue &Hi) {
4118 SDLoc dl(N);
4119 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
4120 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4121 EVT NVT = Lo.getValueType();
4122
4123 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
4124 DAG.getConstant(0, dl, NVT), ISD::SETNE);
4125
4126 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
4127 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
4128
4129 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
4130 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
4131 DAG.getConstant(NVT.getSizeInBits(), dl,
4132 NVT)));
4133 Hi = DAG.getConstant(0, dl, NVT);
4134}
4135
4136void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo,
4137 SDValue &Hi) {
4138 SDLoc dl(N);
4139 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4140 unsigned NBitWidth = NVT.getSizeInBits();
4141
4142 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0));
4143 SDValue Chain = Lo.getValue(1);
4144 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING
4145 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
4146 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
4147
4148 // Legalize the chain result - switch anything that used the old chain to
4149 // use the new one.
4150 ReplaceValueWith(SDValue(N, 1), Chain);
4151}
4152
4153// Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op.
4154static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT,
4155 SDLoc DL, SelectionDAG &DAG) {
4156 if (IsStrict) {
4157 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op});
4158 Chain = Op.getValue(1);
4159 return Op;
4160 }
4161 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Op);
4162}
4163
4164void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
4165 SDValue &Hi) {
4166 SDLoc dl(N);
4167 EVT VT = N->getValueType(0);
4168
4169 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
4170 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
4171 bool IsStrict = N->isStrictFPOpcode();
4172 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4173 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4174 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
4175 Op = GetPromotedFloat(Op);
4176
4177 // If the input is bf16 or needs to be soft promoted, extend to f32.
4178 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf ||
4179 Op.getValueType() == MVT::bf16) {
4180 Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG);
4181 }
4182
4183 // NOTE: We need a variable that lives across makeLibCall so
4184 // CallOptions.setTypeListBeforeSoften can save a reference to it.
4185 EVT OpVT = Op.getValueType();
4186
4187 RTLIB::Libcall LC =
4188 IsSigned ? RTLIB::getFPTOSINT(OpVT, VT) : RTLIB::getFPTOUINT(OpVT, VT);
4189 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
4191 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftenFloat)
4192 CallOptions.setTypeListBeforeSoften(OpVT, VT);
4193 else
4194 CallOptions.setIsSigned(true); // FIXME: Is this needed?
4195 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
4196 CallOptions, dl, Chain);
4197 SplitInteger(Tmp.first, Lo, Hi);
4198
4199 if (IsStrict)
4200 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4201}
4202
4203void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
4204 SDValue &Hi) {
4205 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
4206 SplitInteger(Res, Lo, Hi);
4207}
4208
4209void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
4210 SDValue &Hi) {
4211 SDLoc dl(N);
4212 bool IsStrict = N->isStrictFPOpcode();
4213 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4214 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4215
4216 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
4217 "Input type needs to be promoted!");
4218
4219 EVT VT = Op.getValueType();
4220
4221 if (VT == MVT::f16) {
4222 // Extend to f32.
4223 VT = MVT::f32;
4224 Op = fpExtendHelper(Op, Chain, IsStrict, VT, dl, DAG);
4225 }
4226
4227 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4228 if (N->getOpcode() == ISD::LROUND ||
4229 N->getOpcode() == ISD::STRICT_LROUND) {
4230 if (VT == MVT::f32)
4231 LC = RTLIB::LROUND_F32;
4232 else if (VT == MVT::f64)
4233 LC = RTLIB::LROUND_F64;
4234 else if (VT == MVT::f80)
4235 LC = RTLIB::LROUND_F80;
4236 else if (VT == MVT::f128)
4237 LC = RTLIB::LROUND_F128;
4238 else if (VT == MVT::ppcf128)
4239 LC = RTLIB::LROUND_PPCF128;
4240 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!");
4241 } else if (N->getOpcode() == ISD::LRINT ||
4242 N->getOpcode() == ISD::STRICT_LRINT) {
4243 if (VT == MVT::f32)
4244 LC = RTLIB::LRINT_F32;
4245 else if (VT == MVT::f64)
4246 LC = RTLIB::LRINT_F64;
4247 else if (VT == MVT::f80)
4248 LC = RTLIB::LRINT_F80;
4249 else if (VT == MVT::f128)
4250 LC = RTLIB::LRINT_F128;
4251 else if (VT == MVT::ppcf128)
4252 LC = RTLIB::LRINT_PPCF128;
4253 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!");
4254 } else if (N->getOpcode() == ISD::LLROUND ||
4255 N->getOpcode() == ISD::STRICT_LLROUND) {
4256 if (VT == MVT::f32)
4257 LC = RTLIB::LLROUND_F32;
4258 else if (VT == MVT::f64)
4259 LC = RTLIB::LLROUND_F64;
4260 else if (VT == MVT::f80)
4261 LC = RTLIB::LLROUND_F80;
4262 else if (VT == MVT::f128)
4263 LC = RTLIB::LLROUND_F128;
4264 else if (VT == MVT::ppcf128)
4265 LC = RTLIB::LLROUND_PPCF128;
4266 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
4267 } else if (N->getOpcode() == ISD::LLRINT ||
4268 N->getOpcode() == ISD::STRICT_LLRINT) {
4269 if (VT == MVT::f32)
4270 LC = RTLIB::LLRINT_F32;
4271 else if (VT == MVT::f64)
4272 LC = RTLIB::LLRINT_F64;
4273 else if (VT == MVT::f80)
4274 LC = RTLIB::LLRINT_F80;
4275 else if (VT == MVT::f128)
4276 LC = RTLIB::LLRINT_F128;
4277 else if (VT == MVT::ppcf128)
4278 LC = RTLIB::LLRINT_PPCF128;
4279 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
4280 } else
4281 llvm_unreachable("Unexpected opcode!");
4282
4283 EVT RetVT = N->getValueType(0);
4284
4286 CallOptions.setIsSigned(true);
4287 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4288 Op, CallOptions, dl,
4289 Chain);
4290 SplitInteger(Tmp.first, Lo, Hi);
4291
4292 if (N->isStrictFPOpcode())
4293 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4294}
4295
4296void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
4297 SDValue &Lo, SDValue &Hi) {
4298 assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?");
4299
4300 if (ISD::isNormalLoad(N)) {
4301 ExpandRes_NormalLoad(N, Lo, Hi);
4302 return;
4303 }
4304
4305 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
4306
4307 EVT VT = N->getValueType(0);
4308 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4309 SDValue Ch = N->getChain();
4310 SDValue Ptr = N->getBasePtr();
4311 ISD::LoadExtType ExtType = N->getExtensionType();
4312 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4313 AAMDNodes AAInfo = N->getAAInfo();
4314 SDLoc dl(N);
4315
4316 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4317
4318 if (N->getMemoryVT().bitsLE(NVT)) {
4319 EVT MemVT = N->getMemoryVT();
4320
4321 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
4322 N->getBaseAlign(), MMOFlags, AAInfo);
4323
4324 // Remember the chain.
4325 Ch = Lo.getValue(1);
4326
4327 if (ExtType == ISD::SEXTLOAD) {
4328 // The high part is obtained by SRA'ing all but one of the bits of the
4329 // lo part.
4330 unsigned LoSize = Lo.getValueSizeInBits();
4331 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
4332 DAG.getConstant(LoSize - 1, dl,
4333 TLI.getPointerTy(DAG.getDataLayout())));
4334 } else if (ExtType == ISD::ZEXTLOAD) {
4335 // The high part is just a zero.
4336 Hi = DAG.getConstant(0, dl, NVT);
4337 } else {
4338 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
4339 // The high part is undefined.
4340 Hi = DAG.getUNDEF(NVT);
4341 }
4342 } else if (DAG.getDataLayout().isLittleEndian()) {
4343 // Little-endian - low bits are at low addresses.
4344 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), N->getBaseAlign(),
4345 MMOFlags, AAInfo);
4346
4347 unsigned ExcessBits =
4348 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4349 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4350
4351 // Increment the pointer to the other half.
4352 unsigned IncrementSize = NVT.getSizeInBits()/8;
4353 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
4354 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
4355 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4356 N->getBaseAlign(), MMOFlags, AAInfo);
4357
4358 // Build a factor node to remember that this load is independent of the
4359 // other one.
4360 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4361 Hi.getValue(1));
4362 } else {
4363 // Big-endian - high bits are at low addresses. Favor aligned loads at
4364 // the cost of some bit-fiddling.
4365 EVT MemVT = N->getMemoryVT();
4366 unsigned EBytes = MemVT.getStoreSize();
4367 unsigned IncrementSize = NVT.getSizeInBits()/8;
4368 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4369
4370 // Load both the high bits and maybe some of the low bits.
4371 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
4373 MemVT.getSizeInBits() - ExcessBits),
4374 N->getBaseAlign(), MMOFlags, AAInfo);
4375
4376 // Increment the pointer to the other half.
4377 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
4378 // Load the rest of the low bits.
4379 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
4380 N->getPointerInfo().getWithOffset(IncrementSize),
4381 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4382 N->getBaseAlign(), MMOFlags, AAInfo);
4383
4384 // Build a factor node to remember that this load is independent of the
4385 // other one.
4386 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4387 Hi.getValue(1));
4388
4389 if (ExcessBits < NVT.getSizeInBits()) {
4390 // Transfer low bits from the bottom of Hi to the top of Lo.
4391 Lo = DAG.getNode(
4392 ISD::OR, dl, NVT, Lo,
4393 DAG.getNode(ISD::SHL, dl, NVT, Hi,
4394 DAG.getConstant(ExcessBits, dl,
4395 TLI.getPointerTy(DAG.getDataLayout()))));
4396 // Move high bits to the right position in Hi.
4397 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
4398 Hi,
4399 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4400 TLI.getPointerTy(DAG.getDataLayout())));
4401 }
4402 }
4403
4404 // Legalize the chain result - switch anything that used the old chain to
4405 // use the new one.
4406 ReplaceValueWith(SDValue(N, 1), Ch);
4407}
4408
4409void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
4410 SDValue &Lo, SDValue &Hi) {
4411 SDLoc dl(N);
4412 SDValue LL, LH, RL, RH;
4413 GetExpandedInteger(N->getOperand(0), LL, LH);
4414 GetExpandedInteger(N->getOperand(1), RL, RH);
4415
4417 if (N->getOpcode() == ISD::OR)
4418 Flags.setDisjoint(N->getFlags().hasDisjoint());
4419
4420 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL, Flags);
4421 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH, Flags);
4422}
4423
4424void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
4425 SDValue &Lo, SDValue &Hi) {
4426 EVT VT = N->getValueType(0);
4427 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4428 SDLoc dl(N);
4429
4430 SDValue LL, LH, RL, RH;
4431 GetExpandedInteger(N->getOperand(0), LL, LH);
4432 GetExpandedInteger(N->getOperand(1), RL, RH);
4433
4434 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
4436 LL, LH, RL, RH))
4437 return;
4438
4439 // If nothing else, we can make a libcall.
4440 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4441 if (VT == MVT::i16)
4442 LC = RTLIB::MUL_I16;
4443 else if (VT == MVT::i32)
4444 LC = RTLIB::MUL_I32;
4445 else if (VT == MVT::i64)
4446 LC = RTLIB::MUL_I64;
4447 else if (VT == MVT::i128)
4448 LC = RTLIB::MUL_I128;
4449
4450 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
4451 // Perform a wide multiplication where the wide type is the original VT and
4452 // the 4 parts are the split arguments.
4453 TLI.forceExpandMultiply(DAG, dl, /*Signed=*/false, Lo, Hi, LL, RL, LH, RH);
4454 return;
4455 }
4456
4457 // Note that we don't need to do a wide MUL here since we don't care about the
4458 // upper half of the result if it exceeds VT.
4459 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4461 CallOptions.setIsSigned(true);
4462 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4463 Lo, Hi);
4464}
4465
4466void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo,
4467 SDValue &Hi) {
4468 SDLoc DL(N);
4469 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4470 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
4471 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
4472 Lo = R.getValue(0);
4473 Hi = R.getValue(1);
4474 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
4475}
4476
4477void DAGTypeLegalizer::ExpandIntRes_AVG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4478 SDValue Result = TLI.expandAVG(N, DAG);
4479 SplitInteger(Result, Lo, Hi);
4480}
4481
4482void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
4483 SDValue &Hi) {
4484 SDValue Result = TLI.expandAddSubSat(N, DAG);
4485 SplitInteger(Result, Lo, Hi);
4486}
4487
4488void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
4489 SDValue &Hi) {
4490 SDValue Result = TLI.expandShlSat(N, DAG);
4491 SplitInteger(Result, Lo, Hi);
4492}
4493
4494/// This performs an expansion of the integer result for a fixed point
4495/// multiplication. The default expansion performs rounding down towards
4496/// negative infinity, though targets that do care about rounding should specify
4497/// a target hook for rounding and provide their own expansion or lowering of
4498/// fixed point multiplication to be consistent with rounding.
4499void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
4500 SDValue &Hi) {
4501 SDLoc dl(N);
4502 EVT VT = N->getValueType(0);
4503 unsigned VTSize = VT.getScalarSizeInBits();
4504 SDValue LHS = N->getOperand(0);
4505 SDValue RHS = N->getOperand(1);
4506 uint64_t Scale = N->getConstantOperandVal(2);
4507 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
4508 N->getOpcode() == ISD::UMULFIXSAT);
4509 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
4510 N->getOpcode() == ISD::SMULFIXSAT);
4511
4512 // Handle special case when scale is equal to zero.
4513 if (!Scale) {
4515 if (!Saturating) {
4516 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
4517 } else {
4518 EVT BoolVT = getSetCCResultType(VT);
4519 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
4520 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
4521 SDValue Product = Result.getValue(0);
4522 SDValue Overflow = Result.getValue(1);
4523 if (Signed) {
4524 APInt MinVal = APInt::getSignedMinValue(VTSize);
4525 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
4526 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
4527 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4528 SDValue Zero = DAG.getConstant(0, dl, VT);
4529 // Xor the inputs, if resulting sign bit is 0 the product will be
4530 // positive, else negative.
4531 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4532 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
4533 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
4534 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
4535 } else {
4536 // For unsigned multiplication, we only need to check the max since we
4537 // can't really overflow towards zero.
4538 APInt MaxVal = APInt::getMaxValue(VTSize);
4539 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4540 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
4541 }
4542 }
4543 SplitInteger(Result, Lo, Hi);
4544 return;
4545 }
4546
4547 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
4548 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
4549 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
4550
4551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4552 SDValue LL, LH, RL, RH;
4553 GetExpandedInteger(LHS, LL, LH);
4554 GetExpandedInteger(RHS, RL, RH);
4556
4557 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
4558 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4560 LL, LH, RL, RH)) {
4561 Result.clear();
4562 Result.resize(4);
4563
4564 SDValue LoTmp, HiTmp;
4565 TLI.forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, LoTmp, HiTmp);
4566 SplitInteger(LoTmp, Result[0], Result[1]);
4567 SplitInteger(HiTmp, Result[2], Result[3]);
4568 }
4569 assert(Result.size() == 4 && "Unexpected number of partlets in the result");
4570
4571 unsigned NVTSize = NVT.getScalarSizeInBits();
4572 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
4573 "the size of the current value type");
4574
4575 // After getting the multiplication result in 4 parts, we need to perform a
4576 // shift right by the amount of the scale to get the result in that scale.
4577 //
4578 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
4579 // 128 bits that are cut into 4 32-bit parts:
4580 //
4581 // HH HL LH LL
4582 // |---32---|---32---|---32---|---32---|
4583 // 128 96 64 32 0
4584 //
4585 // |------VTSize-----|
4586 //
4587 // |NVTSize-|
4588 //
4589 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
4590 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
4591 // amount and get Lo and Hi using two funnel shifts. Or for the special case
4592 // when Scale is a multiple of NVTSize we can just pick the result without
4593 // shifting.
4594 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
4595 if (Scale % NVTSize) {
4596 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl);
4597 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
4598 ShiftAmount);
4599 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
4600 ShiftAmount);
4601 } else {
4602 Lo = Result[Part0];
4603 Hi = Result[Part0 + 1];
4604 }
4605
4606 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
4607 if (!Saturating)
4608 return;
4609
4610 // Can not overflow when there is no integer part.
4611 if (Scale == VTSize)
4612 return;
4613
4614 // To handle saturation we must check for overflow in the multiplication.
4615 //
4616 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
4617 // aren't all zeroes.
4618 //
4619 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
4620 // aren't all ones or all zeroes.
4621 //
4622 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
4623 // highest bit of HH determines saturation direction in the event of signed
4624 // saturation.
4625
4626 SDValue ResultHL = Result[2];
4627 SDValue ResultHH = Result[3];
4628
4629 SDValue SatMax, SatMin;
4630 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
4631 SDValue NVTNeg1 = DAG.getAllOnesConstant(dl, NVT);
4632 EVT BoolNVT = getSetCCResultType(NVT);
4633
4634 if (!Signed) {
4635 if (Scale < NVTSize) {
4636 // Overflow happened if ((HH | (HL >> Scale)) != 0).
4637 SDValue HLAdjusted =
4638 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4639 DAG.getShiftAmountConstant(Scale, NVT, dl));
4640 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
4641 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
4642 } else if (Scale == NVTSize) {
4643 // Overflow happened if (HH != 0).
4644 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
4645 } else if (Scale < VTSize) {
4646 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
4647 SDValue HLAdjusted =
4648 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4649 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl));
4650 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
4651 } else
4652 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
4653 "(and saturation can't happen with Scale==VTSize).");
4654
4655 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
4656 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
4657 return;
4658 }
4659
4660 if (Scale < NVTSize) {
4661 // The number of overflow bits we can check are VTSize - Scale + 1 (we
4662 // include the sign bit). If these top bits are > 0, then we overflowed past
4663 // the max value. If these top bits are < -1, then we overflowed past the
4664 // min value. Otherwise, we did not overflow.
4665 unsigned OverflowBits = VTSize - Scale + 1;
4666 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4667 "Extent of overflow bits must start within HL");
4668 SDValue HLHiMask = DAG.getConstant(
4669 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
4670 SDValue HLLoMask = DAG.getConstant(
4671 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
4672 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
4673 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4674 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4675 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
4676 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4677 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
4678 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
4679 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4680 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4681 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
4682 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4683 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
4684 } else if (Scale == NVTSize) {
4685 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
4686 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4687 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4688 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
4689 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4690 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
4691 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
4692 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4693 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4694 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
4695 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4696 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
4697 } else if (Scale < VTSize) {
4698 // This is similar to the case when we saturate if Scale < NVTSize, but we
4699 // only need to check HH.
4700 unsigned OverflowBits = VTSize - Scale + 1;
4701 SDValue HHHiMask = DAG.getConstant(
4702 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
4703 SDValue HHLoMask = DAG.getConstant(
4704 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
4705 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
4706 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
4707 } else
4708 llvm_unreachable("Illegal scale for signed fixed point mul.");
4709
4710 // Saturate to signed maximum.
4711 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
4712 APInt MaxLo = APInt::getAllOnes(NVTSize);
4713 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
4714 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
4715 // Saturate to signed minimum.
4716 APInt MinHi = APInt::getSignedMinValue(NVTSize);
4717 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
4718 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
4719}
4720
4721void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
4722 SDValue &Hi) {
4723 SDLoc dl(N);
4724 // Try expanding in the existing type first.
4725 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
4726 N->getOperand(1),
4727 N->getConstantOperandVal(2), DAG);
4728
4729 if (!Res)
4730 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
4731 N->getConstantOperandVal(2), TLI, DAG);
4732 SplitInteger(Res, Lo, Hi);
4733}
4734
4735void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
4736 SDValue &Lo, SDValue &Hi) {
4737 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
4738 "Node has unexpected Opcode");
4739 SDValue LHS = Node->getOperand(0);
4740 SDValue RHS = Node->getOperand(1);
4741 SDLoc dl(Node);
4742
4743 SDValue Ovf;
4744
4745 bool IsAdd = Node->getOpcode() == ISD::SADDO;
4746 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
4747
4748 bool HasCarryOp = TLI.isOperationLegalOrCustom(
4749 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
4750
4751 if (HasCarryOp) {
4752 // Expand the subcomponents.
4753 SDValue LHSL, LHSH, RHSL, RHSH;
4754 GetExpandedInteger(LHS, LHSL, LHSH);
4755 GetExpandedInteger(RHS, RHSL, RHSH);
4756 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
4757
4758 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
4759 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
4760
4761 Ovf = Hi.getValue(1);
4762 } else {
4763 // Expand the result by simply replacing it with the equivalent
4764 // non-overflow-checking operation.
4765 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
4766 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
4767 LHS, RHS);
4768 SplitInteger(Sum, Lo, Hi);
4769
4770 // Compute the overflow.
4771 //
4772 // LHSSign -> LHS < 0
4773 // RHSSign -> RHS < 0
4774 // SumSign -> Sum < 0
4775 //
4776 // Add:
4777 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
4778 // Sub:
4779 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
4780 //
4781 // To get better codegen we can rewrite this by doing bitwise math on
4782 // the integers and extract the final sign bit at the end. So the
4783 // above becomes:
4784 //
4785 // Add:
4786 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
4787 // Sub:
4788 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
4789 //
4790 // NOTE: This is different than the expansion we do in expandSADDSUBO
4791 // because it is more costly to determine the RHS is > 0 for SSUBO with the
4792 // integers split.
4793 EVT VT = LHS.getValueType();
4794 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4795 if (IsAdd)
4796 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
4797
4798 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
4799 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
4800 EVT OType = Node->getValueType(1);
4801 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
4802 }
4803
4804 // Use the calculated overflow everywhere.
4805 ReplaceValueWith(SDValue(Node, 1), Ovf);
4806}
4807
4808void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
4809 SDValue &Lo, SDValue &Hi) {
4810 EVT VT = N->getValueType(0);
4811 SDLoc dl(N);
4812 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4813
4815 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4816 SplitInteger(Res.getValue(0), Lo, Hi);
4817 return;
4818 }
4819
4820 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4821 if (VT == MVT::i16)
4822 LC = RTLIB::SDIV_I16;
4823 else if (VT == MVT::i32)
4824 LC = RTLIB::SDIV_I32;
4825 else if (VT == MVT::i64)
4826 LC = RTLIB::SDIV_I64;
4827 else if (VT == MVT::i128)
4828 LC = RTLIB::SDIV_I128;
4829 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
4830
4832 CallOptions.setIsSigned(true);
4833 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4834}
4835
4836void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo,
4837 SDValue &Hi) {
4838 SDLoc dl(N);
4839 SDValue Shiftee = N->getOperand(0);
4840 EVT VT = Shiftee.getValueType();
4841 SDValue ShAmt = N->getOperand(1);
4842 EVT ShAmtVT = ShAmt.getValueType();
4843
4844 EVT LoadVT = VT;
4845 do {
4846 LoadVT = TLI.getTypeToTransformTo(*DAG.getContext(), LoadVT);
4847 } while (!TLI.isTypeLegal(LoadVT));
4848
4849 const unsigned ShiftUnitInBits = LoadVT.getStoreSizeInBits();
4850 assert(ShiftUnitInBits <= VT.getScalarSizeInBits());
4851 assert(isPowerOf2_32(ShiftUnitInBits) &&
4852 "Shifting unit is not a a power of two!");
4853
4854 const bool IsOneStepShift =
4856 Log2_32(ShiftUnitInBits);
4857
4858 // If we can't do it as one step, we'll have two uses of shift amount,
4859 // and thus must freeze it.
4860 if (!IsOneStepShift)
4861 ShAmt = DAG.getFreeze(ShAmt);
4862
4863 unsigned VTBitWidth = VT.getScalarSizeInBits();
4864 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?");
4865 unsigned VTByteWidth = VTBitWidth / 8;
4866 assert(isPowerOf2_32(VTByteWidth) &&
4867 "Shiftee type size is not a power of two!");
4868 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4869 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4870 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth);
4871
4872 // Get a temporary stack slot 2x the width of our VT.
4873 // FIXME: reuse stack slots?
4874 Align StackAlign = DAG.getReducedAlign(StackSlotVT, /*UseABI=*/false);
4876 DAG.CreateStackTemporary(StackSlotVT.getStoreSize(), StackAlign);
4877 EVT PtrTy = StackPtr.getValueType();
4878 SDValue Ch = DAG.getEntryNode();
4879
4881 DAG.getMachineFunction(),
4882 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex());
4883
4884 // Extend the value, that is being shifted, to the entire stack slot's width.
4885 SDValue Init;
4886 if (N->getOpcode() != ISD::SHL) {
4887 unsigned WideningOpc =
4888 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4889 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4890 } else {
4891 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width.
4892 SDValue AllZeros = DAG.getConstant(0, dl, VT);
4893 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee);
4894 }
4895 // And spill it into the stack slot.
4896 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackAlign);
4897
4898 // Now, compute the full-byte offset into stack slot from where we can load.
4899 // We have shift amount, which is in bits. Offset should point to an aligned
4900 // address.
4902 Flags.setExact(IsOneStepShift);
4903 SDValue SrlTmp = DAG.getNode(
4904 ISD::SRL, dl, ShAmtVT, ShAmt,
4905 DAG.getConstant(Log2_32(ShiftUnitInBits), dl, ShAmtVT), Flags);
4906 SDValue BitOffset =
4907 DAG.getNode(ISD::SHL, dl, ShAmtVT, SrlTmp,
4908 DAG.getConstant(Log2_32(ShiftUnitInBits), dl, ShAmtVT));
4909
4910 SDValue ByteOffset =
4911 DAG.getNode(ISD::SRL, dl, ShAmtVT, BitOffset,
4912 DAG.getConstant(3, dl, ShAmtVT), SDNodeFlags::Exact);
4913 // And clamp it, because OOB load is an immediate UB,
4914 // while shift overflow would have *just* been poison.
4915 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset,
4916 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT));
4917 // We have exactly two strategies on indexing into stack slot here:
4918 // 1. upwards starting from the beginning of the slot
4919 // 2. downwards starting from the middle of the slot
4920 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift
4921 // and vice versa on big-endian machine.
4922 bool WillIndexUpwards = N->getOpcode() != ISD::SHL;
4923 if (DAG.getDataLayout().isBigEndian())
4924 WillIndexUpwards = !WillIndexUpwards;
4925
4926 SDValue AdjStackPtr;
4927 if (WillIndexUpwards) {
4928 AdjStackPtr = StackPtr;
4929 } else {
4930 AdjStackPtr = DAG.getMemBasePlusOffset(
4931 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl);
4932 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT);
4933 }
4934
4935 // Get the pointer somewhere into the stack slot from which we need to load.
4936 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy);
4937 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl);
4938
4939 // And load it! While the load is not legal, legalizing it is obvious.
4940 SDValue Res =
4941 DAG.getLoad(VT, dl, Ch, AdjStackPtr,
4943 commonAlignment(StackAlign, LoadVT.getStoreSize()));
4944
4945 // If we may still have a remaining bits to shift by, do so now.
4946 if (!IsOneStepShift) {
4947 SDValue ShAmtRem =
4948 DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
4949 DAG.getConstant(ShiftUnitInBits - 1, dl, ShAmtVT));
4950 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem);
4951 }
4952
4953 // Finally, split the computed value.
4954 SplitInteger(Res, Lo, Hi);
4955}
4956
4957void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
4958 SDValue &Lo, SDValue &Hi) {
4959 EVT VT = N->getValueType(0);
4960 unsigned Opc = N->getOpcode();
4961 SDLoc dl(N);
4962
4963 // If we can emit an efficient shift operation, do so now. Check to see if
4964 // the RHS is a constant.
4965 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
4966 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
4967
4968 // If we can determine that the high bit of the shift is zero or one, even if
4969 // the low bits are variable, emit this shift in an optimized form.
4970 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
4971 return;
4972
4973 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
4974 unsigned PartsOpc;
4975 if (Opc == ISD::SHL) {
4976 PartsOpc = ISD::SHL_PARTS;
4977 } else if (Opc == ISD::SRL) {
4978 PartsOpc = ISD::SRL_PARTS;
4979 } else {
4980 assert(Opc == ISD::SRA && "Unknown shift!");
4981 PartsOpc = ISD::SRA_PARTS;
4982 }
4983
4984 // Next check to see if the target supports this SHL_PARTS operation or if it
4985 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
4986 // size, but create a libcall instead.
4987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4988 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
4989 const bool LegalOrCustom =
4990 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
4991 Action == TargetLowering::Custom;
4992
4993 unsigned ExpansionFactor = 1;
4994 // That VT->NVT expansion is one step. But will we re-expand NVT?
4995 for (EVT TmpVT = NVT;;) {
4996 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT);
4997 if (NewTMPVT == TmpVT)
4998 break;
4999 TmpVT = NewTMPVT;
5000 ++ExpansionFactor;
5001 }
5002
5004 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor);
5005
5007 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi);
5008
5009 if (LegalOrCustom &&
5011 // Expand the subcomponents.
5012 SDValue LHSL, LHSH;
5013 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
5014 EVT VT = LHSL.getValueType();
5015
5016 // If the shift amount operand is coming from a vector legalization it may
5017 // have an illegal type. Fix that first by casting the operand, otherwise
5018 // the new SHL_PARTS operation would need further legalization.
5019 SDValue ShiftOp = N->getOperand(1);
5020 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5021 if (ShiftOp.getValueType() != ShiftTy)
5022 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
5023
5024 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
5025 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
5026 Hi = Lo.getValue(1);
5027 return;
5028 }
5029
5030 // Otherwise, emit a libcall.
5031 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5032 bool isSigned;
5033 if (Opc == ISD::SHL) {
5034 isSigned = false; /*sign irrelevant*/
5035 if (VT == MVT::i16)
5036 LC = RTLIB::SHL_I16;
5037 else if (VT == MVT::i32)
5038 LC = RTLIB::SHL_I32;
5039 else if (VT == MVT::i64)
5040 LC = RTLIB::SHL_I64;
5041 else if (VT == MVT::i128)
5042 LC = RTLIB::SHL_I128;
5043 } else if (Opc == ISD::SRL) {
5044 isSigned = false;
5045 if (VT == MVT::i16)
5046 LC = RTLIB::SRL_I16;
5047 else if (VT == MVT::i32)
5048 LC = RTLIB::SRL_I32;
5049 else if (VT == MVT::i64)
5050 LC = RTLIB::SRL_I64;
5051 else if (VT == MVT::i128)
5052 LC = RTLIB::SRL_I128;
5053 } else {
5054 assert(Opc == ISD::SRA && "Unknown shift!");
5055 isSigned = true;
5056 if (VT == MVT::i16)
5057 LC = RTLIB::SRA_I16;
5058 else if (VT == MVT::i32)
5059 LC = RTLIB::SRA_I32;
5060 else if (VT == MVT::i64)
5061 LC = RTLIB::SRA_I64;
5062 else if (VT == MVT::i128)
5063 LC = RTLIB::SRA_I128;
5064 }
5065
5066 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
5067 EVT ShAmtTy =
5069 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
5070 SDValue Ops[2] = {N->getOperand(0), ShAmt};
5072 CallOptions.setIsSigned(isSigned);
5073 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5074 return;
5075 }
5076
5077 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
5078 llvm_unreachable("Unsupported shift!");
5079}
5080
5081void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
5082 SDValue &Lo, SDValue &Hi) {
5083 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5084 SDLoc dl(N);
5085 SDValue Op = N->getOperand(0);
5086 if (Op.getValueType().bitsLE(NVT)) {
5087 // The low part is sign extension of the input (degenerates to a copy).
5088 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
5089 // The high part is obtained by SRA'ing all but one of the bits of low part.
5090 unsigned LoSize = NVT.getSizeInBits();
5091 Hi = DAG.getNode(
5092 ISD::SRA, dl, NVT, Lo,
5093 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
5094 } else {
5095 // For example, extension of an i48 to an i64. The operand type necessarily
5096 // promotes to the result type, so will end up being expanded too.
5097 assert(getTypeAction(Op.getValueType()) ==
5099 "Only know how to promote this result!");
5100 SDValue Res = GetPromotedInteger(Op);
5101 assert(Res.getValueType() == N->getValueType(0) &&
5102 "Operand over promoted?");
5103 // Split the promoted operand. This will simplify when it is expanded.
5104 SplitInteger(Res, Lo, Hi);
5105 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5106 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
5108 ExcessBits)));
5109 }
5110}
5111
5112void DAGTypeLegalizer::
5113ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
5114 SDLoc dl(N);
5115 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5116 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
5117
5118 if (EVT.bitsLE(Lo.getValueType())) {
5119 // sext_inreg the low part if needed.
5120 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
5121 N->getOperand(1));
5122
5123 // The high part gets the sign extension from the lo-part. This handles
5124 // things like sextinreg V:i64 from i8.
5125 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
5126 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
5127 TLI.getPointerTy(DAG.getDataLayout())));
5128 } else {
5129 // For example, extension of an i48 to an i64. Leave the low part alone,
5130 // sext_inreg the high part.
5131 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
5132 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
5134 ExcessBits)));
5135 }
5136}
5137
5138void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
5139 SDValue &Lo, SDValue &Hi) {
5140 EVT VT = N->getValueType(0);
5141 SDLoc dl(N);
5142 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5143
5145 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5146 SplitInteger(Res.getValue(1), Lo, Hi);
5147 return;
5148 }
5149
5150 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5151 if (VT == MVT::i16)
5152 LC = RTLIB::SREM_I16;
5153 else if (VT == MVT::i32)
5154 LC = RTLIB::SREM_I32;
5155 else if (VT == MVT::i64)
5156 LC = RTLIB::SREM_I64;
5157 else if (VT == MVT::i128)
5158 LC = RTLIB::SREM_I128;
5159 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
5160
5162 CallOptions.setIsSigned(true);
5163 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5164}
5165
5166void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
5167 SDValue &Lo, SDValue &Hi) {
5168 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5169 SDLoc dl(N);
5170 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
5171 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
5172 N->getOperand(0),
5173 DAG.getConstant(NVT.getSizeInBits(), dl,
5174 TLI.getPointerTy(DAG.getDataLayout())));
5175 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
5176}
5177
5178void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
5179 SDValue &Lo, SDValue &Hi) {
5180 EVT VT = N->getValueType(0);
5181 SDLoc dl(N);
5182
5183 if (N->getOpcode() == ISD::UMULO) {
5184 // This section expands the operation into the following sequence of
5185 // instructions. `iNh` here refers to a type which has half the bit width of
5186 // the type the original operation operated on.
5187 //
5188 // %0 = %LHS.HI != 0 && %RHS.HI != 0
5189 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
5190 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
5191 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
5192 // %4 = add iNh %1.0, %2.0 as iN
5193 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
5194 //
5195 // %lo = %3.LO
5196 // %hi = %5.0
5197 // %ovf = %0 || %1.1 || %2.1 || %5.1
5198 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
5199 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
5200 GetExpandedInteger(LHS, LHSLow, LHSHigh);
5201 GetExpandedInteger(RHS, RHSLow, RHSHigh);
5202 EVT HalfVT = LHSLow.getValueType();
5203 EVT BitVT = N->getValueType(1);
5204 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
5205
5206 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
5207 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
5208 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
5209 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
5210
5211 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
5212 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
5213
5214 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
5215 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
5216
5217 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
5218
5219 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
5220 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
5221 // operation recursively legalized?).
5222 //
5223 // Many backends understand this pattern and will convert into LOHI
5224 // themselves, if applicable.
5225 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
5226 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
5227 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
5228 SplitInteger(Three, Lo, Hi);
5229
5230 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
5231 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
5232 ReplaceValueWith(SDValue(N, 1), Overflow);
5233 return;
5234 }
5235
5236 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
5237 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5238 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
5239
5240 // Replace this with a libcall that will check overflow.
5241 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5242 if (VT == MVT::i32)
5243 LC = RTLIB::MULO_I32;
5244 else if (VT == MVT::i64)
5245 LC = RTLIB::MULO_I64;
5246 else if (VT == MVT::i128)
5247 LC = RTLIB::MULO_I128;
5248
5249 // If we don't have the libcall or if the function we are compiling is the
5250 // implementation of the expected libcall (avoid inf-loop), expand inline.
5251 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) ||
5252 TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) {
5253 // FIXME: This is not an optimal expansion, but better than crashing.
5254 SDValue MulLo, MulHi;
5255 TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, N->getOperand(0),
5256 N->getOperand(1), MulLo, MulHi);
5257 SDValue SRA =
5258 DAG.getNode(ISD::SRA, dl, VT, MulLo,
5259 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT));
5260 SDValue Overflow =
5261 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
5262 SplitInteger(MulLo, Lo, Hi);
5263 ReplaceValueWith(SDValue(N, 1), Overflow);
5264 return;
5265 }
5266
5267 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
5268 // Temporary for the overflow value, default it to zero.
5269 SDValue Chain =
5270 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
5272
5274 for (const SDValue &Op : N->op_values()) {
5275 EVT ArgVT = Op.getValueType();
5276 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
5278 Entry.IsSExt = true;
5279 Entry.IsZExt = false;
5280 Args.push_back(Entry);
5281 }
5282
5283 // Also pass the address of the overflow check.
5285 Temp, PointerType::getUnqual(PtrTy->getContext()));
5286 Entry.IsSExt = true;
5287 Entry.IsZExt = false;
5288 Args.push_back(Entry);
5289
5290 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
5291
5293 CLI.setDebugLoc(dl)
5294 .setChain(Chain)
5295 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
5296 .setSExtResult();
5297
5298 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
5299
5300 SplitInteger(CallInfo.first, Lo, Hi);
5301 SDValue Temp2 =
5302 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
5303 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
5304 DAG.getConstant(0, dl, PtrVT),
5305 ISD::SETNE);
5306 // Use the overflow from the libcall everywhere.
5307 ReplaceValueWith(SDValue(N, 1), Ofl);
5308}
5309
5310void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
5311 SDValue &Lo, SDValue &Hi) {
5312 EVT VT = N->getValueType(0);
5313 SDLoc dl(N);
5314 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5315
5317 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5318 SplitInteger(Res.getValue(0), Lo, Hi);
5319 return;
5320 }
5321
5322 // Try to expand UDIV by constant.
5323 if (isa<ConstantSDNode>(N->getOperand(1))) {
5324 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5325 // Only if the new type is legal.
5326 if (isTypeLegal(NVT)) {
5327 SDValue InL, InH;
5328 GetExpandedInteger(N->getOperand(0), InL, InH);
5330 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
5331 Lo = Result[0];
5332 Hi = Result[1];
5333 return;
5334 }
5335 }
5336 }
5337
5338 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5339 if (VT == MVT::i16)
5340 LC = RTLIB::UDIV_I16;
5341 else if (VT == MVT::i32)
5342 LC = RTLIB::UDIV_I32;
5343 else if (VT == MVT::i64)
5344 LC = RTLIB::UDIV_I64;
5345 else if (VT == MVT::i128)
5346 LC = RTLIB::UDIV_I128;
5347 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
5348
5350 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5351}
5352
5353void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
5354 SDValue &Lo, SDValue &Hi) {
5355 EVT VT = N->getValueType(0);
5356 SDLoc dl(N);
5357 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
5358
5360 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
5361 SplitInteger(Res.getValue(1), Lo, Hi);
5362 return;
5363 }
5364
5365 // Try to expand UREM by constant.
5366 if (isa<ConstantSDNode>(N->getOperand(1))) {
5367 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5368 // Only if the new type is legal.
5369 if (isTypeLegal(NVT)) {
5370 SDValue InL, InH;
5371 GetExpandedInteger(N->getOperand(0), InL, InH);
5373 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
5374 Lo = Result[0];
5375 Hi = Result[1];
5376 return;
5377 }
5378 }
5379 }
5380
5381 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5382 if (VT == MVT::i16)
5383 LC = RTLIB::UREM_I16;
5384 else if (VT == MVT::i32)
5385 LC = RTLIB::UREM_I32;
5386 else if (VT == MVT::i64)
5387 LC = RTLIB::UREM_I64;
5388 else if (VT == MVT::i128)
5389 LC = RTLIB::UREM_I128;
5390 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
5391
5393 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
5394}
5395
5396void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
5397 SDValue &Lo, SDValue &Hi) {
5398 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5399 SDLoc dl(N);
5400 SDValue Op = N->getOperand(0);
5401 if (Op.getValueType().bitsLE(NVT)) {
5402 // The low part is zero extension of the input (degenerates to a copy).
5403 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
5404 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
5405 } else {
5406 // For example, extension of an i48 to an i64. The operand type necessarily
5407 // promotes to the result type, so will end up being expanded too.
5408 assert(getTypeAction(Op.getValueType()) ==
5410 "Only know how to promote this result!");
5411 SDValue Res = GetPromotedInteger(Op);
5412 assert(Res.getValueType() == N->getValueType(0) &&
5413 "Operand over promoted?");
5414 // Split the promoted operand. This will simplify when it is expanded.
5415 SplitInteger(Res, Lo, Hi);
5416 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5417 Hi = DAG.getZeroExtendInReg(Hi, dl,
5419 ExcessBits));
5420 }
5421}
5422
5423void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
5424 SDValue &Lo, SDValue &Hi) {
5425 SDLoc dl(N);
5426 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
5427 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
5428 SDValue Zero = DAG.getConstant(0, dl, VT);
5429 SDValue Swap = DAG.getAtomicCmpSwap(
5431 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
5432 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
5433
5434 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
5435 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
5436}
5437
5438void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
5439 SDValue &Lo, SDValue &Hi) {
5440 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
5441 // both halves independently.
5442 SDValue Res = TLI.expandVecReduce(N, DAG);
5443 SplitInteger(Res, Lo, Hi);
5444}
5445
5446void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
5447 SDValue &Lo, SDValue &Hi) {
5448 // Delegate to funnel-shift expansion.
5449 SDLoc DL(N);
5450 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
5451 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0),
5452 N->getOperand(0), N->getOperand(1));
5453 SplitInteger(Res, Lo, Hi);
5454}
5455
5456void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
5457 SDValue &Hi) {
5458 // Values numbered from least significant to most significant.
5459 SDValue In1, In2, In3, In4;
5460 GetExpandedInteger(N->getOperand(0), In3, In4);
5461 GetExpandedInteger(N->getOperand(1), In1, In2);
5462 EVT HalfVT = In1.getValueType();
5463
5464 SDLoc DL(N);
5465 unsigned Opc = N->getOpcode();
5466 SDValue ShAmt = N->getOperand(2);
5467 EVT ShAmtVT = ShAmt.getValueType();
5468 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5469
5470 // If the shift amount is at least half the bitwidth, swap the inputs.
5471 unsigned HalfVTBits = HalfVT.getScalarSizeInBits();
5472 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt,
5473 DAG.getConstant(HalfVTBits, DL, ShAmtVT));
5474 SDValue Cond =
5475 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT),
5477
5478 // Expand to a pair of funnel shifts.
5479 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5480 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT);
5481
5482 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2);
5483 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3);
5484 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4);
5485 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt);
5486 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
5487}
5488
5489void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
5490 SDValue &Hi) {
5491 EVT VT = N->getValueType(0);
5492 EVT HalfVT =
5493 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
5494 SDLoc dl(N);
5495
5496 // We assume VSCALE(1) fits into a legal integer.
5497 APInt One(HalfVT.getSizeInBits(), 1);
5498 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
5499 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
5500 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
5501 SplitInteger(Res, Lo, Hi);
5502}
5503
5504void DAGTypeLegalizer::ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo,
5505 SDValue &Hi) {
5506 const Function &Fn = DAG.getMachineFunction().getFunction();
5508 "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc()));
5509 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
5510 EVT LoVT, HiVT;
5511 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
5512 Lo = DAG.getPOISON(LoVT);
5513 Hi = DAG.getPOISON(HiVT);
5514}
5515
5516//===----------------------------------------------------------------------===//
5517// Integer Operand Expansion
5518//===----------------------------------------------------------------------===//
5519
5520/// ExpandIntegerOperand - This method is called when the specified operand of
5521/// the specified node is found to need expansion. At this point, all of the
5522/// result types of the node are known to be legal, but other operands of the
5523/// node may need promotion or expansion as well as the specified one.
5524bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
5525 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG));
5526 SDValue Res = SDValue();
5527
5528 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
5529 return false;
5530
5531 switch (N->getOpcode()) {
5532 default:
5533 #ifndef NDEBUG
5534 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
5535 N->dump(&DAG); dbgs() << "\n";
5536 #endif
5537 report_fatal_error("Do not know how to expand this operator's operand!");
5538
5539 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
5540 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
5541 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
5542 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
5543 case ISD::FAKE_USE:
5544 Res = ExpandOp_FAKE_USE(N);
5545 break;
5546 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
5547 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
5548 case ISD::EXPERIMENTAL_VP_SPLAT:
5549 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
5550 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
5551 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
5552 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
5554 case ISD::SINT_TO_FP:
5556 case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break;
5557 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
5558 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
5559
5560 case ISD::SHL:
5561 case ISD::SRA:
5562 case ISD::SRL:
5563 case ISD::ROTL:
5564 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
5565 case ISD::RETURNADDR:
5566 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
5567
5568 case ISD::SCMP:
5569 case ISD::UCMP: Res = ExpandIntOp_CMP(N); break;
5570
5571 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
5572 case ISD::STACKMAP:
5573 Res = ExpandIntOp_STACKMAP(N, OpNo);
5574 break;
5575 case ISD::PATCHPOINT:
5576 Res = ExpandIntOp_PATCHPOINT(N, OpNo);
5577 break;
5578 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5579 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5580 Res = ExpandIntOp_VP_STRIDED(N, OpNo);
5581 break;
5583 Res = ExpandIntOp_WRITE_REGISTER(N, OpNo);
5584 break;
5585 }
5586
5587 // If the result is null, the sub-method took care of registering results etc.
5588 if (!Res.getNode()) return false;
5589
5590 // If the result is N, the sub-method updated N in place. Tell the legalizer
5591 // core about this.
5592 if (Res.getNode() == N)
5593 return true;
5594
5595 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
5596 "Invalid operand expansion");
5597
5598 ReplaceValueWith(SDValue(N, 0), Res);
5599 return false;
5600}
5601
5602/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
5603/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
5604void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
5605 SDValue &NewRHS,
5606 ISD::CondCode &CCCode,
5607 const SDLoc &dl) {
5608 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5609 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5610 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5611
5612 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
5613 if (RHSLo == RHSHi && isAllOnesConstant(RHSLo)) {
5614 // Equality comparison to -1.
5615 NewLHS = DAG.getNode(ISD::AND, dl, LHSLo.getValueType(), LHSLo, LHSHi);
5616 NewRHS = RHSLo;
5617 return;
5618 }
5619
5620 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
5621 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
5622 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
5623 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
5624 return;
5625 }
5626
5627 // If this is a comparison of the sign bit, just look at the top part.
5628 // X > -1, x < 0
5629 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
5630 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
5631 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
5632 NewLHS = LHSHi;
5633 NewRHS = RHSHi;
5634 return;
5635 }
5636
5637 // FIXME: This generated code sucks.
5638 ISD::CondCode LowCC;
5639 switch (CCCode) {
5640 default: llvm_unreachable("Unknown integer setcc!");
5641 case ISD::SETLT:
5642 case ISD::SETULT: LowCC = ISD::SETULT; break;
5643 case ISD::SETGT:
5644 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
5645 case ISD::SETLE:
5646 case ISD::SETULE: LowCC = ISD::SETULE; break;
5647 case ISD::SETGE:
5648 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
5649 }
5650
5651 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
5652 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
5653 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
5654
5655 // NOTE: on targets without efficient SELECT of bools, we can always use
5656 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
5657 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
5658 nullptr);
5659 SDValue LoCmp, HiCmp;
5660 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
5661 TLI.isTypeLegal(RHSLo.getValueType()))
5662 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
5663 RHSLo, LowCC, false, DagCombineInfo, dl);
5664 if (!LoCmp.getNode())
5665 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
5666 RHSLo, LowCC);
5667 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
5668 TLI.isTypeLegal(RHSHi.getValueType()))
5669 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
5670 RHSHi, CCCode, false, DagCombineInfo, dl);
5671 if (!HiCmp.getNode())
5672 HiCmp =
5673 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
5674 LHSHi, RHSHi, DAG.getCondCode(CCCode));
5675
5676 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
5677 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
5678
5679 bool EqAllowed = ISD::isTrueWhenEqual(CCCode);
5680
5681 // FIXME: Is the HiCmpC->isOne() here correct for
5682 // ZeroOrNegativeOneBooleanContent.
5683 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
5684 (!EqAllowed &&
5685 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
5686 // For LE / GE, if high part is known false, ignore the low part.
5687 // For LT / GT: if low part is known false, return the high part.
5688 // if high part is known true, ignore the low part.
5689 NewLHS = HiCmp;
5690 NewRHS = SDValue();
5691 return;
5692 }
5693
5694 if (LHSHi == RHSHi) {
5695 // Comparing the low bits is enough.
5696 NewLHS = LoCmp;
5697 NewRHS = SDValue();
5698 return;
5699 }
5700
5701 // Lower with SETCCCARRY if the target supports it.
5702 EVT HiVT = LHSHi.getValueType();
5703 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
5704 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
5705
5706 // FIXME: Make all targets support this, then remove the other lowering.
5707 if (HasSETCCCARRY) {
5708 // SETCCCARRY can detect < and >= directly. For > and <=, flip
5709 // operands and condition code.
5710 bool FlipOperands = false;
5711 switch (CCCode) {
5712 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
5713 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
5714 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
5715 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
5716 default: break;
5717 }
5718 if (FlipOperands) {
5719 std::swap(LHSLo, RHSLo);
5720 std::swap(LHSHi, RHSHi);
5721 }
5722 // Perform a wide subtraction, feeding the carry from the low part into
5723 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
5724 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
5725 // zero or positive iff LHS >= RHS.
5726 EVT LoVT = LHSLo.getValueType();
5727 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
5728 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
5729 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
5730 LHSHi, RHSHi, LowCmp.getValue(1),
5731 DAG.getCondCode(CCCode));
5732 NewLHS = Res;
5733 NewRHS = SDValue();
5734 return;
5735 }
5736
5737 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
5738 false, DagCombineInfo, dl);
5739 if (!NewLHS.getNode())
5740 NewLHS =
5741 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
5742 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
5743 NewRHS = SDValue();
5744}
5745
5746SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
5747 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
5748 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
5749 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5750
5751 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5752 // against zero to select between true and false values.
5753 if (!NewRHS.getNode()) {
5754 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
5755 CCCode = ISD::SETNE;
5756 }
5757
5758 // Update N to have the operands specified.
5759 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
5760 DAG.getCondCode(CCCode), NewLHS, NewRHS,
5761 N->getOperand(4)), 0);
5762}
5763
5764SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
5765 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
5766 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
5767 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5768
5769 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5770 // against zero to select between true and false values.
5771 if (!NewRHS.getNode()) {
5772 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
5773 CCCode = ISD::SETNE;
5774 }
5775
5776 // Update N to have the operands specified.
5777 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
5778 N->getOperand(2), N->getOperand(3),
5779 DAG.getCondCode(CCCode)), 0);
5780}
5781
5782SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
5783 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
5784 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
5785 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5786
5787 // If ExpandSetCCOperands returned a scalar, use it.
5788 if (!NewRHS.getNode()) {
5789 assert(NewLHS.getValueType() == N->getValueType(0) &&
5790 "Unexpected setcc expansion!");
5791 return NewLHS;
5792 }
5793
5794 // Otherwise, update N to have the operands specified.
5795 return SDValue(
5796 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
5797}
5798
5799SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
5800 SDValue LHS = N->getOperand(0);
5801 SDValue RHS = N->getOperand(1);
5802 SDValue Carry = N->getOperand(2);
5803 SDValue Cond = N->getOperand(3);
5804 SDLoc dl = SDLoc(N);
5805
5806 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5807 GetExpandedInteger(LHS, LHSLo, LHSHi);
5808 GetExpandedInteger(RHS, RHSLo, RHSHi);
5809
5810 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high.
5811 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
5812 SDValue LowCmp =
5813 DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry);
5814 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
5815 LowCmp.getValue(1), Cond);
5816}
5817
5818SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
5819 // Split the operand and replace with SPLAT_VECTOR_PARTS.
5820 SDValue Lo, Hi;
5821 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5822 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
5823 Hi);
5824}
5825
5826SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
5827 // The value being shifted is legal, but the shift amount is too big.
5828 // It follows that either the result of the shift is undefined, or the
5829 // upper half of the shift amount is zero. Just use the lower half.
5830 SDValue Lo, Hi;
5831 GetExpandedInteger(N->getOperand(1), Lo, Hi);
5832 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
5833}
5834
5835SDValue DAGTypeLegalizer::ExpandIntOp_CMP(SDNode *N) {
5836 return TLI.expandCMP(N, DAG);
5837}
5838
5839SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
5840 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
5841 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
5842 // constant to valid type.
5843 SDValue Lo, Hi;
5844 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5845 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
5846}
5847
5848SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) {
5849 bool IsStrict = N->isStrictFPOpcode();
5850 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
5851 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
5852 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
5853 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
5854 EVT DstVT = N->getValueType(0);
5855 RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(Op.getValueType(), DstVT)
5856 : RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
5857 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5858 "Don't know how to expand this XINT_TO_FP!");
5860 CallOptions.setIsSigned(true);
5861 std::pair<SDValue, SDValue> Tmp =
5862 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
5863
5864 if (!IsStrict)
5865 return Tmp.first;
5866
5867 ReplaceValueWith(SDValue(N, 1), Tmp.second);
5868 ReplaceValueWith(SDValue(N, 0), Tmp.first);
5869 return SDValue();
5870}
5871
5872SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
5873 assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?");
5874
5875 if (ISD::isNormalStore(N))
5876 return ExpandOp_NormalStore(N, OpNo);
5877
5878 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
5879 assert(OpNo == 1 && "Can only expand the stored value so far");
5880
5881 EVT VT = N->getOperand(1).getValueType();
5882 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5883 SDValue Ch = N->getChain();
5884 SDValue Ptr = N->getBasePtr();
5885 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
5886 AAMDNodes AAInfo = N->getAAInfo();
5887 SDLoc dl(N);
5888 SDValue Lo, Hi;
5889
5890 assert(NVT.isByteSized() && "Expanded type not byte sized!");
5891
5892 if (N->getMemoryVT().bitsLE(NVT)) {
5893 GetExpandedInteger(N->getValue(), Lo, Hi);
5894 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
5895 N->getMemoryVT(), N->getBaseAlign(), MMOFlags,
5896 AAInfo);
5897 }
5898
5899 if (DAG.getDataLayout().isLittleEndian()) {
5900 // Little-endian - low bits are at low addresses.
5901 GetExpandedInteger(N->getValue(), Lo, Hi);
5902
5903 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), N->getBaseAlign(),
5904 MMOFlags, AAInfo);
5905
5906 unsigned ExcessBits =
5907 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
5908 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
5909
5910 // Increment the pointer to the other half.
5911 unsigned IncrementSize = NVT.getSizeInBits()/8;
5912 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
5913 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
5914 N->getPointerInfo().getWithOffset(IncrementSize),
5915 NEVT, N->getBaseAlign(), MMOFlags, AAInfo);
5916 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
5917 }
5918
5919 // Big-endian - high bits are at low addresses. Favor aligned stores at
5920 // the cost of some bit-fiddling.
5921 GetExpandedInteger(N->getValue(), Lo, Hi);
5922
5923 EVT ExtVT = N->getMemoryVT();
5924 unsigned EBytes = ExtVT.getStoreSize();
5925 unsigned IncrementSize = NVT.getSizeInBits()/8;
5926 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5927 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
5928 ExtVT.getSizeInBits() - ExcessBits);
5929
5930 if (ExcessBits < NVT.getSizeInBits()) {
5931 // Transfer high bits from the top of Lo to the bottom of Hi.
5932 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
5933 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
5934 TLI.getPointerTy(DAG.getDataLayout())));
5935 Hi = DAG.getNode(
5936 ISD::OR, dl, NVT, Hi,
5937 DAG.getNode(ISD::SRL, dl, NVT, Lo,
5938 DAG.getConstant(ExcessBits, dl,
5939 TLI.getPointerTy(DAG.getDataLayout()))));
5940 }
5941
5942 // Store both the high bits and maybe some of the low bits.
5943 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
5944 N->getBaseAlign(), MMOFlags, AAInfo);
5945
5946 // Increment the pointer to the other half.
5947 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
5948 // Store the lowest ExcessBits bits in the second half.
5949 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
5950 N->getPointerInfo().getWithOffset(IncrementSize),
5951 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
5952 N->getBaseAlign(), MMOFlags, AAInfo);
5953 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
5954}
5955
5956SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
5957 SDValue InL, InH;
5958 GetExpandedInteger(N->getOperand(0), InL, InH);
5959 // Just truncate the low part of the source.
5960 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
5961}
5962
5963SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
5964 SDLoc dl(N);
5965 SDValue Swap =
5966 DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(),
5967 N->getOperand(0), N->getOperand(2), N->getOperand(1),
5968 cast<AtomicSDNode>(N)->getMemOperand());
5969 return Swap.getValue(1);
5970}
5971
5972SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
5973 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5974 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5975
5976 SDValue Hi; // The upper half is dropped out.
5977 SmallVector<SDValue, 8> NewOps(N->ops());
5978 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi);
5979
5980 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
5981}
5982
5983SDValue DAGTypeLegalizer::ExpandIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo) {
5984 const Function &Fn = DAG.getMachineFunction().getFunction();
5986 "cannot use llvm.write_register with illegal type", Fn,
5987 N->getDebugLoc()));
5988
5989 return N->getOperand(0);
5990}
5991
5992SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
5993 SDLoc dl(N);
5994
5995 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5996 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5997 EVT OutVT = V0.getValueType();
5998
5999 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
6000}
6001
6002SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
6003 SDLoc DL(N);
6004 unsigned Factor = N->getNumOperands();
6005
6006 SmallVector<SDValue, 8> Ops(Factor);
6007 for (unsigned i = 0; i != Factor; i++)
6008 Ops[i] = GetPromotedInteger(N->getOperand(i));
6009
6010 SmallVector<EVT, 8> ResVTs(Factor, Ops[0].getValueType());
6011 SDValue Res = DAG.getNode(N->getOpcode(), DL, DAG.getVTList(ResVTs), Ops);
6012
6013 for (unsigned i = 0; i != Factor; i++)
6014 SetPromotedInteger(SDValue(N, i), Res.getValue(i));
6015
6016 return SDValue();
6017}
6018
6019SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
6020
6021 EVT OutVT = N->getValueType(0);
6022 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6023 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6024 EVT NOutVTElem = NOutVT.getVectorElementType();
6025
6026 SDLoc dl(N);
6027 SDValue BaseIdx = N->getOperand(1);
6028
6029 // TODO: We may be able to use this for types other than scalable
6030 // vectors and fix those tests that expect BUILD_VECTOR to be used
6031 if (OutVT.isScalableVector()) {
6032 SDValue InOp0 = N->getOperand(0);
6033 EVT InVT = InOp0.getValueType();
6034
6035 // Try and extract from a smaller type so that it eventually falls
6036 // into the promotion code below.
6037 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
6038 getTypeAction(InVT) == TargetLowering::TypeLegal) {
6039 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
6040 unsigned NElts = NInVT.getVectorMinNumElements();
6041 uint64_t IdxVal = BaseIdx->getAsZExtVal();
6042
6043 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
6044 DAG.getConstant(alignDown(IdxVal, NElts), dl,
6045 BaseIdx.getValueType()));
6046 SDValue Step2 = DAG.getNode(
6047 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
6048 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
6049 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
6050 }
6051
6052 // Try and extract from a widened type.
6053 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
6054 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
6055 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
6056 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
6057 }
6058
6059 // Promote operands and see if this is handled by target lowering,
6060 // Otherwise, use the BUILD_VECTOR approach below
6061 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
6062 // Collect the (promoted) operands
6063 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
6064
6065 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
6066 assert(PromEltVT.bitsLE(NOutVTElem) &&
6067 "Promoted operand has an element type greater than result");
6068
6069 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
6070 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
6071 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
6072 }
6073 }
6074
6075 if (OutVT.isScalableVector())
6076 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
6077
6078 SDValue InOp0 = N->getOperand(0);
6079 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
6080 InOp0 = GetPromotedInteger(InOp0);
6081
6082 EVT InVT = InOp0.getValueType();
6083 EVT InSVT = InVT.getVectorElementType();
6084
6085 unsigned OutNumElems = OutVT.getVectorNumElements();
6087 Ops.reserve(OutNumElems);
6088 for (unsigned i = 0; i != OutNumElems; ++i) {
6089 // Extract the element from the original vector.
6090 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), BaseIdx,
6091 DAG.getConstant(i, dl, BaseIdx.getValueType()));
6092 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InSVT,
6093 N->getOperand(0), Index);
6094 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
6095 // Insert the converted element to the new vector.
6096 Ops.push_back(Op);
6097 }
6098
6099 return DAG.getBuildVector(NOutVT, dl, Ops);
6100}
6101
6102SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
6103 EVT OutVT = N->getValueType(0);
6104 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6105 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6106
6107 SDLoc dl(N);
6108 SDValue Vec = N->getOperand(0);
6109 SDValue SubVec = N->getOperand(1);
6110 SDValue Idx = N->getOperand(2);
6111
6112 EVT SubVecVT = SubVec.getValueType();
6113 EVT NSubVT =
6115 SubVecVT.getVectorElementCount());
6116
6117 Vec = GetPromotedInteger(Vec);
6118 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);
6119
6120 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
6121}
6122
6123SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
6124 SDLoc dl(N);
6125
6126 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6127 EVT OutVT = V0.getValueType();
6128
6129 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
6130}
6131
6132SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
6133 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
6134 EVT VT = N->getValueType(0);
6135 SDLoc dl(N);
6136
6137 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
6138
6139 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6140 SDValue V1 = GetPromotedInteger(N->getOperand(1));
6141 EVT OutVT = V0.getValueType();
6142
6143 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
6144}
6145
6146SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
6147 EVT OutVT = N->getValueType(0);
6148 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6149 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6150 unsigned NumElems = N->getNumOperands();
6151 EVT NOutVTElem = NOutVT.getVectorElementType();
6152 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT);
6153 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType);
6154 SDLoc dl(N);
6155
6157 Ops.reserve(NumElems);
6158 for (unsigned i = 0; i != NumElems; ++i) {
6159 SDValue Op = N->getOperand(i);
6160 EVT OpVT = Op.getValueType();
6161 // BUILD_VECTOR integer operand types are allowed to be larger than the
6162 // result's element type. This may still be true after the promotion. For
6163 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
6164 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
6165 if (OpVT.bitsLT(NOutVTElem)) {
6166 unsigned ExtOpc = ISD::ANY_EXTEND;
6167 // Attempt to extend constant bool vectors to match target's BooleanContent.
6168 // While not necessary, this improves chances of the constant correctly
6169 // folding with compare results (e.g. for NOT patterns).
6170 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant)
6171 ExtOpc = NOutExtOpc;
6172 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op);
6173 }
6174 Ops.push_back(Op);
6175 }
6176
6177 return DAG.getBuildVector(NOutVT, dl, Ops);
6178}
6179
6180SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) {
6181
6182 SDLoc dl(N);
6183
6184 assert(!N->getOperand(0).getValueType().isVector() &&
6185 "Input must be a scalar");
6186
6187 EVT OutVT = N->getValueType(0);
6188 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6189 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6190 EVT NOutElemVT = NOutVT.getVectorElementType();
6191
6192 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0));
6193 if (N->isVPOpcode())
6194 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op, N->getOperand(1),
6195 N->getOperand(2));
6196
6197 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op);
6198}
6199
6200SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
6201 SDLoc dl(N);
6202 EVT OutVT = N->getValueType(0);
6203 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6204 assert(NOutVT.isScalableVector() &&
6205 "Type must be promoted to a scalable vector type");
6206 const APInt &StepVal = N->getConstantOperandAPInt(0);
6207 return DAG.getStepVector(dl, NOutVT,
6208 StepVal.sext(NOutVT.getScalarSizeInBits()));
6209}
6210
6211SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
6212 SDLoc dl(N);
6213
6214 EVT OutVT = N->getValueType(0);
6215 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6216 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6217
6218 unsigned NumOperands = N->getNumOperands();
6219 unsigned NumOutElem = NOutVT.getVectorMinNumElements();
6220 EVT OutElemTy = NOutVT.getVectorElementType();
6221 if (OutVT.isScalableVector()) {
6222 // Find the largest promoted element type for each of the operands.
6223 SDUse *MaxSizedValue = std::max_element(
6224 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) {
6225 EVT AVT = A.getValueType().getVectorElementType();
6226 EVT BVT = B.getValueType().getVectorElementType();
6227 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
6228 });
6229 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType();
6230
6231 // Then promote all vectors to the largest element type.
6233 for (unsigned I = 0; I < NumOperands; ++I) {
6234 SDValue Op = N->getOperand(I);
6235 EVT OpVT = Op.getValueType();
6236 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger)
6237 Op = GetPromotedInteger(Op);
6238 else
6239 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal &&
6240 "Unhandled legalization type");
6241
6243 MaxElementVT.getScalarSizeInBits())
6244 Op = DAG.getAnyExtOrTrunc(Op, dl,
6245 OpVT.changeVectorElementType(MaxElementVT));
6246 Ops.push_back(Op);
6247 }
6248
6249 // Do the CONCAT on the promoted type and finally truncate to (the promoted)
6250 // NOutVT.
6251 return DAG.getAnyExtOrTrunc(
6253 OutVT.changeVectorElementType(MaxElementVT), Ops),
6254 dl, NOutVT);
6255 }
6256
6257 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
6258 assert(NumElem * NumOperands == NumOutElem &&
6259 "Unexpected number of elements");
6260
6261 // Take the elements from the first vector.
6262 SmallVector<SDValue, 8> Ops(NumOutElem);
6263 for (unsigned i = 0; i < NumOperands; ++i) {
6264 SDValue Op = N->getOperand(i);
6265 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
6266 Op = GetPromotedInteger(Op);
6267 EVT SclrTy = Op.getValueType().getVectorElementType();
6268 assert(NumElem == Op.getValueType().getVectorNumElements() &&
6269 "Unexpected number of elements");
6270
6271 for (unsigned j = 0; j < NumElem; ++j) {
6272 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
6273 DAG.getVectorIdxConstant(j, dl));
6274 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
6275 }
6276 }
6277
6278 return DAG.getBuildVector(NOutVT, dl, Ops);
6279}
6280
6281SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
6282 EVT VT = N->getValueType(0);
6283 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6284 assert(NVT.isVector() && "This type must be promoted to a vector type");
6285
6286 SDLoc dl(N);
6287
6288 // For operands whose TypeAction is to promote, extend the promoted node
6289 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
6290 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
6291 // type..
6292 if (getTypeAction(N->getOperand(0).getValueType())
6294 SDValue Promoted;
6295
6296 switch(N->getOpcode()) {
6298 Promoted = SExtPromotedInteger(N->getOperand(0));
6299 break;
6301 Promoted = ZExtPromotedInteger(N->getOperand(0));
6302 break;
6304 Promoted = GetPromotedInteger(N->getOperand(0));
6305 break;
6306 default:
6307 llvm_unreachable("Node has unexpected Opcode");
6308 }
6309 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
6310 }
6311
6312 // Directly extend to the appropriate transform-to type.
6313 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
6314}
6315
6316SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
6317 EVT VT = N->getValueType(0);
6318 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6319 return DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, SDLoc(N), NVT, N->ops());
6320}
6321
6322SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
6323 EVT VT = N->getValueType(0);
6324 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6325 return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
6326}
6327
6328SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
6329 SDLoc DL(N);
6330 EVT VT = N->getValueType(0);
6331 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6332 SDValue ExtAcc = GetPromotedInteger(N->getOperand(0));
6333 return DAG.getNode(N->getOpcode(), DL, NVT, ExtAcc, N->getOperand(1),
6334 N->getOperand(2));
6335}
6336
6337SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
6338 EVT OutVT = N->getValueType(0);
6339 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6340 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6341
6342 EVT NOutVTElem = NOutVT.getVectorElementType();
6343
6344 SDLoc dl(N);
6345 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6346
6347 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
6348 NOutVTElem, N->getOperand(1));
6349 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
6350 V0, ConvElem, N->getOperand(2));
6351}
6352
6353SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
6354 // The VECREDUCE result size may be larger than the element size, so
6355 // we can simply change the result type.
6356 SDLoc dl(N);
6357 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6358 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
6359}
6360
6361SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
6362 // The VP_REDUCE result size may be larger than the element size, so we can
6363 // simply change the result type. However the start value and result must be
6364 // the same.
6365 SDLoc DL(N);
6366 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
6367 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
6368 N->getOperand(1), N->getOperand(2), N->getOperand(3));
6369}
6370
6371SDValue DAGTypeLegalizer::PromoteIntRes_PATCHPOINT(SDNode *N) {
6372 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6373 SDLoc dl(N);
6374
6375 assert(N->getNumValues() == 3 && "Expected 3 values for PATCHPOINT");
6376 SDVTList VTList = DAG.getVTList({NVT, MVT::Other, MVT::Glue});
6377
6378 SmallVector<SDValue> Ops(N->ops());
6379 SDValue Res = DAG.getNode(ISD::PATCHPOINT, dl, VTList, Ops);
6380
6381 // Replace chain and glue uses with the new patchpoint.
6382 SDValue From[] = {SDValue(N, 1), SDValue(N, 2)};
6383 SDValue To[] = {Res.getValue(1), Res.getValue(2)};
6384 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
6385
6386 return Res.getValue(0);
6387}
6388
6389SDValue DAGTypeLegalizer::PromoteIntRes_READ_REGISTER(SDNode *N) {
6390 const Function &Fn = DAG.getMachineFunction().getFunction();
6392 "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc()));
6393
6394 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
6395 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
6396 return DAG.getPOISON(NVT);
6397}
6398
6399SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
6400 SDLoc dl(N);
6401 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6402 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
6403 TLI.getVectorIdxTy(DAG.getDataLayout()));
6405 V0->getValueType(0).getScalarType(), V0, V1);
6406
6407 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
6408 // element types. If this is the case then we need to expand the outgoing
6409 // value and not truncate it.
6410 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
6411}
6412
6413SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
6414 SDLoc dl(N);
6415 // The result type is equal to the first input operand's type, so the
6416 // type that needs promoting must be the second source vector.
6417 SDValue V0 = N->getOperand(0);
6418 SDValue V1 = GetPromotedInteger(N->getOperand(1));
6419 SDValue Idx = N->getOperand(2);
6420 EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
6423 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
6424 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
6425 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
6426}
6427
6428// FIXME: We wouldn't need this if clang could promote short integers
6429// that are arguments to FAKE_USE.
6430SDValue DAGTypeLegalizer::PromoteIntOp_FAKE_USE(SDNode *N) {
6431 SDLoc dl(N);
6432 SDValue V0 = N->getOperand(0);
6433 SDValue V1 = N->getOperand(1);
6434 EVT InVT1 = V1.getValueType();
6435 SDValue VPromoted =
6436 DAG.getNode(ISD::ANY_EXTEND, dl,
6437 TLI.getTypeToTransformTo(*DAG.getContext(), InVT1), V1);
6438 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), V0, VPromoted);
6439}
6440
6441SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
6442 SDLoc dl(N);
6443 SDValue V0 = GetPromotedInteger(N->getOperand(0));
6444 MVT InVT = V0.getValueType().getSimpleVT();
6446 N->getValueType(0).getVectorNumElements());
6447 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
6448 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
6449}
6450
6451SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
6452 SDLoc dl(N);
6453
6454 EVT ResVT = N->getValueType(0);
6455 unsigned NumElems = N->getNumOperands();
6456
6457 if (ResVT.isScalableVector()) {
6458 SDValue ResVec = DAG.getUNDEF(ResVT);
6459
6460 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6461 SDValue Op = N->getOperand(OpIdx);
6462 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
6463 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
6464 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
6465 }
6466
6467 return ResVec;
6468 }
6469
6470 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
6471
6473 NewOps.reserve(NumElems);
6474
6475 // For each incoming vector
6476 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
6477 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
6478 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
6479 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
6480
6481 for (unsigned i=0; i<NumElem; ++i) {
6482 // Extract element from incoming vector
6483 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
6484 DAG.getVectorIdxConstant(i, dl));
6485 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
6486 NewOps.push_back(Tr);
6487 }
6488 }
6489
6490 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
6491}
6492
6493SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
6494 assert(OpNo > 1);
6495 SDValue Op = N->getOperand(OpNo);
6496
6497 // FIXME: Non-constant operands are not yet handled:
6498 // - https://github.com/llvm/llvm-project/issues/26431
6499 // - https://github.com/llvm/llvm-project/issues/55957
6500 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
6501 if (!CN)
6502 return SDValue();
6503
6504 // Copy operands before the one being expanded.
6505 SmallVector<SDValue> NewOps;
6506 for (unsigned I = 0; I < OpNo; I++)
6507 NewOps.push_back(N->getOperand(I));
6508
6509 EVT Ty = Op.getValueType();
6510 SDLoc DL = SDLoc(N);
6511 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6512 NewOps.push_back(
6513 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6514 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6515 } else {
6516 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6517 return SDValue();
6518 }
6519
6520 // Copy remaining operands.
6521 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6522 NewOps.push_back(N->getOperand(I));
6523
6524 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6525
6526 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6527 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6528
6529 return SDValue(); // Signal that we have replaced the node already.
6530}
6531
6532SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
6533 assert(OpNo >= 7);
6534 SDValue Op = N->getOperand(OpNo);
6535
6536 // FIXME: Non-constant operands are not yet handled:
6537 // - https://github.com/llvm/llvm-project/issues/26431
6538 // - https://github.com/llvm/llvm-project/issues/55957
6539 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
6540 if (!CN)
6541 return SDValue();
6542
6543 // Copy operands before the one being expanded.
6544 SmallVector<SDValue> NewOps;
6545 for (unsigned I = 0; I < OpNo; I++)
6546 NewOps.push_back(N->getOperand(I));
6547
6548 EVT Ty = Op.getValueType();
6549 SDLoc DL = SDLoc(N);
6550 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6551 NewOps.push_back(
6552 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6553 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6554 } else {
6555 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6556 return SDValue();
6557 }
6558
6559 // Copy remaining operands.
6560 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6561 NewOps.push_back(N->getOperand(I));
6562
6563 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6564
6565 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6566 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6567
6568 return SDValue(); // Signal that we have replaced the node already.
6569}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:546
#define I(x, y, z)
Definition: MD5.cpp:58
MachineInstr unsigned OpIdx
const SmallVectorImpl< MachineOperand > & Cond
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
#define LLVM_DEBUG(...)
Definition: Debug.h:119
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition: APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1512
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:936
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:206
unsigned countLeadingOnes() const
Definition: APInt.h:1624
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1182
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:209
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1249
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:219
unsigned countTrailingZeros() const
Definition: APInt.h:1647
unsigned countLeadingZeros() const
Definition: APInt.h:1606
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:985
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1257
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:306
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:296
unsigned countTrailingOnes() const
Definition: APInt.h:1662
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:239
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:851
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1221
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:191
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:154
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
Definition: Constant.h:43
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:198
bool isBigEndian() const
Definition: DataLayout.h:199
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:359
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:720
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:229
LLVM_ABI Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
LLVM_ABI SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
LLVM_ABI SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
LLVM_ABI void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:868
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:498
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
LLVM_ABI std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:707
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:505
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:493
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:511
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:581
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
Definition: SmallVector.h:664
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:684
void push_back(const T &Elt)
Definition: SmallVector.h:414
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1197
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
virtual unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT, EVT ToVT) const
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, const SDValue LHS, const SDValue RHS, SDValue &Lo, SDValue &Hi) const
Calculate full product of LHS and RHS either via a libcall or through brute force expansion of the mu...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed, SDValue &Lo, SDValue &Hi, SDValue LHS, SDValue RHS, SDValue HiLHS=SDValue(), SDValue HiRHS=SDValue()) const
Calculate the product twice the width of LHS and RHS.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:346
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
This class is used to represent a VP_LOAD node.
This class is used to represent a VP_STORE node.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
Definition: TypeSize.h:272
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
Definition: TypeSize.h:280
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:169
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:255
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:126
@ Entry
Definition: COFF.h:862
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:41
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:801
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:256
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:774
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:504
@ POISON
POISON - A poison node.
Definition: ISDOpcodes.h:231
@ PARTIAL_REDUCE_SMLA
Definition: ISDOpcodes.h:1510
@ LOOP_DEPENDENCE_RAW_MASK
Definition: ISDOpcodes.h:1565
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
Definition: ISDOpcodes.h:1401
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1491
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:270
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1379
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:587
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:765
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:387
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1381
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1351
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1382
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:289
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:259
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1141
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:393
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:835
@ VECTOR_FIND_LAST_ACTIVE
Definition: ISDOpcodes.h:1550
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1364
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:862
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:571
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:738
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:892
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:275
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1490
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:505
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:985
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
Definition: ISDOpcodes.h:1424
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1377
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:975
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:249
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1378
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:1018
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:400
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:957
@ PARTIAL_REDUCE_UMLA
Definition: ISDOpcodes.h:1511
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:826
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:706
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:478
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:656
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1298
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:773
@ WRITE_REGISTER
Definition: ISDOpcodes.h:135
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:809
@ STRICT_LROUND
Definition: ISDOpcodes.h:459
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1187
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:347
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1380
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition: ISDOpcodes.h:622
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:682
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:535
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:369
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:778
@ STRICT_FPOWI
Definition: ISDOpcodes.h:433
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1347
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:228
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1492
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:242
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:663
@ GET_ACTIVE_LANE_MASK
Definition: ISDOpcodes.h:1559
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:343
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1335
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1485
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:952
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:988
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:987
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:756
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1376
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:636
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1375
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:601
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:134
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:563
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:832
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:928
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:793
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1448
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1358
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1383
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:379
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:351
@ STRICT_LRINT
Definition: ISDOpcodes.h:461
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:881
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:870
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:718
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:627
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:406
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:960
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:787
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:323
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:477
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
Definition: ISDOpcodes.h:1413
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1493
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:994
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:110
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1373
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:471
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:470
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1374
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:908
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1292
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:498
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:730
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:997
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:726
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:701
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:299
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition: ISDOpcodes.h:672
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:236
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:552
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:53
@ STRICT_LLRINT
Definition: ISDOpcodes.h:462
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:648
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1372
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1025
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition: ISDOpcodes.h:690
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:434
@ STRICT_LLROUND
Definition: ISDOpcodes.h:460
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:903
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Definition: ISDOpcodes.h:1546
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:927
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:838
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1256
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1180
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:815
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:62
@ PARTIAL_REDUCE_SUMLA
Definition: ISDOpcodes.h:1512
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:360
@ AssertZext
Definition: ISDOpcodes.h:63
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition: ISDOpcodes.h:611
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition: ISDOpcodes.h:713
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:333
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:543
@ LOOP_DEPENDENCE_WAR_MASK
Definition: ISDOpcodes.h:1564
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1749
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:1724
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1691
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1671
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
Definition: ISDOpcodes.h:1730
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Definition: ISDOpcodes.h:1736
LLVM_ABI Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:551
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:336
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:288
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:207
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition: Error.cpp:167
@ Success
The lock was released successfully.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ AfterLegalizeTypes
Definition: DAGCombine.h:17
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:390
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:137
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:295
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:345
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:368
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:238
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:354
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:380
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition: ValueTypes.h:407
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:311
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:318
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:287
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:251
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:216
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:174
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:323
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:102
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:331
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:303
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:448
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:80
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:235
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
Definition: KnownBits.h:289
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)