Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1122, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeIntegerTypes.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/build-llvm -ferror-limit 19 -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-26-234817-15343-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type. For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type. For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//
19
20#include "LegalizeTypes.h"
21#include "llvm/Analysis/TargetLibraryInfo.h"
22#include "llvm/IR/DerivedTypes.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/KnownBits.h"
25#include "llvm/Support/raw_ostream.h"
26using namespace llvm;
27
28#define DEBUG_TYPE"legalize-types" "legalize-types"
29
30//===----------------------------------------------------------------------===//
31// Integer Result Promotion
32//===----------------------------------------------------------------------===//
33
34/// PromoteIntegerResult - This method is called when a result of a node is
35/// found to be in need of promotion to a larger type. At this point, the node
36/// may also have invalid operands or may have other results that need
37/// expansion, we just know that (at least) one result needs promotion.
38void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
40 dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
;
41 SDValue Res = SDValue();
42
43 // See if the target wants to custom expand this node.
44 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
45 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Node has been custom expanded, done\n"
; } } while (false)
;
46 return;
47 }
48
49 switch (N->getOpcode()) {
50 default:
51#ifndef NDEBUG
52 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
53 N->dump(&DAG); dbgs() << "\n";
54#endif
55 llvm_unreachable("Do not know how to promote this operator!")::llvm::llvm_unreachable_internal("Do not know how to promote this operator!"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 55)
;
56 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
57 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
58 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
59 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
60 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
61 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
62 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
63 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64 case ISD::CTLZ_ZERO_UNDEF:
65 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
66 case ISD::PARITY:
67 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
68 case ISD::CTTZ_ZERO_UNDEF:
69 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
72 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
73 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
74 break;
75 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
76 break;
77 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
78 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
79 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
80 case ISD::STRICT_FSETCC:
81 case ISD::STRICT_FSETCCS:
82 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
83 case ISD::SMIN:
84 case ISD::SMAX:
85 Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ false);
86 break;
87 case ISD::UMIN:
88 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
89
90 case ISD::SHL:
91 Res = PromoteIntRes_SHL(N, /*IsVP*/ false);
92 break;
93 case ISD::SIGN_EXTEND_INREG:
94 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
95 case ISD::SRA:
96 Res = PromoteIntRes_SRA(N, /*IsVP*/ false);
97 break;
98 case ISD::SRL:
99 Res = PromoteIntRes_SRL(N, /*IsVP*/ false);
100 break;
101 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
102 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
103 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
104 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
105
106 case ISD::EXTRACT_SUBVECTOR:
107 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
108 case ISD::INSERT_SUBVECTOR:
109 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
110 case ISD::VECTOR_REVERSE:
111 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
112 case ISD::VECTOR_SHUFFLE:
113 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
114 case ISD::VECTOR_SPLICE:
115 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
116 case ISD::INSERT_VECTOR_ELT:
117 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
118 case ISD::BUILD_VECTOR:
119 Res = PromoteIntRes_BUILD_VECTOR(N); break;
120 case ISD::SCALAR_TO_VECTOR:
121 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
122 case ISD::SPLAT_VECTOR:
123 Res = PromoteIntRes_SPLAT_VECTOR(N); break;
124 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
125 case ISD::CONCAT_VECTORS:
126 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
127
128 case ISD::ANY_EXTEND_VECTOR_INREG:
129 case ISD::SIGN_EXTEND_VECTOR_INREG:
130 case ISD::ZERO_EXTEND_VECTOR_INREG:
131 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
132
133 case ISD::SIGN_EXTEND:
134 case ISD::ZERO_EXTEND:
135 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
136
137 case ISD::STRICT_FP_TO_SINT:
138 case ISD::STRICT_FP_TO_UINT:
139 case ISD::FP_TO_SINT:
140 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
141
142 case ISD::FP_TO_SINT_SAT:
143 case ISD::FP_TO_UINT_SAT:
144 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
145
146 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
147
148 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
149
150 case ISD::AND:
151 case ISD::OR:
152 case ISD::XOR:
153 case ISD::ADD:
154 case ISD::SUB:
155 case ISD::MUL:
156 Res = PromoteIntRes_SimpleIntBinOp(N, /*IsVP*/ false);
157 break;
158
159 case ISD::SDIV:
160 case ISD::SREM:
161 Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ false);
162 break;
163
164 case ISD::UDIV:
165 case ISD::UREM:
166 Res = PromoteIntRes_ZExtIntBinOp(N, /*IsVP*/ false);
167 break;
168
169 case ISD::SADDO:
170 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
171 case ISD::UADDO:
172 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
173 case ISD::SMULO:
174 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
175
176 case ISD::ADDE:
177 case ISD::SUBE:
178 case ISD::ADDCARRY:
179 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
180
181 case ISD::SADDO_CARRY:
182 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
183
184 case ISD::SADDSAT:
185 case ISD::UADDSAT:
186 case ISD::SSUBSAT:
187 case ISD::USUBSAT:
188 case ISD::SSHLSAT:
189 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
190
191 case ISD::SMULFIX:
192 case ISD::SMULFIXSAT:
193 case ISD::UMULFIX:
194 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
195
196 case ISD::SDIVFIX:
197 case ISD::SDIVFIXSAT:
198 case ISD::UDIVFIX:
199 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
200
201 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
202
203 case ISD::ATOMIC_LOAD:
204 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
205
206 case ISD::ATOMIC_LOAD_ADD:
207 case ISD::ATOMIC_LOAD_SUB:
208 case ISD::ATOMIC_LOAD_AND:
209 case ISD::ATOMIC_LOAD_CLR:
210 case ISD::ATOMIC_LOAD_OR:
211 case ISD::ATOMIC_LOAD_XOR:
212 case ISD::ATOMIC_LOAD_NAND:
213 case ISD::ATOMIC_LOAD_MIN:
214 case ISD::ATOMIC_LOAD_MAX:
215 case ISD::ATOMIC_LOAD_UMIN:
216 case ISD::ATOMIC_LOAD_UMAX:
217 case ISD::ATOMIC_SWAP:
218 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
219
220 case ISD::ATOMIC_CMP_SWAP:
221 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
222 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
223 break;
224
225 case ISD::VECREDUCE_ADD:
226 case ISD::VECREDUCE_MUL:
227 case ISD::VECREDUCE_AND:
228 case ISD::VECREDUCE_OR:
229 case ISD::VECREDUCE_XOR:
230 case ISD::VECREDUCE_SMAX:
231 case ISD::VECREDUCE_SMIN:
232 case ISD::VECREDUCE_UMAX:
233 case ISD::VECREDUCE_UMIN:
234 Res = PromoteIntRes_VECREDUCE(N);
235 break;
236
237 case ISD::VP_REDUCE_ADD:
238 case ISD::VP_REDUCE_MUL:
239 case ISD::VP_REDUCE_AND:
240 case ISD::VP_REDUCE_OR:
241 case ISD::VP_REDUCE_XOR:
242 case ISD::VP_REDUCE_SMAX:
243 case ISD::VP_REDUCE_SMIN:
244 case ISD::VP_REDUCE_UMAX:
245 case ISD::VP_REDUCE_UMIN:
246 Res = PromoteIntRes_VP_REDUCE(N);
247 break;
248
249 case ISD::FREEZE:
250 Res = PromoteIntRes_FREEZE(N);
251 break;
252
253 case ISD::ROTL:
254 case ISD::ROTR:
255 Res = PromoteIntRes_Rotate(N);
256 break;
257
258 case ISD::FSHL:
259 case ISD::FSHR:
260 Res = PromoteIntRes_FunnelShift(N);
261 break;
262
263 case ISD::VP_AND:
264 case ISD::VP_OR:
265 case ISD::VP_XOR:
266 case ISD::VP_ADD:
267 case ISD::VP_SUB:
268 case ISD::VP_MUL:
269 Res = PromoteIntRes_SimpleIntBinOp(N, /*IsVP*/ true);
270 break;
271 case ISD::VP_SDIV:
272 case ISD::VP_SREM:
273 Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ true);
274 break;
275 case ISD::VP_UDIV:
276 case ISD::VP_UREM:
277 Res = PromoteIntRes_ZExtIntBinOp(N, /*IsVP*/ true);
278 break;
279 case ISD::VP_SHL:
280 Res = PromoteIntRes_SHL(N, /*IsVP*/ true);
281 break;
282 case ISD::VP_ASHR:
283 Res = PromoteIntRes_SRA(N, /*IsVP*/ true);
284 break;
285 case ISD::VP_LSHR:
286 Res = PromoteIntRes_SRL(N, /*IsVP*/ true);
287 break;
288 }
289
290 // If the result is null then the sub-method took care of registering it.
291 if (Res.getNode())
292 SetPromotedInteger(SDValue(N, ResNo), Res);
293}
294
295SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
296 unsigned ResNo) {
297 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
298 return GetPromotedInteger(Op);
299}
300
301SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
302 // Sign-extend the new bits, and continue the assertion.
303 SDValue Op = SExtPromotedInteger(N->getOperand(0));
304 return DAG.getNode(ISD::AssertSext, SDLoc(N),
305 Op.getValueType(), Op, N->getOperand(1));
306}
307
308SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
309 // Zero the new bits, and continue the assertion.
310 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
311 return DAG.getNode(ISD::AssertZext, SDLoc(N),
312 Op.getValueType(), Op, N->getOperand(1));
313}
314
315SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
316 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
317 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
318 N->getMemoryVT(), ResVT,
319 N->getChain(), N->getBasePtr(),
320 N->getMemOperand());
321 // Legalize the chain result - switch anything that used the old chain to
322 // use the new one.
323 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
324 return Res;
325}
326
327SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
328 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
329 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
330 N->getMemoryVT(),
331 N->getChain(), N->getBasePtr(),
332 Op2, N->getMemOperand());
333 // Legalize the chain result - switch anything that used the old chain to
334 // use the new one.
335 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
336 return Res;
337}
338
339SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
340 unsigned ResNo) {
341 if (ResNo == 1) {
342 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS)(static_cast <bool> (N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 342, __extension__ __PRETTY_FUNCTION__))
;
343 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
344 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
345
346 // Only use the result of getSetCCResultType if it is legal,
347 // otherwise just use the promoted result type (NVT).
348 if (!TLI.isTypeLegal(SVT))
349 SVT = NVT;
350
351 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
352 SDValue Res = DAG.getAtomicCmpSwap(
353 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
354 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
355 N->getMemOperand());
356 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
357 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
358 return Res.getValue(1);
359 }
360
361 // Op2 is used for the comparison and thus must be extended according to the
362 // target's atomic operations. Op3 is merely stored and so can be left alone.
363 SDValue Op2 = N->getOperand(2);
364 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
365 switch (TLI.getExtendForAtomicCmpSwapArg()) {
366 case ISD::SIGN_EXTEND:
367 Op2 = SExtPromotedInteger(Op2);
368 break;
369 case ISD::ZERO_EXTEND:
370 Op2 = ZExtPromotedInteger(Op2);
371 break;
372 case ISD::ANY_EXTEND:
373 Op2 = GetPromotedInteger(Op2);
374 break;
375 default:
376 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 376)
;
377 }
378
379 SDVTList VTs =
380 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
381 SDValue Res = DAG.getAtomicCmpSwap(
382 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
383 N->getBasePtr(), Op2, Op3, N->getMemOperand());
384 // Update the use to N with the newly created Res.
385 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
386 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
387 return Res;
388}
389
390SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
391 SDValue InOp = N->getOperand(0);
392 EVT InVT = InOp.getValueType();
393 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
394 EVT OutVT = N->getValueType(0);
395 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
396 SDLoc dl(N);
397
398 switch (getTypeAction(InVT)) {
399 case TargetLowering::TypeLegal:
400 break;
401 case TargetLowering::TypePromoteInteger:
402 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
403 // The input promotes to the same size. Convert the promoted value.
404 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
405 break;
406 case TargetLowering::TypeSoftenFloat:
407 // Promote the integer operand by hand.
408 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
409 case TargetLowering::TypeSoftPromoteHalf:
410 // Promote the integer operand by hand.
411 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
412 case TargetLowering::TypePromoteFloat: {
413 // Convert the promoted float by hand.
414 if (!NOutVT.isVector())
415 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
416 break;
417 }
418 case TargetLowering::TypeExpandInteger:
419 case TargetLowering::TypeExpandFloat:
420 break;
421 case TargetLowering::TypeScalarizeVector:
422 // Convert the element to an integer and promote it by hand.
423 if (!NOutVT.isVector())
424 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
425 BitConvertToInteger(GetScalarizedVector(InOp)));
426 break;
427 case TargetLowering::TypeScalarizeScalableVector:
428 report_fatal_error("Scalarization of scalable vectors is not supported.");
429 case TargetLowering::TypeSplitVector: {
430 if (!NOutVT.isVector()) {
431 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
432 // pieces of the input into integers and reassemble in the final type.
433 SDValue Lo, Hi;
434 GetSplitVector(N->getOperand(0), Lo, Hi);
435 Lo = BitConvertToInteger(Lo);
436 Hi = BitConvertToInteger(Hi);
437
438 if (DAG.getDataLayout().isBigEndian())
439 std::swap(Lo, Hi);
440
441 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
442 EVT::getIntegerVT(*DAG.getContext(),
443 NOutVT.getSizeInBits()),
444 JoinIntegers(Lo, Hi));
445 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
446 }
447 break;
448 }
449 case TargetLowering::TypeWidenVector:
450 // The input is widened to the same size. Convert to the widened value.
451 // Make sure that the outgoing value is not a vector, because this would
452 // make us bitcast between two vectors which are legalized in different ways.
453 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
454 SDValue Res =
455 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
456
457 // For big endian targets we need to shift the casted value or the
458 // interesting bits will end up at the wrong place.
459 if (DAG.getDataLayout().isBigEndian()) {
460 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
461 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
462 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!")(static_cast <bool> (ShiftAmt < NOutVT.getSizeInBits
() && "Too large shift amount!") ? void (0) : __assert_fail
("ShiftAmt < NOutVT.getSizeInBits() && \"Too large shift amount!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 462, __extension__ __PRETTY_FUNCTION__))
;
463 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
464 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
465 }
466 return Res;
467 }
468 // If the output type is also a vector and widening it to the same size
469 // as the widened input type would be a legal type, we can widen the bitcast
470 // and handle the promotion after.
471 if (NOutVT.isVector()) {
472 unsigned WidenInSize = NInVT.getSizeInBits();
473 unsigned OutSize = OutVT.getSizeInBits();
474 if (WidenInSize % OutSize == 0) {
475 unsigned Scale = WidenInSize / OutSize;
476 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
477 OutVT.getVectorElementType(),
478 OutVT.getVectorNumElements() * Scale);
479 if (isTypeLegal(WideOutVT)) {
480 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
481 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
482 DAG.getVectorIdxConstant(0, dl));
483 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
484 }
485 }
486 }
487 }
488
489 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
490 CreateStackStoreLoad(InOp, OutVT));
491}
492
493// Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
494// in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
495static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI,
496 SelectionDAG &DAG) {
497 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
498 // If any possible shift value won't fit in the prefered type, just use
499 // something safe. It will be legalized when the shift is expanded.
500 if (!ShiftVT.isVector() &&
501 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
502 ShiftVT = MVT::i32;
503 return ShiftVT;
504}
505
506SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
507 SDValue V = GetPromotedInteger(N->getOperand(0));
508 return DAG.getNode(ISD::FREEZE, SDLoc(N),
509 V.getValueType(), V);
510}
511
512SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
513 SDValue Op = GetPromotedInteger(N->getOperand(0));
514 EVT OVT = N->getValueType(0);
515 EVT NVT = Op.getValueType();
516 SDLoc dl(N);
517
518 // If the larger BSWAP isn't supported by the target, try to expand now.
519 // If we expand later we'll end up with more operations since we lost the
520 // original type. We only do this for scalars since we have a shuffle
521 // based lowering for vectors in LegalizeVectorOps.
522 if (!OVT.isVector() &&
523 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) {
524 if (SDValue Res = TLI.expandBSWAP(N, DAG))
525 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
526 }
527
528 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
529 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
530 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
531 DAG.getConstant(DiffBits, dl, ShiftVT));
532}
533
534SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
535 SDValue Op = GetPromotedInteger(N->getOperand(0));
536 EVT OVT = N->getValueType(0);
537 EVT NVT = Op.getValueType();
538 SDLoc dl(N);
539
540 // If the larger BITREVERSE isn't supported by the target, try to expand now.
541 // If we expand later we'll end up with more operations since we lost the
542 // original type. We only do this for scalars since we have a shuffle
543 // based lowering for vectors in LegalizeVectorOps.
544 if (!OVT.isVector() && OVT.isSimple() &&
545 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) {
546 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
547 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
548 }
549
550 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
551 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
552 return DAG.getNode(ISD::SRL, dl, NVT,
553 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
554 DAG.getConstant(DiffBits, dl, ShiftVT));
555}
556
557SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
558 // The pair element type may be legal, or may not promote to the same type as
559 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
560 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
561 TLI.getTypeToTransformTo(*DAG.getContext(),
562 N->getValueType(0)), JoinIntegers(N->getOperand(0),
563 N->getOperand(1)));
564}
565
566SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
567 EVT VT = N->getValueType(0);
568 // FIXME there is no actual debug info here
569 SDLoc dl(N);
570 // Zero extend things like i1, sign extend everything else. It shouldn't
571 // matter in theory which one we pick, but this tends to give better code?
572 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
573 SDValue Result = DAG.getNode(Opc, dl,
574 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
575 SDValue(N, 0));
576 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?")(static_cast <bool> (isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Result) && \"Didn't constant fold ext?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 576, __extension__ __PRETTY_FUNCTION__))
;
577 return Result;
578}
579
580SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
581 // Zero extend to the promoted type and do the count there.
582 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
583 SDLoc dl(N);
584 EVT OVT = N->getValueType(0);
585 EVT NVT = Op.getValueType();
586 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
587 // Subtract off the extra leading bits in the bigger type.
588 return DAG.getNode(
589 ISD::SUB, dl, NVT, Op,
590 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
591 NVT));
592}
593
594SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
595 // Zero extend to the promoted type and do the count or parity there.
596 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
597 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
598}
599
600SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
601 SDValue Op = GetPromotedInteger(N->getOperand(0));
602 EVT OVT = N->getValueType(0);
603 EVT NVT = Op.getValueType();
604 SDLoc dl(N);
605 if (N->getOpcode() == ISD::CTTZ) {
606 // The count is the same in the promoted type except if the original
607 // value was zero. This can be handled by setting the bit just off
608 // the top of the original type.
609 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
610 OVT.getScalarSizeInBits());
611 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
612 }
613 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
614}
615
616SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
617 SDLoc dl(N);
618 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
619
620 SDValue Op0 = N->getOperand(0);
621 SDValue Op1 = N->getOperand(1);
622
623 // If the input also needs to be promoted, do that first so we can get a
624 // get a good idea for the output type.
625 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
626 == TargetLowering::TypePromoteInteger) {
627 SDValue In = GetPromotedInteger(Op0);
628
629 // If the new type is larger than NVT, use it. We probably won't need to
630 // promote it again.
631 EVT SVT = In.getValueType().getScalarType();
632 if (SVT.bitsGE(NVT)) {
633 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
634 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
635 }
636 }
637
638 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
639}
640
641SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
642 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
643 unsigned NewOpc = N->getOpcode();
644 SDLoc dl(N);
645
646 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
647 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
648 // and SINT conversions are Custom, there is no way to tell which is
649 // preferable. We choose SINT because that's the right thing on PPC.)
650 if (N->getOpcode() == ISD::FP_TO_UINT &&
651 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
652 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
653 NewOpc = ISD::FP_TO_SINT;
654
655 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
656 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
657 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
658 NewOpc = ISD::STRICT_FP_TO_SINT;
659
660 SDValue Res;
661 if (N->isStrictFPOpcode()) {
662 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
663 {N->getOperand(0), N->getOperand(1)});
664 // Legalize the chain result - switch anything that used the old chain to
665 // use the new one.
666 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
667 } else
668 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
669
670 // Assert that the converted value fits in the original type. If it doesn't
671 // (eg: because the value being converted is too big), then the result of the
672 // original operation was undefined anyway, so the assert is still correct.
673 //
674 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
675 // before legalization: fp-to-uint16, 65534. -> 0xfffe
676 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
677 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
678 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
679 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
680 DAG.getValueType(N->getValueType(0).getScalarType()));
681}
682
683SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
684 // Promote the result type, while keeping the original width in Op1.
685 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
686 SDLoc dl(N);
687 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
688 N->getOperand(1));
689}
690
691SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
692 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
693 SDLoc dl(N);
694
695 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
696}
697
698SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
699 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
700 SDLoc dl(N);
701
702 SDValue Res =
703 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
704
705 // Legalize the chain result - switch anything that used the old chain to
706 // use the new one.
707 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
708 return Res;
709}
710
711SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
712 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
713 SDLoc dl(N);
714
715 if (getTypeAction(N->getOperand(0).getValueType())
716 == TargetLowering::TypePromoteInteger) {
717 SDValue Res = GetPromotedInteger(N->getOperand(0));
718 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!")(static_cast <bool> (Res.getValueType().bitsLE(NVT) &&
"Extension doesn't make sense!") ? void (0) : __assert_fail (
"Res.getValueType().bitsLE(NVT) && \"Extension doesn't make sense!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 718, __extension__ __PRETTY_FUNCTION__))
;
719
720 // If the result and operand types are the same after promotion, simplify
721 // to an in-register extension.
722 if (NVT == Res.getValueType()) {
723 // The high bits are not guaranteed to be anything. Insert an extend.
724 if (N->getOpcode() == ISD::SIGN_EXTEND)
725 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
726 DAG.getValueType(N->getOperand(0).getValueType()));
727 if (N->getOpcode() == ISD::ZERO_EXTEND)
728 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
729 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!")(static_cast <bool> (N->getOpcode() == ISD::ANY_EXTEND
&& "Unknown integer extension!") ? void (0) : __assert_fail
("N->getOpcode() == ISD::ANY_EXTEND && \"Unknown integer extension!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 729, __extension__ __PRETTY_FUNCTION__))
;
730 return Res;
731 }
732 }
733
734 // Otherwise, just extend the original operand all the way to the larger type.
735 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
736}
737
738SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
739 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDLoad(N) &&
"Indexed load during type legalization!") ? void (0) : __assert_fail
("ISD::isUNINDEXEDLoad(N) && \"Indexed load during type legalization!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 739, __extension__ __PRETTY_FUNCTION__))
;
740 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
741 ISD::LoadExtType ExtType =
742 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
743 SDLoc dl(N);
744 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
745 N->getMemoryVT(), N->getMemOperand());
746
747 // Legalize the chain result - switch anything that used the old chain to
748 // use the new one.
749 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
750 return Res;
751}
752
753SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
754 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
755 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
756
757 SDLoc dl(N);
758 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
759 N->getOffset(), N->getMask(), ExtPassThru,
760 N->getMemoryVT(), N->getMemOperand(),
761 N->getAddressingMode(), ISD::EXTLOAD);
762 // Legalize the chain result - switch anything that used the old chain to
763 // use the new one.
764 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
765 return Res;
766}
767
768SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
769 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
770 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
771 assert(NVT == ExtPassThru.getValueType() &&(static_cast <bool> (NVT == ExtPassThru.getValueType() &&
"Gather result type and the passThru argument type should be the same"
) ? void (0) : __assert_fail ("NVT == ExtPassThru.getValueType() && \"Gather result type and the passThru argument type should be the same\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 772, __extension__ __PRETTY_FUNCTION__))
772 "Gather result type and the passThru argument type should be the same")(static_cast <bool> (NVT == ExtPassThru.getValueType() &&
"Gather result type and the passThru argument type should be the same"
) ? void (0) : __assert_fail ("NVT == ExtPassThru.getValueType() && \"Gather result type and the passThru argument type should be the same\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 772, __extension__ __PRETTY_FUNCTION__))
;
773
774 ISD::LoadExtType ExtType = N->getExtensionType();
775 if (ExtType == ISD::NON_EXTLOAD)
776 ExtType = ISD::EXTLOAD;
777
778 SDLoc dl(N);
779 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
780 N->getIndex(), N->getScale() };
781 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
782 N->getMemoryVT(), dl, Ops,
783 N->getMemOperand(), N->getIndexType(),
784 ExtType);
785 // Legalize the chain result - switch anything that used the old chain to
786 // use the new one.
787 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
788 return Res;
789}
790
791/// Promote the overflow flag of an overflowing arithmetic node.
792SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
793 // Change the return type of the boolean result while obeying
794 // getSetCCResultType.
795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
796 EVT VT = N->getValueType(0);
797 EVT SVT = getSetCCResultType(VT);
798 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
799 unsigned NumOps = N->getNumOperands();
800 assert(NumOps <= 3 && "Too many operands")(static_cast <bool> (NumOps <= 3 && "Too many operands"
) ? void (0) : __assert_fail ("NumOps <= 3 && \"Too many operands\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 800, __extension__ __PRETTY_FUNCTION__))
;
801 if (NumOps == 3)
802 Ops[2] = N->getOperand(2);
803
804 SDLoc dl(N);
805 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
806 makeArrayRef(Ops, NumOps));
807
808 // Modified the sum result - switch anything that used the old sum to use
809 // the new one.
810 ReplaceValueWith(SDValue(N, 0), Res);
811
812 // Convert to the expected type.
813 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
814}
815
816SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
817 // If the promoted type is legal, we can convert this to:
818 // 1. ANY_EXTEND iN to iM
819 // 2. SHL by M-N
820 // 3. [US][ADD|SUB|SHL]SAT
821 // 4. L/ASHR by M-N
822 // Else it is more efficient to convert this to a min and a max
823 // operation in the higher precision arithmetic.
824 SDLoc dl(N);
825 SDValue Op1 = N->getOperand(0);
826 SDValue Op2 = N->getOperand(1);
827 unsigned OldBits = Op1.getScalarValueSizeInBits();
828
829 unsigned Opcode = N->getOpcode();
830 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
831
832 SDValue Op1Promoted, Op2Promoted;
833 if (IsShift) {
834 Op1Promoted = GetPromotedInteger(Op1);
835 Op2Promoted = ZExtPromotedInteger(Op2);
836 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
837 Op1Promoted = ZExtPromotedInteger(Op1);
838 Op2Promoted = ZExtPromotedInteger(Op2);
839 } else {
840 Op1Promoted = SExtPromotedInteger(Op1);
841 Op2Promoted = SExtPromotedInteger(Op2);
842 }
843 EVT PromotedType = Op1Promoted.getValueType();
844 unsigned NewBits = PromotedType.getScalarSizeInBits();
845
846 if (Opcode == ISD::UADDSAT) {
847 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
848 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
849 SDValue Add =
850 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
851 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
852 }
853
854 // USUBSAT can always be promoted as long as we have zero-extended the args.
855 if (Opcode == ISD::USUBSAT)
856 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
857 Op2Promoted);
858
859 // Shift cannot use a min/max expansion, we can't detect overflow if all of
860 // the bits have been shifted out.
861 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
862 unsigned ShiftOp;
863 switch (Opcode) {
864 case ISD::SADDSAT:
865 case ISD::SSUBSAT:
866 case ISD::SSHLSAT:
867 ShiftOp = ISD::SRA;
868 break;
869 case ISD::USHLSAT:
870 ShiftOp = ISD::SRL;
871 break;
872 default:
873 llvm_unreachable("Expected opcode to be signed or unsigned saturation "::llvm::llvm_unreachable_internal("Expected opcode to be signed or unsigned saturation "
"addition, subtraction or left shift", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 874)
874 "addition, subtraction or left shift")::llvm::llvm_unreachable_internal("Expected opcode to be signed or unsigned saturation "
"addition, subtraction or left shift", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 874)
;
875 }
876
877 unsigned SHLAmount = NewBits - OldBits;
878 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
879 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
880 Op1Promoted =
881 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
882 if (!IsShift)
883 Op2Promoted =
884 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
885
886 SDValue Result =
887 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
888 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
889 }
890
891 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
892 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
893 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
894 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
895 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
896 SDValue Result =
897 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
898 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
899 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
900 return Result;
901}
902
903SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
904 // Can just promote the operands then continue with operation.
905 SDLoc dl(N);
906 SDValue Op1Promoted, Op2Promoted;
907 bool Signed =
908 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
909 bool Saturating =
910 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
911 if (Signed) {
912 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
913 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
914 } else {
915 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
916 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
917 }
918 EVT OldType = N->getOperand(0).getValueType();
919 EVT PromotedType = Op1Promoted.getValueType();
920 unsigned DiffSize =
921 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
922
923 if (Saturating) {
924 // Promoting the operand and result values changes the saturation width,
925 // which is extends the values that we clamp to on saturation. This could be
926 // resolved by shifting one of the operands the same amount, which would
927 // also shift the result we compare against, then shifting back.
928 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
929 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
930 DAG.getConstant(DiffSize, dl, ShiftTy));
931 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
932 Op2Promoted, N->getOperand(2));
933 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
934 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
935 DAG.getConstant(DiffSize, dl, ShiftTy));
936 }
937 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
938 N->getOperand(2));
939}
940
941static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
942 unsigned SatW, bool Signed,
943 const TargetLowering &TLI,
944 SelectionDAG &DAG) {
945 EVT VT = V.getValueType();
946 unsigned VTW = VT.getScalarSizeInBits();
947
948 if (!Signed) {
949 // Saturate to the unsigned maximum by getting the minimum of V and the
950 // maximum.
951 return DAG.getNode(ISD::UMIN, dl, VT, V,
952 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
953 dl, VT));
954 }
955
956 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
957 // signed minimum of it and V.
958 V = DAG.getNode(ISD::SMIN, dl, VT, V,
959 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
960 dl, VT));
961 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
962 // signed maximum of it and V.
963 V = DAG.getNode(ISD::SMAX, dl, VT, V,
964 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
965 dl, VT));
966 return V;
967}
968
969static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
970 unsigned Scale, const TargetLowering &TLI,
971 SelectionDAG &DAG, unsigned SatW = 0) {
972 EVT VT = LHS.getValueType();
973 unsigned VTSize = VT.getScalarSizeInBits();
974 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
975 N->getOpcode() == ISD::SDIVFIXSAT;
976 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
977 N->getOpcode() == ISD::UDIVFIXSAT;
978
979 SDLoc dl(N);
980 // Widen the types by a factor of two. This is guaranteed to expand, since it
981 // will always have enough high bits in the LHS to shift into.
982 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
983 if (VT.isVector())
984 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
985 VT.getVectorElementCount());
986 if (Signed) {
987 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
988 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
989 } else {
990 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
991 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
992 }
993
994 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
995 DAG);
996 assert(Res && "Expanding DIVFIX with wide type failed?")(static_cast <bool> (Res && "Expanding DIVFIX with wide type failed?"
) ? void (0) : __assert_fail ("Res && \"Expanding DIVFIX with wide type failed?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 996, __extension__ __PRETTY_FUNCTION__))
;
997 if (Saturating) {
998 // If the caller has told us to saturate at something less, use that width
999 // instead of the type before doubling. However, it cannot be more than
1000 // what we just widened!
1001 assert(SatW <= VTSize &&(static_cast <bool> (SatW <= VTSize && "Tried to saturate to more than the original type?"
) ? void (0) : __assert_fail ("SatW <= VTSize && \"Tried to saturate to more than the original type?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
1002 "Tried to saturate to more than the original type?")(static_cast <bool> (SatW <= VTSize && "Tried to saturate to more than the original type?"
) ? void (0) : __assert_fail ("SatW <= VTSize && \"Tried to saturate to more than the original type?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__))
;
1003 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1004 TLI, DAG);
1005 }
1006 return DAG.getZExtOrTrunc(Res, dl, VT);
1007}
1008
1009SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1010 SDLoc dl(N);
1011 SDValue Op1Promoted, Op2Promoted;
1012 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1013 N->getOpcode() == ISD::SDIVFIXSAT;
1014 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1015 N->getOpcode() == ISD::UDIVFIXSAT;
1016 if (Signed) {
1017 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1018 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1019 } else {
1020 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1021 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1022 }
1023 EVT PromotedType = Op1Promoted.getValueType();
1024 unsigned Scale = N->getConstantOperandVal(2);
1025
1026 // If the type is already legal and the operation is legal in that type, we
1027 // should not early expand.
1028 if (TLI.isTypeLegal(PromotedType)) {
1029 TargetLowering::LegalizeAction Action =
1030 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1031 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1032 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
1033 unsigned Diff = PromotedType.getScalarSizeInBits() -
1034 N->getValueType(0).getScalarSizeInBits();
1035 if (Saturating)
1036 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1037 DAG.getConstant(Diff, dl, ShiftTy));
1038 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1039 Op2Promoted, N->getOperand(2));
1040 if (Saturating)
1041 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1042 DAG.getConstant(Diff, dl, ShiftTy));
1043 return Res;
1044 }
1045 }
1046
1047 // See if we can perform the division in this type without expanding.
1048 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1049 Op2Promoted, Scale, DAG)) {
1050 if (Saturating)
1051 Res = SaturateWidenedDIVFIX(Res, dl,
1052 N->getValueType(0).getScalarSizeInBits(),
1053 Signed, TLI, DAG);
1054 return Res;
1055 }
1056 // If we cannot, expand it to twice the type width. If we are saturating, give
1057 // it the original width as a saturating width so we don't need to emit
1058 // two saturations.
1059 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1060 N->getValueType(0).getScalarSizeInBits());
1061}
1062
1063SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1064 if (ResNo == 1)
1065 return PromoteIntRes_Overflow(N);
1066
1067 // The operation overflowed iff the result in the larger type is not the
1068 // sign extension of its truncation to the original type.
1069 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1070 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1071 EVT OVT = N->getOperand(0).getValueType();
1072 EVT NVT = LHS.getValueType();
1073 SDLoc dl(N);
1074
1075 // Do the arithmetic in the larger type.
1076 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1077 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1078
1079 // Calculate the overflow flag: sign extend the arithmetic result from
1080 // the original type.
1081 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1082 DAG.getValueType(OVT));
1083 // Overflowed if and only if this is not equal to Res.
1084 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1085
1086 // Use the calculated overflow everywhere.
1087 ReplaceValueWith(SDValue(N, 1), Ofl);
1088
1089 return Res;
1090}
1091
1092SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
1093 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1094 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1095 return DAG.getSelect(SDLoc(N),
1096 LHS.getValueType(), N->getOperand(0), LHS, RHS);
1097}
1098
1099SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
1100 SDValue Mask = N->getOperand(0);
1101
1102 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1103 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1104 return DAG.getNode(ISD::VSELECT, SDLoc(N),
1105 LHS.getValueType(), Mask, LHS, RHS);
1106}
1107
1108SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1109 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1110 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1111 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1112 LHS.getValueType(), N->getOperand(0),
1113 N->getOperand(1), LHS, RHS, N->getOperand(4));
1114}
1115
1116SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1117 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1118 EVT InVT = N->getOperand(OpNo).getValueType();
1119 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1120
1121 EVT SVT = getSetCCResultType(InVT);
1122
1123 // If we got back a type that needs to be promoted, this likely means the
1124 // the input type also needs to be promoted. So get the promoted type for
1125 // the input and try the query again.
1126 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1127 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1128 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1129 SVT = getSetCCResultType(InVT);
1130 } else {
1131 // Input type isn't promoted, just use the default promoted type.
1132 SVT = NVT;
1133 }
1134 }
1135
1136 SDLoc dl(N);
1137 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&(static_cast <bool> (SVT.isVector() == N->getOperand
(OpNo).getValueType().isVector() && "Vector compare must return a vector result!"
) ? void (0) : __assert_fail ("SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && \"Vector compare must return a vector result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1138, __extension__ __PRETTY_FUNCTION__))
1138 "Vector compare must return a vector result!")(static_cast <bool> (SVT.isVector() == N->getOperand
(OpNo).getValueType().isVector() && "Vector compare must return a vector result!"
) ? void (0) : __assert_fail ("SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && \"Vector compare must return a vector result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1138, __extension__ __PRETTY_FUNCTION__))
;
1139
1140 // Get the SETCC result using the canonical SETCC type.
1141 SDValue SetCC;
1142 if (N->isStrictFPOpcode()) {
1143 EVT VTs[] = {SVT, MVT::Other};
1144 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1145 N->getOperand(2), N->getOperand(3)};
1146 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1147 // Legalize the chain result - switch anything that used the old chain to
1148 // use the new one.
1149 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1150 } else
1151 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1152 N->getOperand(1), N->getOperand(2));
1153
1154 // Convert to the expected type.
1155 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1156}
1157
1158SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N, bool IsVP) {
1159 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1160 SDValue RHS = N->getOperand(1);
1161 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1162 RHS = ZExtPromotedInteger(RHS);
1163 if (!IsVP)
1164 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1165 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1166 N->getOperand(2), N->getOperand(3));
1167}
1168
1169SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1170 SDValue Op = GetPromotedInteger(N->getOperand(0));
1171 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1172 Op.getValueType(), Op, N->getOperand(1));
1173}
1174
1175SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N, bool IsVP) {
1176 // The input may have strange things in the top bits of the registers, but
1177 // these operations don't care. They may have weird bits going out, but
1178 // that too is okay if they are integer operations.
1179 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1180 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1181 if (!IsVP)
1182 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1183 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1184 N->getOperand(2), N->getOperand(3));
1185}
1186
1187SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N, bool IsVP) {
1188 // Sign extend the input.
1189 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1190 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1191 if (!IsVP)
1192 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1193 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1194 N->getOperand(2), N->getOperand(3));
1195}
1196
1197SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N, bool IsVP) {
1198 // Zero extend the input.
1199 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1200 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1201 if (!IsVP)
1202 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1203 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1204 N->getOperand(2), N->getOperand(3));
1205}
1206
1207SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1208 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1209 // whatever is best for the target.
1210 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1211 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1212 return DAG.getNode(N->getOpcode(), SDLoc(N),
1213 LHS.getValueType(), LHS, RHS);
1214}
1215
1216SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N, bool IsVP) {
1217 // The input value must be properly sign extended.
1218 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1219 SDValue RHS = N->getOperand(1);
1220 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1221 RHS = ZExtPromotedInteger(RHS);
1222 if (!IsVP)
1223 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1224 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1225 N->getOperand(2), N->getOperand(3));
1226}
1227
1228SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N, bool IsVP) {
1229 // The input value must be properly zero extended.
1230 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1231 SDValue RHS = N->getOperand(1);
1232 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1233 RHS = ZExtPromotedInteger(RHS);
1234 if (!IsVP)
1235 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1236 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1237 N->getOperand(2), N->getOperand(3));
1238}
1239
1240SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1241 // Lower the rotate to shifts and ORs which can be promoted.
1242 SDValue Res;
1243 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
1244 ReplaceValueWith(SDValue(N, 0), Res);
1245 return SDValue();
1246}
1247
1248SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1249 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1250 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1251 SDValue Amount = GetPromotedInteger(N->getOperand(2));
1252
1253 SDLoc DL(N);
1254 EVT OldVT = N->getOperand(0).getValueType();
1255 EVT VT = Lo.getValueType();
1256 unsigned Opcode = N->getOpcode();
1257 bool IsFSHR = Opcode == ISD::FSHR;
1258 unsigned OldBits = OldVT.getScalarSizeInBits();
1259 unsigned NewBits = VT.getScalarSizeInBits();
1260
1261 // Amount has to be interpreted modulo the old bit width.
1262 Amount =
1263 DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
1264
1265 // If the promoted type is twice the size (or more), then we use the
1266 // traditional funnel 'double' shift codegen. This isn't necessary if the
1267 // shift amount is constant.
1268 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1269 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1270 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
1271 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1272 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1273 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1274 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1275 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1276 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
1277 if (!IsFSHR)
1278 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1279 return Res;
1280 }
1281
1282 // Shift Lo up to occupy the upper bits of the promoted type.
1283 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT);
1284 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1285
1286 // Increase Amount to shift the result into the lower bits of the promoted
1287 // type.
1288 if (IsFSHR)
1289 Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
1290
1291 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
1292}
1293
1294SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1295 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1296 SDValue Res;
1297 SDValue InOp = N->getOperand(0);
1298 SDLoc dl(N);
1299
1300 switch (getTypeAction(InOp.getValueType())) {
1301 default: llvm_unreachable("Unknown type action!")::llvm::llvm_unreachable_internal("Unknown type action!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1301)
;
1302 case TargetLowering::TypeLegal:
1303 case TargetLowering::TypeExpandInteger:
1304 Res = InOp;
1305 break;
1306 case TargetLowering::TypePromoteInteger:
1307 Res = GetPromotedInteger(InOp);
1308 break;
1309 case TargetLowering::TypeSplitVector: {
1310 EVT InVT = InOp.getValueType();
1311 assert(InVT.isVector() && "Cannot split scalar types")(static_cast <bool> (InVT.isVector() && "Cannot split scalar types"
) ? void (0) : __assert_fail ("InVT.isVector() && \"Cannot split scalar types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1311, __extension__ __PRETTY_FUNCTION__))
;
1312 ElementCount NumElts = InVT.getVectorElementCount();
1313 assert(NumElts == NVT.getVectorElementCount() &&(static_cast <bool> (NumElts == NVT.getVectorElementCount
() && "Dst and Src must have the same number of elements"
) ? void (0) : __assert_fail ("NumElts == NVT.getVectorElementCount() && \"Dst and Src must have the same number of elements\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1314, __extension__ __PRETTY_FUNCTION__))
1314 "Dst and Src must have the same number of elements")(static_cast <bool> (NumElts == NVT.getVectorElementCount
() && "Dst and Src must have the same number of elements"
) ? void (0) : __assert_fail ("NumElts == NVT.getVectorElementCount() && \"Dst and Src must have the same number of elements\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1314, __extension__ __PRETTY_FUNCTION__))
;
1315 assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&(static_cast <bool> (isPowerOf2_32(NumElts.getKnownMinValue
()) && "Promoted vector type must be a power of two")
? void (0) : __assert_fail ("isPowerOf2_32(NumElts.getKnownMinValue()) && \"Promoted vector type must be a power of two\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1316, __extension__ __PRETTY_FUNCTION__))
1316 "Promoted vector type must be a power of two")(static_cast <bool> (isPowerOf2_32(NumElts.getKnownMinValue
()) && "Promoted vector type must be a power of two")
? void (0) : __assert_fail ("isPowerOf2_32(NumElts.getKnownMinValue()) && \"Promoted vector type must be a power of two\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1316, __extension__ __PRETTY_FUNCTION__))
;
1317
1318 SDValue EOp1, EOp2;
1319 GetSplitVector(InOp, EOp1, EOp2);
1320
1321 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1322 NumElts.divideCoefficientBy(2));
1323 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1324 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1325
1326 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1327 }
1328 case TargetLowering::TypeWidenVector: {
1329 SDValue WideInOp = GetWidenedVector(InOp);
1330
1331 // Truncate widened InOp.
1332 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1333 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1334 N->getValueType(0).getScalarType(), NumElem);
1335 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1336
1337 // Zero extend so that the elements are of same type as those of NVT
1338 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1339 NumElem);
1340 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1341
1342 // Extract the low NVT subvector.
1343 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1344 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1345 }
1346 }
1347
1348 // Truncate to NVT instead of VT
1349 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1350}
1351
1352SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1353 if (ResNo == 1)
1354 return PromoteIntRes_Overflow(N);
1355
1356 // The operation overflowed iff the result in the larger type is not the
1357 // zero extension of its truncation to the original type.
1358 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1359 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1360 EVT OVT = N->getOperand(0).getValueType();
1361 EVT NVT = LHS.getValueType();
1362 SDLoc dl(N);
1363
1364 // Do the arithmetic in the larger type.
1365 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1366 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1367
1368 // Calculate the overflow flag: zero extend the arithmetic result from
1369 // the original type.
1370 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1371 // Overflowed if and only if this is not equal to Res.
1372 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1373
1374 // Use the calculated overflow everywhere.
1375 ReplaceValueWith(SDValue(N, 1), Ofl);
1376
1377 return Res;
1378}
1379
1380// Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1381// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1382// the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
1383SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1384 if (ResNo == 1)
1385 return PromoteIntRes_Overflow(N);
1386
1387 // We need to sign-extend the operands so the carry value computed by the
1388 // wide operation will be equivalent to the carry value computed by the
1389 // narrow operation.
1390 // An ADDCARRY can generate carry only if any of the operands has its
1391 // most significant bit set. Sign extension propagates the most significant
1392 // bit into the higher bits which means the extra bit that the narrow
1393 // addition would need (i.e. the carry) will be propagated through the higher
1394 // bits of the wide addition.
1395 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1396 // preserved by sign extension.
1397 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1398 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1399
1400 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1401
1402 // Do the arithmetic in the wide type.
1403 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1404 LHS, RHS, N->getOperand(2));
1405
1406 // Update the users of the original carry/borrow value.
1407 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1408
1409 return SDValue(Res.getNode(), 0);
1410}
1411
1412SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1413 unsigned ResNo) {
1414 assert(ResNo == 1 && "Don't know how to promote other results yet.")(static_cast <bool> (ResNo == 1 && "Don't know how to promote other results yet."
) ? void (0) : __assert_fail ("ResNo == 1 && \"Don't know how to promote other results yet.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1414, __extension__ __PRETTY_FUNCTION__))
;
1415 return PromoteIntRes_Overflow(N);
1416}
1417
1418SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1419 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1420 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1421}
1422
1423SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1424 // Promote the overflow bit trivially.
1425 if (ResNo == 1)
1426 return PromoteIntRes_Overflow(N);
1427
1428 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1429 SDLoc DL(N);
1430 EVT SmallVT = LHS.getValueType();
1431
1432 // To determine if the result overflowed in a larger type, we extend the
1433 // input to the larger type, do the multiply (checking if it overflows),
1434 // then also check the high bits of the result to see if overflow happened
1435 // there.
1436 if (N->getOpcode() == ISD::SMULO) {
1437 LHS = SExtPromotedInteger(LHS);
1438 RHS = SExtPromotedInteger(RHS);
1439 } else {
1440 LHS = ZExtPromotedInteger(LHS);
1441 RHS = ZExtPromotedInteger(RHS);
1442 }
1443 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1444 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1445
1446 // Overflow occurred if it occurred in the larger type, or if the high part
1447 // of the result does not zero/sign-extend the low part. Check this second
1448 // possibility first.
1449 SDValue Overflow;
1450 if (N->getOpcode() == ISD::UMULO) {
1451 // Unsigned overflow occurred if the high part is non-zero.
1452 unsigned Shift = SmallVT.getScalarSizeInBits();
1453 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1454 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1455 DAG.getConstant(Shift, DL, ShiftTy));
1456 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1457 DAG.getConstant(0, DL, Hi.getValueType()),
1458 ISD::SETNE);
1459 } else {
1460 // Signed overflow occurred if the high part does not sign extend the low.
1461 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1462 Mul, DAG.getValueType(SmallVT));
1463 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1464 }
1465
1466 // The only other way for overflow to occur is if the multiplication in the
1467 // larger type itself overflowed.
1468 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1469 SDValue(Mul.getNode(), 1));
1470
1471 // Use the calculated overflow everywhere.
1472 ReplaceValueWith(SDValue(N, 1), Overflow);
1473 return Mul;
1474}
1475
1476SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1477 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1478 N->getValueType(0)));
1479}
1480
1481SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1482 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1483
1484 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1485 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1486}
1487
1488SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1489 SDValue Chain = N->getOperand(0); // Get the chain.
1490 SDValue Ptr = N->getOperand(1); // Get the pointer.
1491 EVT VT = N->getValueType(0);
1492 SDLoc dl(N);
1493
1494 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1495 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1496 // The argument is passed as NumRegs registers of type RegVT.
1497
1498 SmallVector<SDValue, 8> Parts(NumRegs);
1499 for (unsigned i = 0; i < NumRegs; ++i) {
1500 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1501 N->getConstantOperandVal(3));
1502 Chain = Parts[i].getValue(1);
1503 }
1504
1505 // Handle endianness of the load.
1506 if (DAG.getDataLayout().isBigEndian())
1507 std::reverse(Parts.begin(), Parts.end());
1508
1509 // Assemble the parts in the promoted type.
1510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1511 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1512 for (unsigned i = 1; i < NumRegs; ++i) {
1513 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1514 // Shift it to the right position and "or" it in.
1515 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1516 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1517 TLI.getPointerTy(DAG.getDataLayout())));
1518 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1519 }
1520
1521 // Modified the chain result - switch anything that used the old chain to
1522 // use the new one.
1523 ReplaceValueWith(SDValue(N, 1), Chain);
1524
1525 return Res;
1526}
1527
1528//===----------------------------------------------------------------------===//
1529// Integer Operand Promotion
1530//===----------------------------------------------------------------------===//
1531
1532/// PromoteIntegerOperand - This method is called when the specified operand of
1533/// the specified node is found to need promotion. At this point, all of the
1534/// result types of the node are known to be legal, but other operands of the
1535/// node may need promotion or expansion as well as the specified one.
1536bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1537 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
1538 dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
;
1539 SDValue Res = SDValue();
1540 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1541 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Node has been custom lowered, done\n"
; } } while (false)
;
1542 return false;
1543 }
1544
1545 switch (N->getOpcode()) {
1546 default:
1547 #ifndef NDEBUG
1548 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1549 N->dump(&DAG); dbgs() << "\n";
1550 #endif
1551 llvm_unreachable("Do not know how to promote this operator's operand!")::llvm::llvm_unreachable_internal("Do not know how to promote this operator's operand!"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1551)
;
1552
1553 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1554 case ISD::ATOMIC_STORE:
1555 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1556 break;
1557 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1558 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1559 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1560 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1561 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1562 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1563 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1564 case ISD::INSERT_VECTOR_ELT:
1565 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1566 case ISD::SCALAR_TO_VECTOR:
1567 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1568 case ISD::SPLAT_VECTOR:
1569 Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1570 case ISD::VSELECT:
1571 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1572 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1573 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1574 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1575 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1576 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1577 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1578 OpNo); break;
1579 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1580 OpNo); break;
1581 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1582 OpNo); break;
1583 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1584 OpNo); break;
1585 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1586 OpNo); break;
1587 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1588 case ISD::FP16_TO_FP:
1589 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1590 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1591 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1592 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1593
1594 case ISD::SHL:
1595 case ISD::SRA:
1596 case ISD::SRL:
1597 case ISD::ROTL:
1598 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1599
1600 case ISD::SADDO_CARRY:
1601 case ISD::SSUBO_CARRY:
1602 case ISD::ADDCARRY:
1603 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1604
1605 case ISD::FRAMEADDR:
1606 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1607
1608 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1609
1610 case ISD::SMULFIX:
1611 case ISD::SMULFIXSAT:
1612 case ISD::UMULFIX:
1613 case ISD::UMULFIXSAT:
1614 case ISD::SDIVFIX:
1615 case ISD::SDIVFIXSAT:
1616 case ISD::UDIVFIX:
1617 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1618
1619 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1620
1621 case ISD::VECREDUCE_ADD:
1622 case ISD::VECREDUCE_MUL:
1623 case ISD::VECREDUCE_AND:
1624 case ISD::VECREDUCE_OR:
1625 case ISD::VECREDUCE_XOR:
1626 case ISD::VECREDUCE_SMAX:
1627 case ISD::VECREDUCE_SMIN:
1628 case ISD::VECREDUCE_UMAX:
1629 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1630 case ISD::VP_REDUCE_ADD:
1631 case ISD::VP_REDUCE_MUL:
1632 case ISD::VP_REDUCE_AND:
1633 case ISD::VP_REDUCE_OR:
1634 case ISD::VP_REDUCE_XOR:
1635 case ISD::VP_REDUCE_SMAX:
1636 case ISD::VP_REDUCE_SMIN:
1637 case ISD::VP_REDUCE_UMAX:
1638 case ISD::VP_REDUCE_UMIN:
1639 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1640 break;
1641
1642 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1643 }
1644
1645 // If the result is null, the sub-method took care of registering results etc.
1646 if (!Res.getNode()) return false;
1647
1648 // If the result is N, the sub-method updated N in place. Tell the legalizer
1649 // core about this.
1650 if (Res.getNode() == N)
1651 return true;
1652
1653 const bool IsStrictFp = N->isStrictFPOpcode();
1654 assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) &&
"Invalid operand expansion") ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) && \"Invalid operand expansion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1656, __extension__ __PRETTY_FUNCTION__))
1655 N->getNumValues() == (IsStrictFp ? 2 : 1) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) &&
"Invalid operand expansion") ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) && \"Invalid operand expansion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1656, __extension__ __PRETTY_FUNCTION__))
1656 "Invalid operand expansion")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) &&
"Invalid operand expansion") ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) && \"Invalid operand expansion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1656, __extension__ __PRETTY_FUNCTION__))
;
1657 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Replacing: "; N->dump
(&DAG); dbgs() << " with: "; Res.dump(); } } while
(false)
1658 Res.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Replacing: "; N->dump
(&DAG); dbgs() << " with: "; Res.dump(); } } while
(false)
;
1659
1660 ReplaceValueWith(SDValue(N, 0), Res);
1661 if (IsStrictFp)
1662 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1663
1664 return false;
1665}
1666
1667/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1668/// shared among BR_CC, SELECT_CC, and SETCC handlers.
1669void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1670 ISD::CondCode CCCode) {
1671 // We have to insert explicit sign or zero extends. Note that we could
1672 // insert sign extends for ALL conditions. For those operations where either
1673 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1674 // which will choose the cheapest for the target.
1675 switch (CCCode) {
1676 default: llvm_unreachable("Unknown integer comparison!")::llvm::llvm_unreachable_internal("Unknown integer comparison!"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1676)
;
1677 case ISD::SETEQ:
1678 case ISD::SETNE: {
1679 SDValue OpL = GetPromotedInteger(NewLHS);
1680 SDValue OpR = GetPromotedInteger(NewRHS);
1681
1682 // We would prefer to promote the comparison operand with sign extension.
1683 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1684 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1685 // instruction, which is redundant eventually.
1686 unsigned OpLEffectiveBits =
1687 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1688 unsigned OpREffectiveBits =
1689 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1690 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1691 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1692 NewLHS = OpL;
1693 NewRHS = OpR;
1694 } else {
1695 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1696 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1697 }
1698 break;
1699 }
1700 case ISD::SETUGE:
1701 case ISD::SETUGT:
1702 case ISD::SETULE:
1703 case ISD::SETULT:
1704 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1705 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1706 break;
1707 case ISD::SETGE:
1708 case ISD::SETGT:
1709 case ISD::SETLT:
1710 case ISD::SETLE:
1711 NewLHS = SExtPromotedInteger(NewLHS);
1712 NewRHS = SExtPromotedInteger(NewRHS);
1713 break;
1714 }
1715}
1716
1717SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1718 SDValue Op = GetPromotedInteger(N->getOperand(0));
1719 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1720}
1721
1722SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1723 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1724 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1725 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1726}
1727
1728SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1729 // This should only occur in unusual situations like bitcasting to an
1730 // x86_fp80, so just turn it into a store+load
1731 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1732}
1733
1734SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1735 assert(OpNo == 2 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 2 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 2 && \"Don't know how to promote this operand!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1735, __extension__ __PRETTY_FUNCTION__))
;
1736
1737 SDValue LHS = N->getOperand(2);
1738 SDValue RHS = N->getOperand(3);
1739 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1740
1741 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1742 // legal types.
1743 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1744 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1745 0);
1746}
1747
1748SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1749 assert(OpNo == 1 && "only know how to promote condition")(static_cast <bool> (OpNo == 1 && "only know how to promote condition"
) ? void (0) : __assert_fail ("OpNo == 1 && \"only know how to promote condition\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1749, __extension__ __PRETTY_FUNCTION__))
;
1750
1751 // Promote all the way up to the canonical SetCC type.
1752 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1753
1754 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1755 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1756 N->getOperand(2)), 0);
1757}
1758
1759SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1760 // Since the result type is legal, the operands must promote to it.
1761 EVT OVT = N->getOperand(0).getValueType();
1762 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1763 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1764 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?")(static_cast <bool> (Lo.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Lo.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1764, __extension__ __PRETTY_FUNCTION__))
;
1765 SDLoc dl(N);
1766
1767 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1768 DAG.getConstant(OVT.getSizeInBits(), dl,
1769 TLI.getPointerTy(DAG.getDataLayout())));
1770 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1771}
1772
1773SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1774 // The vector type is legal but the element type is not. This implies
1775 // that the vector is a power-of-two in length and that the element
1776 // type does not have a strange size (eg: it is not i1).
1777 EVT VecVT = N->getValueType(0);
1778 unsigned NumElts = VecVT.getVectorNumElements();
1779 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&(static_cast <bool> (!((NumElts & 1) && (!TLI
.isTypeLegal(VecVT))) && "Legal vector of one illegal element?"
) ? void (0) : __assert_fail ("!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && \"Legal vector of one illegal element?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1780, __extension__ __PRETTY_FUNCTION__))
1780 "Legal vector of one illegal element?")(static_cast <bool> (!((NumElts & 1) && (!TLI
.isTypeLegal(VecVT))) && "Legal vector of one illegal element?"
) ? void (0) : __assert_fail ("!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && \"Legal vector of one illegal element?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1780, __extension__ __PRETTY_FUNCTION__))
;
1781
1782 // Promote the inserted value. The type does not need to match the
1783 // vector element type. Check that any extra bits introduced will be
1784 // truncated away.
1785 assert(N->getOperand(0).getValueSizeInBits() >=(static_cast <bool> (N->getOperand(0).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!")
? void (0) : __assert_fail ("N->getOperand(0).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1787, __extension__ __PRETTY_FUNCTION__))
1786 N->getValueType(0).getScalarSizeInBits() &&(static_cast <bool> (N->getOperand(0).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!")
? void (0) : __assert_fail ("N->getOperand(0).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1787, __extension__ __PRETTY_FUNCTION__))
1787 "Type of inserted value narrower than vector element type!")(static_cast <bool> (N->getOperand(0).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!")
? void (0) : __assert_fail ("N->getOperand(0).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1787, __extension__ __PRETTY_FUNCTION__))
;
1788
1789 SmallVector<SDValue, 16> NewOps;
1790 for (unsigned i = 0; i < NumElts; ++i)
1791 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1792
1793 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1794}
1795
1796SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1797 unsigned OpNo) {
1798 if (OpNo == 1) {
1799 // Promote the inserted value. This is valid because the type does not
1800 // have to match the vector element type.
1801
1802 // Check that any extra bits introduced will be truncated away.
1803 assert(N->getOperand(1).getValueSizeInBits() >=(static_cast <bool> (N->getOperand(1).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!")
? void (0) : __assert_fail ("N->getOperand(1).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1805, __extension__ __PRETTY_FUNCTION__))
1804 N->getValueType(0).getScalarSizeInBits() &&(static_cast <bool> (N->getOperand(1).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!")
? void (0) : __assert_fail ("N->getOperand(1).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1805, __extension__ __PRETTY_FUNCTION__))
1805 "Type of inserted value narrower than vector element type!")(static_cast <bool> (N->getOperand(1).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!")
? void (0) : __assert_fail ("N->getOperand(1).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1805, __extension__ __PRETTY_FUNCTION__))
;
1806 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1807 GetPromotedInteger(N->getOperand(1)),
1808 N->getOperand(2)),
1809 0);
1810 }
1811
1812 assert(OpNo == 2 && "Different operand and result vector types?")(static_cast <bool> (OpNo == 2 && "Different operand and result vector types?"
) ? void (0) : __assert_fail ("OpNo == 2 && \"Different operand and result vector types?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1812, __extension__ __PRETTY_FUNCTION__))
;
1813
1814 // Promote the index.
1815 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1816 TLI.getVectorIdxTy(DAG.getDataLayout()));
1817 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1818 N->getOperand(1), Idx), 0);
1819}
1820
1821SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1822 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1823 // the operand in place.
1824 return SDValue(DAG.UpdateNodeOperands(N,
1825 GetPromotedInteger(N->getOperand(0))), 0);
1826}
1827
1828SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1829 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1830 // operand in place.
1831 return SDValue(
1832 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1833}
1834
1835SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1836 assert(OpNo == 0 && "Only know how to promote the condition!")(static_cast <bool> (OpNo == 0 && "Only know how to promote the condition!"
) ? void (0) : __assert_fail ("OpNo == 0 && \"Only know how to promote the condition!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1836, __extension__ __PRETTY_FUNCTION__))
;
1837 SDValue Cond = N->getOperand(0);
1838 EVT OpTy = N->getOperand(1).getValueType();
1839
1840 if (N->getOpcode() == ISD::VSELECT)
1841 if (SDValue Res = WidenVSELECTMask(N))
1842 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1843 Res, N->getOperand(1), N->getOperand(2));
1844
1845 // Promote all the way up to the canonical SetCC type.
1846 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1847 Cond = PromoteTargetBoolean(Cond, OpVT);
1848
1849 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1850 N->getOperand(2)), 0);
1851}
1852
1853SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1854 assert(OpNo == 0 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 0 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 0 && \"Don't know how to promote this operand!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1854, __extension__ __PRETTY_FUNCTION__))
;
1855
1856 SDValue LHS = N->getOperand(0);
1857 SDValue RHS = N->getOperand(1);
1858 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1859
1860 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1861 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1862 N->getOperand(3), N->getOperand(4)), 0);
1863}
1864
1865SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1866 assert(OpNo == 0 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 0 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 0 && \"Don't know how to promote this operand!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1866, __extension__ __PRETTY_FUNCTION__))
;
1867
1868 SDValue LHS = N->getOperand(0);
1869 SDValue RHS = N->getOperand(1);
1870 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1871
1872 // The CC (#2) is always legal.
1873 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1874}
1875
1876SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1877 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1878 ZExtPromotedInteger(N->getOperand(1))), 0);
1879}
1880
1881SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1882 SDValue Op = GetPromotedInteger(N->getOperand(0));
1883 SDLoc dl(N);
1884 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1885 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1886 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1887}
1888
1889SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1890 return SDValue(DAG.UpdateNodeOperands(N,
1891 SExtPromotedInteger(N->getOperand(0))), 0);
1892}
1893
1894SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1895 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1896 SExtPromotedInteger(N->getOperand(1))), 0);
1897}
1898
1899SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1900 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDStore(N) &&
"Indexed store during type legalization!") ? void (0) : __assert_fail
("ISD::isUNINDEXEDStore(N) && \"Indexed store during type legalization!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1900, __extension__ __PRETTY_FUNCTION__))
;
1901 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1902 SDLoc dl(N);
1903
1904 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1905
1906 // Truncate the value and store the result.
1907 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1908 N->getMemoryVT(), N->getMemOperand());
1909}
1910
1911SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1912 unsigned OpNo) {
1913 SDValue DataOp = N->getValue();
1914 SDValue Mask = N->getMask();
1915
1916 if (OpNo == 4) {
1917 // The Mask. Update in place.
1918 EVT DataVT = DataOp.getValueType();
1919 Mask = PromoteTargetBoolean(Mask, DataVT);
1920 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1921 NewOps[4] = Mask;
1922 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1923 }
1924
1925 assert(OpNo == 1 && "Unexpected operand for promotion")(static_cast <bool> (OpNo == 1 && "Unexpected operand for promotion"
) ? void (0) : __assert_fail ("OpNo == 1 && \"Unexpected operand for promotion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1925, __extension__ __PRETTY_FUNCTION__))
;
1926 DataOp = GetPromotedInteger(DataOp);
1927
1928 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
1929 N->getOffset(), Mask, N->getMemoryVT(),
1930 N->getMemOperand(), N->getAddressingMode(),
1931 /*IsTruncating*/ true, N->isCompressingStore());
1932}
1933
1934SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1935 unsigned OpNo) {
1936 assert(OpNo == 3 && "Only know how to promote the mask!")(static_cast <bool> (OpNo == 3 && "Only know how to promote the mask!"
) ? void (0) : __assert_fail ("OpNo == 3 && \"Only know how to promote the mask!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1936, __extension__ __PRETTY_FUNCTION__))
;
1937 EVT DataVT = N->getValueType(0);
1938 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1939 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1940 NewOps[OpNo] = Mask;
1941 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1942 if (Res == N)
1943 return SDValue(Res, 0);
1944
1945 // Update triggered CSE, do our own replacement since caller can't.
1946 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1947 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1948 return SDValue();
1949}
1950
1951SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1952 unsigned OpNo) {
1953
1954 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1955 if (OpNo == 2) {
1956 // The Mask
1957 EVT DataVT = N->getValueType(0);
1958 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1959 } else if (OpNo == 4) {
1960 // The Index
1961 if (N->isIndexSigned())
1962 // Need to sign extend the index since the bits will likely be used.
1963 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1964 else
1965 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1966 } else
1967 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1968
1969 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1970 if (Res == N)
1971 return SDValue(Res, 0);
1972
1973 // Update triggered CSE, do our own replacement since caller can't.
1974 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1975 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1976 return SDValue();
1977}
1978
1979SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1980 unsigned OpNo) {
1981 bool TruncateStore = N->isTruncatingStore();
1982 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1983 if (OpNo == 2) {
1984 // The Mask
1985 EVT DataVT = N->getValue().getValueType();
1986 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1987 } else if (OpNo == 4) {
1988 // The Index
1989 if (N->isIndexSigned())
1990 // Need to sign extend the index since the bits will likely be used.
1991 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1992 else
1993 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1994
1995 N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(),
1996 N->getMemoryVT(), NewOps[OpNo]));
1997 } else {
1998 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1999 TruncateStore = true;
2000 }
2001
2002 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2003 SDLoc(N), NewOps, N->getMemOperand(),
2004 N->getIndexType(), TruncateStore);
2005}
2006
2007SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2008 SDValue Op = GetPromotedInteger(N->getOperand(0));
2009 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2010}
2011
2012SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2013 return SDValue(DAG.UpdateNodeOperands(N,
2014 ZExtPromotedInteger(N->getOperand(0))), 0);
2015}
2016
2017SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2018 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2019 ZExtPromotedInteger(N->getOperand(1))), 0);
2020}
2021
2022SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2023 SDLoc dl(N);
2024 SDValue Op = GetPromotedInteger(N->getOperand(0));
2025 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2026 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2027}
2028
2029SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
2030 assert(OpNo == 2 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 2 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 2 && \"Don't know how to promote this operand!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2030, __extension__ __PRETTY_FUNCTION__))
;
2031
2032 SDValue LHS = N->getOperand(0);
2033 SDValue RHS = N->getOperand(1);
2034 SDValue Carry = N->getOperand(2);
2035 SDLoc DL(N);
2036
2037 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2038
2039 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2040}
2041
2042SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2043 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2044 return SDValue(
2045 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2046}
2047
2048SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2049 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2050 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2051 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2052}
2053
2054SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
2055 assert(OpNo > 1 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo > 1 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo > 1 && \"Don't know how to promote this operand!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2055, __extension__ __PRETTY_FUNCTION__))
;
2056 // Promote the rw, locality, and cache type arguments to a supported integer
2057 // width.
2058 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2059 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
2060 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
2061 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2062 Op2, Op3, Op4),
2063 0);
2064}
2065
2066SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
2067 // FIXME: Support for promotion of STRICT_FPOWI is not implemented yet.
2068 assert(N->getOpcode() == ISD::FPOWI && "No STRICT_FPOWI support here yet.")(static_cast <bool> (N->getOpcode() == ISD::FPOWI &&
"No STRICT_FPOWI support here yet.") ? void (0) : __assert_fail
("N->getOpcode() == ISD::FPOWI && \"No STRICT_FPOWI support here yet.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2068, __extension__ __PRETTY_FUNCTION__))
;
2069
2070 // The integer operand is the last operand in FPOWI (so the result and
2071 // floating point operand is already type legalized).
2072
2073 // We can't just promote the exponent type in FPOWI, since we want to lower
2074 // the node to a libcall and we if we promote to a type larger than
2075 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2076 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2077 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2078 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
2079 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected fpowi.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2079, __extension__ __PRETTY_FUNCTION__))
;
2080 if (!TLI.getLibcallName(LC)) {
2081 // Some targets don't have a powi libcall; use pow instead.
2082 // FIXME: Implement this if some target needs it.
2083 DAG.getContext()->emitError("Don't know how to promote fpowi to fpow");
2084 return DAG.getUNDEF(N->getValueType(0));
2085 }
2086 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2087 assert(DAG.getLibInfo().getIntSize() ==(static_cast <bool> (DAG.getLibInfo().getIntSize() == N
->getOperand(1).getValueType().getSizeInBits() && "POWI exponent should match with sizeof(int) when doing the libcall."
) ? void (0) : __assert_fail ("DAG.getLibInfo().getIntSize() == N->getOperand(1).getValueType().getSizeInBits() && \"POWI exponent should match with sizeof(int) when doing the libcall.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2089, __extension__ __PRETTY_FUNCTION__))
2088 N->getOperand(1).getValueType().getSizeInBits() &&(static_cast <bool> (DAG.getLibInfo().getIntSize() == N
->getOperand(1).getValueType().getSizeInBits() && "POWI exponent should match with sizeof(int) when doing the libcall."
) ? void (0) : __assert_fail ("DAG.getLibInfo().getIntSize() == N->getOperand(1).getValueType().getSizeInBits() && \"POWI exponent should match with sizeof(int) when doing the libcall.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2089, __extension__ __PRETTY_FUNCTION__))
2089 "POWI exponent should match with sizeof(int) when doing the libcall.")(static_cast <bool> (DAG.getLibInfo().getIntSize() == N
->getOperand(1).getValueType().getSizeInBits() && "POWI exponent should match with sizeof(int) when doing the libcall."
) ? void (0) : __assert_fail ("DAG.getLibInfo().getIntSize() == N->getOperand(1).getValueType().getSizeInBits() && \"POWI exponent should match with sizeof(int) when doing the libcall.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2089, __extension__ __PRETTY_FUNCTION__))
;
2090 TargetLowering::MakeLibCallOptions CallOptions;
2091 CallOptions.setSExt(true);
2092 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2093 std::pair<SDValue, SDValue> Tmp =
2094 TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops,
2095 CallOptions, SDLoc(N), SDValue());
2096 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2097 return SDValue();
2098}
2099
2100static unsigned getExtendForIntVecReduction(SDNode *N) {
2101 switch (N->getOpcode()) {
2102 default:
2103 llvm_unreachable("Expected integer vector reduction")::llvm::llvm_unreachable_internal("Expected integer vector reduction"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2103)
;
2104 case ISD::VECREDUCE_ADD:
2105 case ISD::VECREDUCE_MUL:
2106 case ISD::VECREDUCE_AND:
2107 case ISD::VECREDUCE_OR:
2108 case ISD::VECREDUCE_XOR:
2109 case ISD::VP_REDUCE_ADD:
2110 case ISD::VP_REDUCE_MUL:
2111 case ISD::VP_REDUCE_AND:
2112 case ISD::VP_REDUCE_OR:
2113 case ISD::VP_REDUCE_XOR:
2114 return ISD::ANY_EXTEND;
2115 case ISD::VECREDUCE_SMAX:
2116 case ISD::VECREDUCE_SMIN:
2117 case ISD::VP_REDUCE_SMAX:
2118 case ISD::VP_REDUCE_SMIN:
2119 return ISD::SIGN_EXTEND;
2120 case ISD::VECREDUCE_UMAX:
2121 case ISD::VECREDUCE_UMIN:
2122 case ISD::VP_REDUCE_UMAX:
2123 case ISD::VP_REDUCE_UMIN:
2124 return ISD::ZERO_EXTEND;
2125 }
2126}
2127
2128SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2129 switch (getExtendForIntVecReduction(N)) {
2130 default:
2131 llvm_unreachable("Impossible extension kind for integer reduction")::llvm::llvm_unreachable_internal("Impossible extension kind for integer reduction"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2131)
;
2132 case ISD::ANY_EXTEND:
2133 return GetPromotedInteger(V);
2134 case ISD::SIGN_EXTEND:
2135 return SExtPromotedInteger(V);
2136 case ISD::ZERO_EXTEND:
2137 return ZExtPromotedInteger(V);
2138 }
2139}
2140
2141SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2142 SDLoc dl(N);
2143 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2144
2145 EVT EltVT = Op.getValueType().getVectorElementType();
2146 EVT VT = N->getValueType(0);
2147
2148 if (VT.bitsGE(EltVT))
2149 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
2150
2151 // Result size must be >= element size. If this is not the case after
2152 // promotion, also promote the result type and then truncate.
2153 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
2154 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
2155}
2156
2157SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2158 SDLoc DL(N);
2159 SDValue Op = N->getOperand(OpNo);
2160 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2161
2162 if (OpNo == 2) { // Mask
2163 // Update in place.
2164 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2165 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2166 }
2167
2168 assert(OpNo == 1 && "Unexpected operand for promotion")(static_cast <bool> (OpNo == 1 && "Unexpected operand for promotion"
) ? void (0) : __assert_fail ("OpNo == 1 && \"Unexpected operand for promotion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2168, __extension__ __PRETTY_FUNCTION__))
;
2169
2170 Op = PromoteIntOpVectorReduction(N, Op);
2171
2172 NewOps[OpNo] = Op;
2173
2174 EVT VT = N->getValueType(0);
2175 EVT EltVT = Op.getValueType().getScalarType();
2176
2177 if (VT.bitsGE(EltVT))
2178 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2179
2180 // Result size must be >= element/start-value size. If this is not the case
2181 // after promotion, also promote both the start value and result type and
2182 // then truncate.
2183 NewOps[0] =
2184 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2185 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2186 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2187}
2188
2189SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2190 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2191 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2192}
2193
2194//===----------------------------------------------------------------------===//
2195// Integer Result Expansion
2196//===----------------------------------------------------------------------===//
2197
2198/// ExpandIntegerResult - This method is called when the specified result of the
2199/// specified node is found to need expansion. At this point, the node may also
2200/// have invalid operands or may have other results that need promotion, we just
2201/// know that (at least) one result needs expansion.
2202void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2203 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
1
Assuming 'DebugFlag' is false
2
Loop condition is false. Exiting loop
2204 dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
;
2205 SDValue Lo, Hi;
2206 Lo = Hi = SDValue();
2207
2208 // See if the target wants to custom expand this node.
2209 if (CustomLowerNode(N, N->getValueType(ResNo), true))
3
Assuming the condition is false
4
Taking false branch
2210 return;
2211
2212 switch (N->getOpcode()) {
5
Control jumps to 'case SHL:' at line 2332
2213 default:
2214#ifndef NDEBUG
2215 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2216 N->dump(&DAG); dbgs() << "\n";
2217#endif
2218 report_fatal_error("Do not know how to expand the result of this "
2219 "operator!");
2220
2221 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2222 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
2223 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2224 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2225 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2226
2227 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2228 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2229 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2230 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2231 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2232
2233 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2234 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2235 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2236 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2237 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2238 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2239 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2240 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2241 case ISD::CTLZ_ZERO_UNDEF:
2242 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2243 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2244 case ISD::CTTZ_ZERO_UNDEF:
2245 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2246 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
2247 case ISD::STRICT_FP_TO_SINT:
2248 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
2249 case ISD::STRICT_FP_TO_UINT:
2250 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
2251 case ISD::FP_TO_SINT_SAT:
2252 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2253 case ISD::STRICT_LLROUND:
2254 case ISD::STRICT_LLRINT:
2255 case ISD::LLROUND:
2256 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
2257 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2258 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2259 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2260 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2261 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2262 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2263 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2264 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2265 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2266 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2267 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2268 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2269
2270 case ISD::ATOMIC_LOAD_ADD:
2271 case ISD::ATOMIC_LOAD_SUB:
2272 case ISD::ATOMIC_LOAD_AND:
2273 case ISD::ATOMIC_LOAD_CLR:
2274 case ISD::ATOMIC_LOAD_OR:
2275 case ISD::ATOMIC_LOAD_XOR:
2276 case ISD::ATOMIC_LOAD_NAND:
2277 case ISD::ATOMIC_LOAD_MIN:
2278 case ISD::ATOMIC_LOAD_MAX:
2279 case ISD::ATOMIC_LOAD_UMIN:
2280 case ISD::ATOMIC_LOAD_UMAX:
2281 case ISD::ATOMIC_SWAP:
2282 case ISD::ATOMIC_CMP_SWAP: {
2283 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2284 SplitInteger(Tmp.first, Lo, Hi);
2285 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2286 break;
2287 }
2288 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2289 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2290 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2291 SDValue Tmp = DAG.getAtomicCmpSwap(
2292 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2293 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2294 AN->getMemOperand());
2295
2296 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2297 // success simply by comparing the loaded value against the ingoing
2298 // comparison.
2299 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2300 N->getOperand(2), ISD::SETEQ);
2301
2302 SplitInteger(Tmp, Lo, Hi);
2303 ReplaceValueWith(SDValue(N, 1), Success);
2304 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2305 break;
2306 }
2307
2308 case ISD::AND:
2309 case ISD::OR:
2310 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2311
2312 case ISD::UMAX:
2313 case ISD::SMAX:
2314 case ISD::UMIN:
2315 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2316
2317 case ISD::ADD:
2318 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2319
2320 case ISD::ADDC:
2321 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2322
2323 case ISD::ADDE:
2324 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2325
2326 case ISD::ADDCARRY:
2327 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2328
2329 case ISD::SADDO_CARRY:
2330 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2331
2332 case ISD::SHL:
2333 case ISD::SRA:
2334 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
6
Calling 'DAGTypeLegalizer::ExpandIntRes_Shift'
2335
2336 case ISD::SADDO:
2337 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2338 case ISD::UADDO:
2339 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2340 case ISD::UMULO:
2341 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2342
2343 case ISD::SADDSAT:
2344 case ISD::UADDSAT:
2345 case ISD::SSUBSAT:
2346 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2347
2348 case ISD::SSHLSAT:
2349 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2350
2351 case ISD::SMULFIX:
2352 case ISD::SMULFIXSAT:
2353 case ISD::UMULFIX:
2354 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2355
2356 case ISD::SDIVFIX:
2357 case ISD::SDIVFIXSAT:
2358 case ISD::UDIVFIX:
2359 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2360
2361 case ISD::VECREDUCE_ADD:
2362 case ISD::VECREDUCE_MUL:
2363 case ISD::VECREDUCE_AND:
2364 case ISD::VECREDUCE_OR:
2365 case ISD::VECREDUCE_XOR:
2366 case ISD::VECREDUCE_SMAX:
2367 case ISD::VECREDUCE_SMIN:
2368 case ISD::VECREDUCE_UMAX:
2369 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2370
2371 case ISD::ROTL:
2372 case ISD::ROTR:
2373 ExpandIntRes_Rotate(N, Lo, Hi);
2374 break;
2375
2376 case ISD::FSHL:
2377 case ISD::FSHR:
2378 ExpandIntRes_FunnelShift(N, Lo, Hi);
2379 break;
2380
2381 case ISD::VSCALE:
2382 ExpandIntRes_VSCALE(N, Lo, Hi);
2383 break;
2384 }
2385
2386 // If Lo/Hi is null, the sub-method took care of registering results etc.
2387 if (Lo.getNode())
2388 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2389}
2390
2391/// Lower an atomic node to the appropriate builtin call.
2392std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2393 unsigned Opc = Node->getOpcode();
2394 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2395 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2396 // Lower to outline atomic libcall if outline atomics enabled,
2397 // or to sync libcall otherwise
2398 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2399 EVT RetVT = Node->getValueType(0);
2400 TargetLowering::MakeLibCallOptions CallOptions;
2401 SmallVector<SDValue, 4> Ops;
2402 if (TLI.getLibcallName(LC)) {
2403 Ops.append(Node->op_begin() + 2, Node->op_end());
2404 Ops.push_back(Node->getOperand(1));
2405 } else {
2406 LC = RTLIB::getSYNC(Opc, VT);
2407 assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected atomic op or value type!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2408, __extension__ __PRETTY_FUNCTION__))
2408 "Unexpected atomic op or value type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected atomic op or value type!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2408, __extension__ __PRETTY_FUNCTION__))
;
2409 Ops.append(Node->op_begin() + 1, Node->op_end());
2410 }
2411 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2412 Node->getOperand(0));
2413}
2414
2415/// N is a shift by a value that needs to be expanded,
2416/// and the shift amount is a constant 'Amt'. Expand the operation.
2417void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2418 SDValue &Lo, SDValue &Hi) {
2419 SDLoc DL(N);
2420 // Expand the incoming operand to be shifted, so that we have its parts
2421 SDValue InL, InH;
2422 GetExpandedInteger(N->getOperand(0), InL, InH);
2423
2424 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2425 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2426 if (!Amt) {
2427 Lo = InL;
2428 Hi = InH;
2429 return;
2430 }
2431
2432 EVT NVT = InL.getValueType();
2433 unsigned VTBits = N->getValueType(0).getSizeInBits();
2434 unsigned NVTBits = NVT.getSizeInBits();
2435 EVT ShTy = N->getOperand(1).getValueType();
2436
2437 if (N->getOpcode() == ISD::SHL) {
2438 if (Amt.ugt(VTBits)) {
2439 Lo = Hi = DAG.getConstant(0, DL, NVT);
2440 } else if (Amt.ugt(NVTBits)) {
2441 Lo = DAG.getConstant(0, DL, NVT);
2442 Hi = DAG.getNode(ISD::SHL, DL,
2443 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2444 } else if (Amt == NVTBits) {
2445 Lo = DAG.getConstant(0, DL, NVT);
2446 Hi = InL;
2447 } else {
2448 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2449 Hi = DAG.getNode(ISD::OR, DL, NVT,
2450 DAG.getNode(ISD::SHL, DL, NVT, InH,
2451 DAG.getConstant(Amt, DL, ShTy)),
2452 DAG.getNode(ISD::SRL, DL, NVT, InL,
2453 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2454 }
2455 return;
2456 }
2457
2458 if (N->getOpcode() == ISD::SRL) {
2459 if (Amt.ugt(VTBits)) {
2460 Lo = Hi = DAG.getConstant(0, DL, NVT);
2461 } else if (Amt.ugt(NVTBits)) {
2462 Lo = DAG.getNode(ISD::SRL, DL,
2463 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2464 Hi = DAG.getConstant(0, DL, NVT);
2465 } else if (Amt == NVTBits) {
2466 Lo = InH;
2467 Hi = DAG.getConstant(0, DL, NVT);
2468 } else {
2469 Lo = DAG.getNode(ISD::OR, DL, NVT,
2470 DAG.getNode(ISD::SRL, DL, NVT, InL,
2471 DAG.getConstant(Amt, DL, ShTy)),
2472 DAG.getNode(ISD::SHL, DL, NVT, InH,
2473 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2474 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2475 }
2476 return;
2477 }
2478
2479 assert(N->getOpcode() == ISD::SRA && "Unknown shift!")(static_cast <bool> (N->getOpcode() == ISD::SRA &&
"Unknown shift!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SRA && \"Unknown shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2479, __extension__ __PRETTY_FUNCTION__))
;
2480 if (Amt.ugt(VTBits)) {
2481 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2482 DAG.getConstant(NVTBits - 1, DL, ShTy));
2483 } else if (Amt.ugt(NVTBits)) {
2484 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2485 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2486 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2487 DAG.getConstant(NVTBits - 1, DL, ShTy));
2488 } else if (Amt == NVTBits) {
2489 Lo = InH;
2490 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2491 DAG.getConstant(NVTBits - 1, DL, ShTy));
2492 } else {
2493 Lo = DAG.getNode(ISD::OR, DL, NVT,
2494 DAG.getNode(ISD::SRL, DL, NVT, InL,
2495 DAG.getConstant(Amt, DL, ShTy)),
2496 DAG.getNode(ISD::SHL, DL, NVT, InH,
2497 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2498 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2499 }
2500}
2501
2502/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2503/// this shift based on knowledge of the high bit of the shift amount. If we
2504/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2505/// shift amount.
2506bool DAGTypeLegalizer::
2507ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2508 SDValue Amt = N->getOperand(1);
10
Value assigned to 'Amt.Node'
2509 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2510 EVT ShTy = Amt.getValueType();
11
Calling 'SDValue::getValueType'
2511 unsigned ShBits = ShTy.getScalarSizeInBits();
2512 unsigned NVTBits = NVT.getScalarSizeInBits();
2513 assert(isPowerOf2_32(NVTBits) &&(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2514, __extension__ __PRETTY_FUNCTION__))
2514 "Expanded integer type size not a power of two!")(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2514, __extension__ __PRETTY_FUNCTION__))
;
2515 SDLoc dl(N);
2516
2517 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2518 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2519
2520 // If we don't know anything about the high bits, exit.
2521 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2522 return false;
2523
2524 // Get the incoming operand to be shifted.
2525 SDValue InL, InH;
2526 GetExpandedInteger(N->getOperand(0), InL, InH);
2527
2528 // If we know that any of the high bits of the shift amount are one, then we
2529 // can do this as a couple of simple shifts.
2530 if (Known.One.intersects(HighBitMask)) {
2531 // Mask out the high bit, which we know is set.
2532 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2533 DAG.getConstant(~HighBitMask, dl, ShTy));
2534
2535 switch (N->getOpcode()) {
2536 default: llvm_unreachable("Unknown shift")::llvm::llvm_unreachable_internal("Unknown shift", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2536)
;
2537 case ISD::SHL:
2538 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2539 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2540 return true;
2541 case ISD::SRL:
2542 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2543 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2544 return true;
2545 case ISD::SRA:
2546 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2547 DAG.getConstant(NVTBits - 1, dl, ShTy));
2548 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2549 return true;
2550 }
2551 }
2552
2553 // If we know that all of the high bits of the shift amount are zero, then we
2554 // can do this as a couple of simple shifts.
2555 if (HighBitMask.isSubsetOf(Known.Zero)) {
2556 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2557 // shift if x is zero. We can use XOR here because x is known to be smaller
2558 // than 32.
2559 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2560 DAG.getConstant(NVTBits - 1, dl, ShTy));
2561
2562 unsigned Op1, Op2;
2563 switch (N->getOpcode()) {
2564 default: llvm_unreachable("Unknown shift")::llvm::llvm_unreachable_internal("Unknown shift", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2564)
;
2565 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2566 case ISD::SRL:
2567 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2568 }
2569
2570 // When shifting right the arithmetic for Lo and Hi is swapped.
2571 if (N->getOpcode() != ISD::SHL)
2572 std::swap(InL, InH);
2573
2574 // Use a little trick to get the bits that move from Lo to Hi. First
2575 // shift by one bit.
2576 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2577 // Then compute the remaining shift with amount-1.
2578 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2579
2580 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2581 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2582
2583 if (N->getOpcode() != ISD::SHL)
2584 std::swap(Hi, Lo);
2585 return true;
2586 }
2587
2588 return false;
2589}
2590
2591/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2592/// of any size.
2593bool DAGTypeLegalizer::
2594ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2595 SDValue Amt = N->getOperand(1);
2596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2597 EVT ShTy = Amt.getValueType();
2598 unsigned NVTBits = NVT.getSizeInBits();
2599 assert(isPowerOf2_32(NVTBits) &&(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2600, __extension__ __PRETTY_FUNCTION__))
2600 "Expanded integer type size not a power of two!")(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2600, __extension__ __PRETTY_FUNCTION__))
;
2601 SDLoc dl(N);
2602
2603 // Get the incoming operand to be shifted.
2604 SDValue InL, InH;
2605 GetExpandedInteger(N->getOperand(0), InL, InH);
2606
2607 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2608 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2609 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2610 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2611 Amt, NVBitsNode, ISD::SETULT);
2612 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2613 Amt, DAG.getConstant(0, dl, ShTy),
2614 ISD::SETEQ);
2615
2616 SDValue LoS, HiS, LoL, HiL;
2617 switch (N->getOpcode()) {
2618 default: llvm_unreachable("Unknown shift")::llvm::llvm_unreachable_internal("Unknown shift", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2618)
;
2619 case ISD::SHL:
2620 // Short: ShAmt < NVTBits
2621 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2622 HiS = DAG.getNode(ISD::OR, dl, NVT,
2623 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2624 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2625
2626 // Long: ShAmt >= NVTBits
2627 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2628 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2629
2630 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2631 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2632 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2633 return true;
2634 case ISD::SRL:
2635 // Short: ShAmt < NVTBits
2636 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2637 LoS = DAG.getNode(ISD::OR, dl, NVT,
2638 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2639 // FIXME: If Amt is zero, the following shift generates an undefined result
2640 // on some architectures.
2641 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2642
2643 // Long: ShAmt >= NVTBits
2644 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2645 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2646
2647 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2648 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2649 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2650 return true;
2651 case ISD::SRA:
2652 // Short: ShAmt < NVTBits
2653 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2654 LoS = DAG.getNode(ISD::OR, dl, NVT,
2655 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2656 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2657
2658 // Long: ShAmt >= NVTBits
2659 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2660 DAG.getConstant(NVTBits - 1, dl, ShTy));
2661 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2662
2663 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2664 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2665 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2666 return true;
2667 }
2668}
2669
2670static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2671
2672 switch (Op) {
2673 default: llvm_unreachable("invalid min/max opcode")::llvm::llvm_unreachable_internal("invalid min/max opcode", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2673)
;
2674 case ISD::SMAX:
2675 return std::make_pair(ISD::SETGT, ISD::UMAX);
2676 case ISD::UMAX:
2677 return std::make_pair(ISD::SETUGT, ISD::UMAX);
2678 case ISD::SMIN:
2679 return std::make_pair(ISD::SETLT, ISD::UMIN);
2680 case ISD::UMIN:
2681 return std::make_pair(ISD::SETULT, ISD::UMIN);
2682 }
2683}
2684
2685void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2686 SDValue &Lo, SDValue &Hi) {
2687 SDLoc DL(N);
2688 ISD::NodeType LoOpc;
2689 ISD::CondCode CondC;
2690 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2691
2692 // Expand the subcomponents.
2693 SDValue LHSL, LHSH, RHSL, RHSH;
2694 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2695 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2696
2697 // Value types
2698 EVT NVT = LHSL.getValueType();
2699 EVT CCT = getSetCCResultType(NVT);
2700
2701 // Hi part is always the same op
2702 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2703
2704 // We need to know whether to select Lo part that corresponds to 'winning'
2705 // Hi part or if Hi parts are equal.
2706 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2707 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2708
2709 // Lo part corresponding to the 'winning' Hi part
2710 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2711
2712 // Recursed Lo part if Hi parts are equal, this uses unsigned version
2713 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2714
2715 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2716}
2717
2718void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2719 SDValue &Lo, SDValue &Hi) {
2720 SDLoc dl(N);
2721 // Expand the subcomponents.
2722 SDValue LHSL, LHSH, RHSL, RHSH;
2723 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2724 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2725
2726 EVT NVT = LHSL.getValueType();
2727 SDValue LoOps[2] = { LHSL, RHSL };
2728 SDValue HiOps[3] = { LHSH, RHSH };
2729
2730 bool HasOpCarry = TLI.isOperationLegalOrCustom(
2731 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2732 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2733 if (HasOpCarry) {
2734 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2735 if (N->getOpcode() == ISD::ADD) {
2736 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2737 HiOps[2] = Lo.getValue(1);
2738 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2739 } else {
2740 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2741 HiOps[2] = Lo.getValue(1);
2742 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2743 }
2744 return;
2745 }
2746
2747 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2748 // them. TODO: Teach operation legalization how to expand unsupported
2749 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2750 // a carry of type MVT::Glue, but there doesn't seem to be any way to
2751 // generate a value of this type in the expanded code sequence.
2752 bool hasCarry =
2753 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2754 ISD::ADDC : ISD::SUBC,
2755 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2756
2757 if (hasCarry) {
2758 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2759 if (N->getOpcode() == ISD::ADD) {
2760 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2761 HiOps[2] = Lo.getValue(1);
2762 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2763 } else {
2764 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2765 HiOps[2] = Lo.getValue(1);
2766 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2767 }
2768 return;
2769 }
2770
2771 bool hasOVF =
2772 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2773 ISD::UADDO : ISD::USUBO,
2774 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2775 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2776
2777 if (hasOVF) {
2778 EVT OvfVT = getSetCCResultType(NVT);
2779 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2780 int RevOpc;
2781 if (N->getOpcode() == ISD::ADD) {
2782 RevOpc = ISD::SUB;
2783 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2784 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2785 } else {
2786 RevOpc = ISD::ADD;
2787 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2788 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2789 }
2790 SDValue OVF = Lo.getValue(1);
2791
2792 switch (BoolType) {
2793 case TargetLoweringBase::UndefinedBooleanContent:
2794 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2795 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2796 case TargetLoweringBase::ZeroOrOneBooleanContent:
2797 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2798 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2799 break;
2800 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2801 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2802 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2803 }
2804 return;
2805 }
2806
2807 if (N->getOpcode() == ISD::ADD) {
2808 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2809 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2810 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2811 ISD::SETULT);
2812
2813 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2814 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2815 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2816 return;
2817 }
2818
2819 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2820 DAG.getConstant(1, dl, NVT),
2821 DAG.getConstant(0, dl, NVT));
2822 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2823 ISD::SETULT);
2824 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2825 DAG.getConstant(1, dl, NVT), Carry1);
2826 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2827 } else {
2828 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2829 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2830 SDValue Cmp =
2831 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2832 LoOps[0], LoOps[1], ISD::SETULT);
2833
2834 SDValue Borrow;
2835 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2836 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2837 else
2838 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2839 DAG.getConstant(0, dl, NVT));
2840
2841 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2842 }
2843}
2844
2845void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2846 SDValue &Lo, SDValue &Hi) {
2847 // Expand the subcomponents.
2848 SDValue LHSL, LHSH, RHSL, RHSH;
2849 SDLoc dl(N);
2850 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2851 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2852 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2853 SDValue LoOps[2] = { LHSL, RHSL };
2854 SDValue HiOps[3] = { LHSH, RHSH };
2855
2856 if (N->getOpcode() == ISD::ADDC) {
2857 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2858 HiOps[2] = Lo.getValue(1);
2859 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2860 } else {
2861 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2862 HiOps[2] = Lo.getValue(1);
2863 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2864 }
2865
2866 // Legalized the flag result - switch anything that used the old flag to
2867 // use the new one.
2868 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2869}
2870
2871void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2872 SDValue &Lo, SDValue &Hi) {
2873 // Expand the subcomponents.
2874 SDValue LHSL, LHSH, RHSL, RHSH;
2875 SDLoc dl(N);
2876 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2877 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2878 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2879 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2880 SDValue HiOps[3] = { LHSH, RHSH };
2881
2882 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2883 HiOps[2] = Lo.getValue(1);
2884 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2885
2886 // Legalized the flag result - switch anything that used the old flag to
2887 // use the new one.
2888 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2889}
2890
2891void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2892 SDValue &Lo, SDValue &Hi) {
2893 SDValue LHS = N->getOperand(0);
2894 SDValue RHS = N->getOperand(1);
2895 SDLoc dl(N);
2896
2897 SDValue Ovf;
2898
2899 unsigned CarryOp, NoCarryOp;
2900 ISD::CondCode Cond;
2901 switch(N->getOpcode()) {
2902 case ISD::UADDO:
2903 CarryOp = ISD::ADDCARRY;
2904 NoCarryOp = ISD::ADD;
2905 Cond = ISD::SETULT;
2906 break;
2907 case ISD::USUBO:
2908 CarryOp = ISD::SUBCARRY;
2909 NoCarryOp = ISD::SUB;
2910 Cond = ISD::SETUGT;
2911 break;
2912 default:
2913 llvm_unreachable("Node has unexpected Opcode")::llvm::llvm_unreachable_internal("Node has unexpected Opcode"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2913)
;
2914 }
2915
2916 bool HasCarryOp = TLI.isOperationLegalOrCustom(
2917 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2918
2919 if (HasCarryOp) {
2920 // Expand the subcomponents.
2921 SDValue LHSL, LHSH, RHSL, RHSH;
2922 GetExpandedInteger(LHS, LHSL, LHSH);
2923 GetExpandedInteger(RHS, RHSL, RHSH);
2924 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2925 SDValue LoOps[2] = { LHSL, RHSL };
2926 SDValue HiOps[3] = { LHSH, RHSH };
2927
2928 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2929 HiOps[2] = Lo.getValue(1);
2930 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2931
2932 Ovf = Hi.getValue(1);
2933 } else {
2934 // Expand the result by simply replacing it with the equivalent
2935 // non-overflow-checking operation.
2936 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2937 SplitInteger(Sum, Lo, Hi);
2938
2939 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2940 // overflows iff a - b > a.
2941 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2942 }
2943
2944 // Legalized the flag result - switch anything that used the old flag to
2945 // use the new one.
2946 ReplaceValueWith(SDValue(N, 1), Ovf);
2947}
2948
2949void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2950 SDValue &Lo, SDValue &Hi) {
2951 // Expand the subcomponents.
2952 SDValue LHSL, LHSH, RHSL, RHSH;
2953 SDLoc dl(N);
2954 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2955 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2956 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2957 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2958 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2959
2960 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2961 HiOps[2] = Lo.getValue(1);
2962 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2963
2964 // Legalized the flag result - switch anything that used the old flag to
2965 // use the new one.
2966 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2967}
2968
2969void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
2970 SDValue &Lo, SDValue &Hi) {
2971 // Expand the subcomponents.
2972 SDValue LHSL, LHSH, RHSL, RHSH;
2973 SDLoc dl(N);
2974 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2975 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2976 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2977
2978 // We need to use an unsigned carry op for the lo part.
2979 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
2980 : ISD::SUBCARRY;
2981 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
2982 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
2983
2984 // Legalized the flag result - switch anything that used the old flag to
2985 // use the new one.
2986 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2987}
2988
2989void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2990 SDValue &Lo, SDValue &Hi) {
2991 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2992 SDLoc dl(N);
2993 SDValue Op = N->getOperand(0);
2994 if (Op.getValueType().bitsLE(NVT)) {
2995 // The low part is any extension of the input (which degenerates to a copy).
2996 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2997 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2998 } else {
2999 // For example, extension of an i48 to an i64. The operand type necessarily
3000 // promotes to the result type, so will end up being expanded too.
3001 assert(getTypeAction(Op.getValueType()) ==(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3003, __extension__ __PRETTY_FUNCTION__))
3002 TargetLowering::TypePromoteInteger &&(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3003, __extension__ __PRETTY_FUNCTION__))
3003 "Only know how to promote this result!")(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3003, __extension__ __PRETTY_FUNCTION__))
;
3004 SDValue Res = GetPromotedInteger(Op);
3005 assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3006, __extension__ __PRETTY_FUNCTION__))
3006 "Operand over promoted?")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3006, __extension__ __PRETTY_FUNCTION__))
;
3007 // Split the promoted operand. This will simplify when it is expanded.
3008 SplitInteger(Res, Lo, Hi);
3009 }
3010}
3011
3012void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3013 SDValue &Lo, SDValue &Hi) {
3014 SDLoc dl(N);
3015 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3016 EVT NVT = Lo.getValueType();
3017 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3018 unsigned NVTBits = NVT.getSizeInBits();
3019 unsigned EVTBits = EVT.getSizeInBits();
3020
3021 if (NVTBits < EVTBits) {
3022 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3023 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3024 EVTBits - NVTBits)));
3025 } else {
3026 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3027 // The high part replicates the sign bit of Lo, make it explicit.
3028 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3029 DAG.getConstant(NVTBits - 1, dl,
3030 TLI.getPointerTy(DAG.getDataLayout())));
3031 }
3032}
3033
3034void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3035 SDValue &Lo, SDValue &Hi) {
3036 SDLoc dl(N);
3037 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3038 EVT NVT = Lo.getValueType();
3039 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3040 unsigned NVTBits = NVT.getSizeInBits();
3041 unsigned EVTBits = EVT.getSizeInBits();
3042
3043 if (NVTBits < EVTBits) {
3044 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3045 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3046 EVTBits - NVTBits)));
3047 } else {
3048 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3049 // The high part must be zero, make it explicit.
3050 Hi = DAG.getConstant(0, dl, NVT);
3051 }
3052}
3053
3054void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3055 SDValue &Lo, SDValue &Hi) {
3056 SDLoc dl(N);
3057 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3058 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3059 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3060}
3061
3062void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3063 SDValue &Lo, SDValue &Hi) {
3064 SDLoc dl(N);
3065 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3066 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3067 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3068}
3069
3070void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3071 SDValue &Hi) {
3072 SDLoc dl(N);
3073 // parity(HiLo) -> parity(Lo^Hi)
3074 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3075 EVT NVT = Lo.getValueType();
3076 Lo =
3077 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3078 Hi = DAG.getConstant(0, dl, NVT);
3079}
3080
3081void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3082 SDValue &Lo, SDValue &Hi) {
3083 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3084 unsigned NBitWidth = NVT.getSizeInBits();
3085 auto Constant = cast<ConstantSDNode>(N);
3086 const APInt &Cst = Constant->getAPIntValue();
3087 bool IsTarget = Constant->isTargetOpcode();
3088 bool IsOpaque = Constant->isOpaque();
3089 SDLoc dl(N);
3090 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3091 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3092 IsOpaque);
3093}
3094
3095void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3096 SDLoc dl(N);
3097
3098 SDValue N0 = N->getOperand(0);
3099 GetExpandedInteger(N0, Lo, Hi);
3100 EVT NVT = Lo.getValueType();
3101
3102 // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we
3103 // use in LegalizeDAG. The ADD part of the expansion is based on
3104 // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that
3105 // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded
3106 // if needed. Shift expansion has a special case for filling with sign bits
3107 // so that we will only end up with one SRA.
3108 bool HasAddCarry = TLI.isOperationLegalOrCustom(
3109 ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3110 if (HasAddCarry) {
3111 EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG);
3112 SDValue Sign =
3113 DAG.getNode(ISD::SRA, dl, NVT, Hi,
3114 DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy));
3115 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3116 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign);
3117 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3118 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3119 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3120 return;
3121 }
3122
3123 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3124 EVT VT = N->getValueType(0);
3125 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3126 DAG.getConstant(0, dl, VT), N0);
3127 SDValue NegLo, NegHi;
3128 SplitInteger(Neg, NegLo, NegHi);
3129
3130 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
3131 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
3132 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3133 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3134}
3135
3136void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3137 SDValue &Lo, SDValue &Hi) {
3138 SDLoc dl(N);
3139 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3140 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3141 EVT NVT = Lo.getValueType();
3142
3143 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3144 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3145
3146 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3147 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3148
3149 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3150 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3151 DAG.getConstant(NVT.getSizeInBits(), dl,
3152 NVT)));
3153 Hi = DAG.getConstant(0, dl, NVT);
3154}
3155
3156void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3157 SDValue &Lo, SDValue &Hi) {
3158 SDLoc dl(N);
3159 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3160 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3161 EVT NVT = Lo.getValueType();
3162 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3163 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3164 Hi = DAG.getConstant(0, dl, NVT);
3165}
3166
3167void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3168 SDValue &Lo, SDValue &Hi) {
3169 SDLoc dl(N);
3170 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3171 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3172 EVT NVT = Lo.getValueType();
3173
3174 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3175 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3176
3177 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3178 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3179
3180 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3181 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3182 DAG.getConstant(NVT.getSizeInBits(), dl,
3183 NVT)));
3184 Hi = DAG.getConstant(0, dl, NVT);
3185}
3186
3187void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
3188 SDValue &Hi) {
3189 SDLoc dl(N);
3190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3191 unsigned NBitWidth = NVT.getSizeInBits();
3192
3193 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3194 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
3195 SDValue Chain = Lo.getValue(1);
3196 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
3197 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3198 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
3199
3200 // Legalize the chain result - switch anything that used the old chain to
3201 // use the new one.
3202 ReplaceValueWith(SDValue(N, 1), Chain);
3203}
3204
3205void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
3206 SDValue &Hi) {
3207 SDLoc dl(N);
3208 EVT VT = N->getValueType(0);
3209
3210 bool IsStrict = N->isStrictFPOpcode();
3211 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3212 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3213 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3214 Op = GetPromotedFloat(Op);
3215
3216 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3217 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3218 Op = GetSoftPromotedHalf(Op);
3219 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3220 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
3221 SplitInteger(Op, Lo, Hi);
3222 return;
3223 }
3224
3225 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
3226 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-sint conversion!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected fp-to-sint conversion!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3226, __extension__ __PRETTY_FUNCTION__))
;
3227 TargetLowering::MakeLibCallOptions CallOptions;
3228 CallOptions.setSExt(true);
3229 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3230 CallOptions, dl, Chain);
3231 SplitInteger(Tmp.first, Lo, Hi);
3232
3233 if (IsStrict)
3234 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3235}
3236
3237void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
3238 SDValue &Hi) {
3239 SDLoc dl(N);
3240 EVT VT = N->getValueType(0);
3241
3242 bool IsStrict = N->isStrictFPOpcode();
3243 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3244 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3245 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3246 Op = GetPromotedFloat(Op);
3247
3248 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3249 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3250 Op = GetSoftPromotedHalf(Op);
3251 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3252 Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op);
3253 SplitInteger(Op, Lo, Hi);
3254 return;
3255 }
3256
3257 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
3258 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-uint conversion!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected fp-to-uint conversion!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3258, __extension__ __PRETTY_FUNCTION__))
;
3259 TargetLowering::MakeLibCallOptions CallOptions;
3260 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3261 CallOptions, dl, Chain);
3262 SplitInteger(Tmp.first, Lo, Hi);
3263
3264 if (IsStrict)
3265 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3266}
3267
3268void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3269 SDValue &Hi) {
3270 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3271 SplitInteger(Res, Lo, Hi);
3272}
3273
3274void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
3275 SDValue &Hi) {
3276 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
3277
3278 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&(static_cast <bool> (getTypeAction(Op.getValueType()) !=
TargetLowering::TypePromoteFloat && "Input type needs to be promoted!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && \"Input type needs to be promoted!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3279, __extension__ __PRETTY_FUNCTION__))
3279 "Input type needs to be promoted!")(static_cast <bool> (getTypeAction(Op.getValueType()) !=
TargetLowering::TypePromoteFloat && "Input type needs to be promoted!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && \"Input type needs to be promoted!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3279, __extension__ __PRETTY_FUNCTION__))
;
3280
3281 EVT VT = Op.getValueType();
3282
3283 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3284 if (N->getOpcode() == ISD::LLROUND ||
3285 N->getOpcode() == ISD::STRICT_LLROUND) {
3286 if (VT == MVT::f32)
3287 LC = RTLIB::LLROUND_F32;
3288 else if (VT == MVT::f64)
3289 LC = RTLIB::LLROUND_F64;
3290 else if (VT == MVT::f80)
3291 LC = RTLIB::LLROUND_F80;
3292 else if (VT == MVT::f128)
3293 LC = RTLIB::LLROUND_F128;
3294 else if (VT == MVT::ppcf128)
3295 LC = RTLIB::LLROUND_PPCF128;
3296 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected llround input type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3296, __extension__ __PRETTY_FUNCTION__))
;
3297 } else if (N->getOpcode() == ISD::LLRINT ||
3298 N->getOpcode() == ISD::STRICT_LLRINT) {
3299 if (VT == MVT::f32)
3300 LC = RTLIB::LLRINT_F32;
3301 else if (VT == MVT::f64)
3302 LC = RTLIB::LLRINT_F64;
3303 else if (VT == MVT::f80)
3304 LC = RTLIB::LLRINT_F80;
3305 else if (VT == MVT::f128)
3306 LC = RTLIB::LLRINT_F128;
3307 else if (VT == MVT::ppcf128)
3308 LC = RTLIB::LLRINT_PPCF128;
3309 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected llrint input type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3309, __extension__ __PRETTY_FUNCTION__))
;
3310 } else
3311 llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3311)
;
3312
3313 SDLoc dl(N);
3314 EVT RetVT = N->getValueType(0);
3315 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
3316
3317 TargetLowering::MakeLibCallOptions CallOptions;
3318 CallOptions.setSExt(true);
3319 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3320 Op, CallOptions, dl,
3321 Chain);
3322 SplitInteger(Tmp.first, Lo, Hi);
3323
3324 if (N->isStrictFPOpcode())
3325 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3326}
3327
3328void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3329 SDValue &Lo, SDValue &Hi) {
3330 if (N->isAtomic()) {
3331 // It's typical to have larger CAS than atomic load instructions.
3332 SDLoc dl(N);
3333 EVT VT = N->getMemoryVT();
3334 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3335 SDValue Zero = DAG.getConstant(0, dl, VT);
3336 SDValue Swap = DAG.getAtomicCmpSwap(
3337 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3338 VT, VTs, N->getOperand(0),
3339 N->getOperand(1), Zero, Zero, N->getMemOperand());
3340 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3341 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3342 return;
3343 }
3344
3345 if (ISD::isNormalLoad(N)) {
3346 ExpandRes_NormalLoad(N, Lo, Hi);
3347 return;
3348 }
3349
3350 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDLoad(N) &&
"Indexed load during type legalization!") ? void (0) : __assert_fail
("ISD::isUNINDEXEDLoad(N) && \"Indexed load during type legalization!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3350, __extension__ __PRETTY_FUNCTION__))
;
3351
3352 EVT VT = N->getValueType(0);
3353 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3354 SDValue Ch = N->getChain();
3355 SDValue Ptr = N->getBasePtr();
3356 ISD::LoadExtType ExtType = N->getExtensionType();
3357 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3358 AAMDNodes AAInfo = N->getAAInfo();
3359 SDLoc dl(N);
3360
3361 assert(NVT.isByteSized() && "Expanded type not byte sized!")(static_cast <bool> (NVT.isByteSized() && "Expanded type not byte sized!"
) ? void (0) : __assert_fail ("NVT.isByteSized() && \"Expanded type not byte sized!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3361, __extension__ __PRETTY_FUNCTION__))
;
3362
3363 if (N->getMemoryVT().bitsLE(NVT)) {
3364 EVT MemVT = N->getMemoryVT();
3365
3366 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3367 N->getOriginalAlign(), MMOFlags, AAInfo);
3368
3369 // Remember the chain.
3370 Ch = Lo.getValue(1);
3371
3372 if (ExtType == ISD::SEXTLOAD) {
3373 // The high part is obtained by SRA'ing all but one of the bits of the
3374 // lo part.
3375 unsigned LoSize = Lo.getValueSizeInBits();
3376 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3377 DAG.getConstant(LoSize - 1, dl,
3378 TLI.getPointerTy(DAG.getDataLayout())));
3379 } else if (ExtType == ISD::ZEXTLOAD) {
3380 // The high part is just a zero.
3381 Hi = DAG.getConstant(0, dl, NVT);
3382 } else {
3383 assert(ExtType == ISD::EXTLOAD && "Unknown extload!")(static_cast <bool> (ExtType == ISD::EXTLOAD &&
"Unknown extload!") ? void (0) : __assert_fail ("ExtType == ISD::EXTLOAD && \"Unknown extload!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3383, __extension__ __PRETTY_FUNCTION__))
;
3384 // The high part is undefined.
3385 Hi = DAG.getUNDEF(NVT);
3386 }
3387 } else if (DAG.getDataLayout().isLittleEndian()) {
3388 // Little-endian - low bits are at low addresses.
3389 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3390 N->getOriginalAlign(), MMOFlags, AAInfo);
3391
3392 unsigned ExcessBits =
3393 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3394 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3395
3396 // Increment the pointer to the other half.
3397 unsigned IncrementSize = NVT.getSizeInBits()/8;
3398 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3399 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3400 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3401 N->getOriginalAlign(), MMOFlags, AAInfo);
3402
3403 // Build a factor node to remember that this load is independent of the
3404 // other one.
3405 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3406 Hi.getValue(1));
3407 } else {
3408 // Big-endian - high bits are at low addresses. Favor aligned loads at
3409 // the cost of some bit-fiddling.
3410 EVT MemVT = N->getMemoryVT();
3411 unsigned EBytes = MemVT.getStoreSize();
3412 unsigned IncrementSize = NVT.getSizeInBits()/8;
3413 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3414
3415 // Load both the high bits and maybe some of the low bits.
3416 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3417 EVT::getIntegerVT(*DAG.getContext(),
3418 MemVT.getSizeInBits() - ExcessBits),
3419 N->getOriginalAlign(), MMOFlags, AAInfo);
3420
3421 // Increment the pointer to the other half.
3422 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3423 // Load the rest of the low bits.
3424 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3425 N->getPointerInfo().getWithOffset(IncrementSize),
3426 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3427 N->getOriginalAlign(), MMOFlags, AAInfo);
3428
3429 // Build a factor node to remember that this load is independent of the
3430 // other one.
3431 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3432 Hi.getValue(1));
3433
3434 if (ExcessBits < NVT.getSizeInBits()) {
3435 // Transfer low bits from the bottom of Hi to the top of Lo.
3436 Lo = DAG.getNode(
3437 ISD::OR, dl, NVT, Lo,
3438 DAG.getNode(ISD::SHL, dl, NVT, Hi,
3439 DAG.getConstant(ExcessBits, dl,
3440 TLI.getPointerTy(DAG.getDataLayout()))));
3441 // Move high bits to the right position in Hi.
3442 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3443 Hi,
3444 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3445 TLI.getPointerTy(DAG.getDataLayout())));
3446 }
3447 }
3448
3449 // Legalize the chain result - switch anything that used the old chain to
3450 // use the new one.
3451 ReplaceValueWith(SDValue(N, 1), Ch);
3452}
3453
3454void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3455 SDValue &Lo, SDValue &Hi) {
3456 SDLoc dl(N);
3457 SDValue LL, LH, RL, RH;
3458 GetExpandedInteger(N->getOperand(0), LL, LH);
3459 GetExpandedInteger(N->getOperand(1), RL, RH);
3460 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3461 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3462}
3463
3464void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3465 SDValue &Lo, SDValue &Hi) {
3466 EVT VT = N->getValueType(0);
3467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3468 SDLoc dl(N);
3469
3470 SDValue LL, LH, RL, RH;
3471 GetExpandedInteger(N->getOperand(0), LL, LH);
3472 GetExpandedInteger(N->getOperand(1), RL, RH);
3473
3474 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3475 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3476 LL, LH, RL, RH))
3477 return;
3478
3479 // If nothing else, we can make a libcall.
3480 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3481 if (VT == MVT::i16)
3482 LC = RTLIB::MUL_I16;
3483 else if (VT == MVT::i32)
3484 LC = RTLIB::MUL_I32;
3485 else if (VT == MVT::i64)
3486 LC = RTLIB::MUL_I64;
3487 else if (VT == MVT::i128)
3488 LC = RTLIB::MUL_I128;
3489
3490 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3491 // We'll expand the multiplication by brute force because we have no other
3492 // options. This is a trivially-generalized version of the code from
3493 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3494 // 4.3.1).
3495 unsigned Bits = NVT.getSizeInBits();
3496 unsigned HalfBits = Bits >> 1;
3497 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3498 NVT);
3499 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3500 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3501
3502 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3503 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3504
3505 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3506 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
3507 // The type from TLI is too small to fit the shift amount we want.
3508 // Override it with i32. The shift will have to be legalized.
3509 ShiftAmtTy = MVT::i32;
3510 }
3511 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
3512 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3513 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3514 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3515
3516 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3517 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3518 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3519 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3520
3521 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3522 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3523 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3524
3525 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3526 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3527 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3528 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3529 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3530
3531 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3532 DAG.getNode(ISD::ADD, dl, NVT,
3533 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3534 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3535 return;
3536 }
3537
3538 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3539 TargetLowering::MakeLibCallOptions CallOptions;
3540 CallOptions.setSExt(true);
3541 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3542 Lo, Hi);
3543}
3544
3545void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3546 SDValue &Hi) {
3547 SDLoc DL(N);
3548 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3549 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3550 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3551 Lo = R.getValue(0);
3552 Hi = R.getValue(1);
3553 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3554}
3555
3556void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3557 SDValue &Hi) {
3558 SDValue Result = TLI.expandAddSubSat(N, DAG);
3559 SplitInteger(Result, Lo, Hi);
3560}
3561
3562void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
3563 SDValue &Hi) {
3564 SDValue Result = TLI.expandShlSat(N, DAG);
3565 SplitInteger(Result, Lo, Hi);
3566}
3567
3568/// This performs an expansion of the integer result for a fixed point
3569/// multiplication. The default expansion performs rounding down towards
3570/// negative infinity, though targets that do care about rounding should specify
3571/// a target hook for rounding and provide their own expansion or lowering of
3572/// fixed point multiplication to be consistent with rounding.
3573void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3574 SDValue &Hi) {
3575 SDLoc dl(N);
3576 EVT VT = N->getValueType(0);
3577 unsigned VTSize = VT.getScalarSizeInBits();
3578 SDValue LHS = N->getOperand(0);
3579 SDValue RHS = N->getOperand(1);
3580 uint64_t Scale = N->getConstantOperandVal(2);
3581 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3582 N->getOpcode() == ISD::UMULFIXSAT);
3583 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3584 N->getOpcode() == ISD::SMULFIXSAT);
3585
3586 // Handle special case when scale is equal to zero.
3587 if (!Scale) {
3588 SDValue Result;
3589 if (!Saturating) {
3590 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3591 } else {
3592 EVT BoolVT = getSetCCResultType(VT);
3593 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3594 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3595 SDValue Product = Result.getValue(0);
3596 SDValue Overflow = Result.getValue(1);
3597 if (Signed) {
3598 APInt MinVal = APInt::getSignedMinValue(VTSize);
3599 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3600 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3601 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3602 SDValue Zero = DAG.getConstant(0, dl, VT);
3603 // Xor the inputs, if resulting sign bit is 0 the product will be
3604 // positive, else negative.
3605 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3606 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
3607 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
3608 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3609 } else {
3610 // For unsigned multiplication, we only need to check the max since we
3611 // can't really overflow towards zero.
3612 APInt MaxVal = APInt::getMaxValue(VTSize);
3613 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3614 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3615 }
3616 }
3617 SplitInteger(Result, Lo, Hi);
3618 return;
3619 }
3620
3621 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3622 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3623 assert(Scale <= VTSize && "Scale can't be larger than the value type size.")(static_cast <bool> (Scale <= VTSize && "Scale can't be larger than the value type size."
) ? void (0) : __assert_fail ("Scale <= VTSize && \"Scale can't be larger than the value type size.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3623, __extension__ __PRETTY_FUNCTION__))
;
3624
3625 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3626 SDValue LL, LH, RL, RH;
3627 GetExpandedInteger(LHS, LL, LH);
3628 GetExpandedInteger(RHS, RL, RH);
3629 SmallVector<SDValue, 4> Result;
3630
3631 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3632 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3633 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3634 LL, LH, RL, RH)) {
3635 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3636 return;
3637 }
3638
3639 unsigned NVTSize = NVT.getScalarSizeInBits();
3640 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "(static_cast <bool> ((VTSize == NVTSize * 2) &&
"Expected the new value type to be half " "the size of the current value type"
) ? void (0) : __assert_fail ("(VTSize == NVTSize * 2) && \"Expected the new value type to be half \" \"the size of the current value type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3641, __extension__ __PRETTY_FUNCTION__))
3641 "the size of the current value type")(static_cast <bool> ((VTSize == NVTSize * 2) &&
"Expected the new value type to be half " "the size of the current value type"
) ? void (0) : __assert_fail ("(VTSize == NVTSize * 2) && \"Expected the new value type to be half \" \"the size of the current value type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3641, __extension__ __PRETTY_FUNCTION__))
;
3642 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3643
3644 // After getting the multiplication result in 4 parts, we need to perform a
3645 // shift right by the amount of the scale to get the result in that scale.
3646 //
3647 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3648 // 128 bits that are cut into 4 32-bit parts:
3649 //
3650 // HH HL LH LL
3651 // |---32---|---32---|---32---|---32---|
3652 // 128 96 64 32 0
3653 //
3654 // |------VTSize-----|
3655 //
3656 // |NVTSize-|
3657 //
3658 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3659 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3660 // amount and get Lo and Hi using two funnel shifts. Or for the special case
3661 // when Scale is a multiple of NVTSize we can just pick the result without
3662 // shifting.
3663 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3664 if (Scale % NVTSize) {
3665 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3666 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3667 ShiftAmount);
3668 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3669 ShiftAmount);
3670 } else {
3671 Lo = Result[Part0];
3672 Hi = Result[Part0 + 1];
3673 }
3674
3675 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3676 if (!Saturating)
3677 return;
3678
3679 // Can not overflow when there is no integer part.
3680 if (Scale == VTSize)
3681 return;
3682
3683 // To handle saturation we must check for overflow in the multiplication.
3684 //
3685 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3686 // aren't all zeroes.
3687 //
3688 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3689 // aren't all ones or all zeroes.
3690 //
3691 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3692 // highest bit of HH determines saturation direction in the event of signed
3693 // saturation.
3694
3695 SDValue ResultHL = Result[2];
3696 SDValue ResultHH = Result[3];
3697
3698 SDValue SatMax, SatMin;
3699 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3700 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3701 EVT BoolNVT = getSetCCResultType(NVT);
3702
3703 if (!Signed) {
3704 if (Scale < NVTSize) {
3705 // Overflow happened if ((HH | (HL >> Scale)) != 0).
3706 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3707 DAG.getConstant(Scale, dl, ShiftTy));
3708 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3709 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3710 } else if (Scale == NVTSize) {
3711 // Overflow happened if (HH != 0).
3712 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3713 } else if (Scale < VTSize) {
3714 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3715 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3716 DAG.getConstant(Scale - NVTSize, dl,
3717 ShiftTy));
3718 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3719 } else
3720 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"::llvm::llvm_unreachable_internal("Scale must be less or equal to VTSize for UMULFIXSAT"
"(and saturation can't happen with Scale==VTSize).", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3721)
3721 "(and saturation can't happen with Scale==VTSize).")::llvm::llvm_unreachable_internal("Scale must be less or equal to VTSize for UMULFIXSAT"
"(and saturation can't happen with Scale==VTSize).", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3721)
;
3722
3723 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3724 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3725 return;
3726 }
3727
3728 if (Scale < NVTSize) {
3729 // The number of overflow bits we can check are VTSize - Scale + 1 (we
3730 // include the sign bit). If these top bits are > 0, then we overflowed past
3731 // the max value. If these top bits are < -1, then we overflowed past the
3732 // min value. Otherwise, we did not overflow.
3733 unsigned OverflowBits = VTSize - Scale + 1;
3734 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&(static_cast <bool> (OverflowBits <= VTSize &&
OverflowBits > NVTSize && "Extent of overflow bits must start within HL"
) ? void (0) : __assert_fail ("OverflowBits <= VTSize && OverflowBits > NVTSize && \"Extent of overflow bits must start within HL\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3735, __extension__ __PRETTY_FUNCTION__))
3735 "Extent of overflow bits must start within HL")(static_cast <bool> (OverflowBits <= VTSize &&
OverflowBits > NVTSize && "Extent of overflow bits must start within HL"
) ? void (0) : __assert_fail ("OverflowBits <= VTSize && OverflowBits > NVTSize && \"Extent of overflow bits must start within HL\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3735, __extension__ __PRETTY_FUNCTION__))
;
3736 SDValue HLHiMask = DAG.getConstant(
3737 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3738 SDValue HLLoMask = DAG.getConstant(
3739 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3740 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3741 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3742 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3743 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3744 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3745 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3746 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3747 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3748 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3749 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3750 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3751 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3752 } else if (Scale == NVTSize) {
3753 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3754 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3755 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3756 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3757 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3758 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3759 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3760 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3761 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3762 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3763 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3764 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3765 } else if (Scale < VTSize) {
3766 // This is similar to the case when we saturate if Scale < NVTSize, but we
3767 // only need to check HH.
3768 unsigned OverflowBits = VTSize - Scale + 1;
3769 SDValue HHHiMask = DAG.getConstant(
3770 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3771 SDValue HHLoMask = DAG.getConstant(
3772 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3773 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3774 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3775 } else
3776 llvm_unreachable("Illegal scale for signed fixed point mul.")::llvm::llvm_unreachable_internal("Illegal scale for signed fixed point mul."
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3776)
;
3777
3778 // Saturate to signed maximum.
3779 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3780 APInt MaxLo = APInt::getAllOnes(NVTSize);
3781 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3782 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3783 // Saturate to signed minimum.
3784 APInt MinHi = APInt::getSignedMinValue(NVTSize);
3785 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3786 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3787}
3788
3789void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3790 SDValue &Hi) {
3791 SDLoc dl(N);
3792 // Try expanding in the existing type first.
3793 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3794 N->getOperand(1),
3795 N->getConstantOperandVal(2), DAG);
3796
3797 if (!Res)
3798 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3799 N->getConstantOperandVal(2), TLI, DAG);
3800 SplitInteger(Res, Lo, Hi);
3801}
3802
3803void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3804 SDValue &Lo, SDValue &Hi) {
3805 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&(static_cast <bool> ((Node->getOpcode() == ISD::SADDO
|| Node->getOpcode() == ISD::SSUBO) && "Node has unexpected Opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && \"Node has unexpected Opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3806, __extension__ __PRETTY_FUNCTION__))
3806 "Node has unexpected Opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SADDO
|| Node->getOpcode() == ISD::SSUBO) && "Node has unexpected Opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && \"Node has unexpected Opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3806, __extension__ __PRETTY_FUNCTION__))
;
3807 SDValue LHS = Node->getOperand(0);
3808 SDValue RHS = Node->getOperand(1);
3809 SDLoc dl(Node);
3810
3811 SDValue Ovf;
3812
3813 bool IsAdd = Node->getOpcode() == ISD::SADDO;
3814 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
3815
3816 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3817 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3818
3819 if (HasCarryOp) {
3820 // Expand the subcomponents.
3821 SDValue LHSL, LHSH, RHSL, RHSH;
3822 GetExpandedInteger(LHS, LHSL, LHSH);
3823 GetExpandedInteger(RHS, RHSL, RHSH);
3824 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
3825
3826 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
3827 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3828
3829 Ovf = Hi.getValue(1);
3830 } else {
3831 // Expand the result by simply replacing it with the equivalent
3832 // non-overflow-checking operation.
3833 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3834 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3835 LHS, RHS);
3836 SplitInteger(Sum, Lo, Hi);
3837
3838 // Compute the overflow.
3839 //
3840 // LHSSign -> LHS < 0
3841 // RHSSign -> RHS < 0
3842 // SumSign -> Sum < 0
3843 //
3844 // Add:
3845 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3846 // Sub:
3847 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3848 //
3849 // To get better codegen we can rewrite this by doing bitwise math on
3850 // the integers and extract the final sign bit at the end. So the
3851 // above becomes:
3852 //
3853 // Add:
3854 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
3855 // Sub:
3856 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
3857 //
3858 // NOTE: This is different than the expansion we do in expandSADDSUBO
3859 // because it is more costly to determine the RHS is > 0 for SSUBO with the
3860 // integers split.
3861 EVT VT = LHS.getValueType();
3862 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3863 if (IsAdd)
3864 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
3865
3866 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
3867 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
3868 EVT OType = Node->getValueType(1);
3869 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
3870 }
3871
3872 // Use the calculated overflow everywhere.
3873 ReplaceValueWith(SDValue(Node, 1), Ovf);
3874}
3875
3876void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3877 SDValue &Lo, SDValue &Hi) {
3878 EVT VT = N->getValueType(0);
3879 SDLoc dl(N);
3880 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3881
3882 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3883 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3884 SplitInteger(Res.getValue(0), Lo, Hi);
3885 return;
3886 }
3887
3888 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3889 if (VT == MVT::i16)
3890 LC = RTLIB::SDIV_I16;
3891 else if (VT == MVT::i32)
3892 LC = RTLIB::SDIV_I32;
3893 else if (VT == MVT::i64)
3894 LC = RTLIB::SDIV_I64;
3895 else if (VT == MVT::i128)
3896 LC = RTLIB::SDIV_I128;
3897 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported SDIV!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3897, __extension__ __PRETTY_FUNCTION__))
;
3898
3899 TargetLowering::MakeLibCallOptions CallOptions;
3900 CallOptions.setSExt(true);
3901 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3902}
3903
3904void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3905 SDValue &Lo, SDValue &Hi) {
3906 EVT VT = N->getValueType(0);
3907 SDLoc dl(N);
3908
3909 // If we can emit an efficient shift operation, do so now. Check to see if
3910 // the RHS is a constant.
3911 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
7
Assuming 'CN' is null
8
Taking false branch
3912 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3913
3914 // If we can determine that the high bit of the shift is zero or one, even if
3915 // the low bits are variable, emit this shift in an optimized form.
3916 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
9
Calling 'DAGTypeLegalizer::ExpandShiftWithKnownAmountBit'
3917 return;
3918
3919 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3920 unsigned PartsOpc;
3921 if (N->getOpcode() == ISD::SHL) {
3922 PartsOpc = ISD::SHL_PARTS;
3923 } else if (N->getOpcode() == ISD::SRL) {
3924 PartsOpc = ISD::SRL_PARTS;
3925 } else {
3926 assert(N->getOpcode() == ISD::SRA && "Unknown shift!")(static_cast <bool> (N->getOpcode() == ISD::SRA &&
"Unknown shift!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SRA && \"Unknown shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3926, __extension__ __PRETTY_FUNCTION__))
;
3927 PartsOpc = ISD::SRA_PARTS;
3928 }
3929
3930 // Next check to see if the target supports this SHL_PARTS operation or if it
3931 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3932 // size, but create a libcall instead.
3933 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3934 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3935 const bool LegalOrCustom =
3936 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3937 Action == TargetLowering::Custom;
3938
3939 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3940 // Expand the subcomponents.
3941 SDValue LHSL, LHSH;
3942 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3943 EVT VT = LHSL.getValueType();
3944
3945 // If the shift amount operand is coming from a vector legalization it may
3946 // have an illegal type. Fix that first by casting the operand, otherwise
3947 // the new SHL_PARTS operation would need further legalization.
3948 SDValue ShiftOp = N->getOperand(1);
3949 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3950 assert(ShiftTy.getScalarSizeInBits() >=(static_cast <bool> (ShiftTy.getScalarSizeInBits() >=
Log2_32_Ceil(VT.getScalarSizeInBits()) && "ShiftAmountTy is too small to cover the range of this type!"
) ? void (0) : __assert_fail ("ShiftTy.getScalarSizeInBits() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && \"ShiftAmountTy is too small to cover the range of this type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3952, __extension__ __PRETTY_FUNCTION__))
3951 Log2_32_Ceil(VT.getScalarSizeInBits()) &&(static_cast <bool> (ShiftTy.getScalarSizeInBits() >=
Log2_32_Ceil(VT.getScalarSizeInBits()) && "ShiftAmountTy is too small to cover the range of this type!"
) ? void (0) : __assert_fail ("ShiftTy.getScalarSizeInBits() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && \"ShiftAmountTy is too small to cover the range of this type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3952, __extension__ __PRETTY_FUNCTION__))
3952 "ShiftAmountTy is too small to cover the range of this type!")(static_cast <bool> (ShiftTy.getScalarSizeInBits() >=
Log2_32_Ceil(VT.getScalarSizeInBits()) && "ShiftAmountTy is too small to cover the range of this type!"
) ? void (0) : __assert_fail ("ShiftTy.getScalarSizeInBits() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && \"ShiftAmountTy is too small to cover the range of this type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3952, __extension__ __PRETTY_FUNCTION__))
;
3953 if (ShiftOp.getValueType() != ShiftTy)
3954 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3955
3956 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3957 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3958 Hi = Lo.getValue(1);
3959 return;
3960 }
3961
3962 // Otherwise, emit a libcall.
3963 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3964 bool isSigned;
3965 if (N->getOpcode() == ISD::SHL) {
3966 isSigned = false; /*sign irrelevant*/
3967 if (VT == MVT::i16)
3968 LC = RTLIB::SHL_I16;
3969 else if (VT == MVT::i32)
3970 LC = RTLIB::SHL_I32;
3971 else if (VT == MVT::i64)
3972 LC = RTLIB::SHL_I64;
3973 else if (VT == MVT::i128)
3974 LC = RTLIB::SHL_I128;
3975 } else if (N->getOpcode() == ISD::SRL) {
3976 isSigned = false;
3977 if (VT == MVT::i16)
3978 LC = RTLIB::SRL_I16;
3979 else if (VT == MVT::i32)
3980 LC = RTLIB::SRL_I32;
3981 else if (VT == MVT::i64)
3982 LC = RTLIB::SRL_I64;
3983 else if (VT == MVT::i128)
3984 LC = RTLIB::SRL_I128;
3985 } else {
3986 assert(N->getOpcode() == ISD::SRA && "Unknown shift!")(static_cast <bool> (N->getOpcode() == ISD::SRA &&
"Unknown shift!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SRA && \"Unknown shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3986, __extension__ __PRETTY_FUNCTION__))
;
3987 isSigned = true;
3988 if (VT == MVT::i16)
3989 LC = RTLIB::SRA_I16;
3990 else if (VT == MVT::i32)
3991 LC = RTLIB::SRA_I32;
3992 else if (VT == MVT::i64)
3993 LC = RTLIB::SRA_I64;
3994 else if (VT == MVT::i128)
3995 LC = RTLIB::SRA_I128;
3996 }
3997
3998 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3999 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4000 TargetLowering::MakeLibCallOptions CallOptions;
4001 CallOptions.setSExt(isSigned);
4002 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4003 return;
4004 }
4005
4006 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
4007 llvm_unreachable("Unsupported shift!")::llvm::llvm_unreachable_internal("Unsupported shift!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4007)
;
4008}
4009
4010void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
4011 SDValue &Lo, SDValue &Hi) {
4012 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4013 SDLoc dl(N);
4014 SDValue Op = N->getOperand(0);
4015 if (Op.getValueType().bitsLE(NVT)) {
4016 // The low part is sign extension of the input (degenerates to a copy).
4017 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
4018 // The high part is obtained by SRA'ing all but one of the bits of low part.
4019 unsigned LoSize = NVT.getSizeInBits();
4020 Hi = DAG.getNode(
4021 ISD::SRA, dl, NVT, Lo,
4022 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
4023 } else {
4024 // For example, extension of an i48 to an i64. The operand type necessarily
4025 // promotes to the result type, so will end up being expanded too.
4026 assert(getTypeAction(Op.getValueType()) ==(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4028, __extension__ __PRETTY_FUNCTION__))
4027 TargetLowering::TypePromoteInteger &&(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4028, __extension__ __PRETTY_FUNCTION__))
4028 "Only know how to promote this result!")(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4028, __extension__ __PRETTY_FUNCTION__))
;
4029 SDValue Res = GetPromotedInteger(Op);
4030 assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4031, __extension__ __PRETTY_FUNCTION__))
4031 "Operand over promoted?")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4031, __extension__ __PRETTY_FUNCTION__))
;
4032 // Split the promoted operand. This will simplify when it is expanded.
4033 SplitInteger(Res, Lo, Hi);
4034 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4035 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4036 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4037 ExcessBits)));
4038 }
4039}
4040
4041void DAGTypeLegalizer::
4042ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4043 SDLoc dl(N);
4044 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4045 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4046
4047 if (EVT.bitsLE(Lo.getValueType())) {
4048 // sext_inreg the low part if needed.
4049 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
4050 N->getOperand(1));
4051
4052 // The high part gets the sign extension from the lo-part. This handles
4053 // things like sextinreg V:i64 from i8.
4054 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
4055 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
4056 TLI.getPointerTy(DAG.getDataLayout())));
4057 } else {
4058 // For example, extension of an i48 to an i64. Leave the low part alone,
4059 // sext_inreg the high part.
4060 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
4061 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4062 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4063 ExcessBits)));
4064 }
4065}
4066
4067void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
4068 SDValue &Lo, SDValue &Hi) {
4069 EVT VT = N->getValueType(0);
4070 SDLoc dl(N);
4071 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4072
4073 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
4074 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4075 SplitInteger(Res.getValue(1), Lo, Hi);
4076 return;
4077 }
4078
4079 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4080 if (VT == MVT::i16)
4081 LC = RTLIB::SREM_I16;
4082 else if (VT == MVT::i32)
4083 LC = RTLIB::SREM_I32;
4084 else if (VT == MVT::i64)
4085 LC = RTLIB::SREM_I64;
4086 else if (VT == MVT::i128)
4087 LC = RTLIB::SREM_I128;
4088 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported SREM!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4088, __extension__ __PRETTY_FUNCTION__))
;
4089
4090 TargetLowering::MakeLibCallOptions CallOptions;
4091 CallOptions.setSExt(true);
4092 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4093}
4094
4095void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
4096 SDValue &Lo, SDValue &Hi) {
4097 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4098 SDLoc dl(N);
4099 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
4100 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
4101 N->getOperand(0),
4102 DAG.getConstant(NVT.getSizeInBits(), dl,
4103 TLI.getPointerTy(DAG.getDataLayout())));
4104 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
4105}
4106
4107void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
4108 SDValue &Lo, SDValue &Hi) {
4109 EVT VT = N->getValueType(0);
4110 SDLoc dl(N);
4111
4112 if (N->getOpcode() == ISD::UMULO) {
4113 // This section expands the operation into the following sequence of
4114 // instructions. `iNh` here refers to a type which has half the bit width of
4115 // the type the original operation operated on.
4116 //
4117 // %0 = %LHS.HI != 0 && %RHS.HI != 0
4118 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
4119 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
4120 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
4121 // %4 = add iNh %1.0, %2.0 as iN
4122 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
4123 //
4124 // %lo = %3.LO
4125 // %hi = %5.0
4126 // %ovf = %0 || %1.1 || %2.1 || %5.1
4127 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
4128 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4129 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4130 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4131 EVT HalfVT = LHSLow.getValueType();
4132 EVT BitVT = N->getValueType(1);
4133 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
4134
4135 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
4136 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
4137 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
4138 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
4139
4140 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
4141 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
4142
4143 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
4144 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
4145
4146 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
4147
4148 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
4149 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
4150 // operation recursively legalized?).
4151 //
4152 // Many backends understand this pattern and will convert into LOHI
4153 // themselves, if applicable.
4154 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
4155 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
4156 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
4157 SplitInteger(Three, Lo, Hi);
4158
4159 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
4160 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
4161 ReplaceValueWith(SDValue(N, 1), Overflow);
4162 return;
4163 }
4164
4165 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
4166 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
4167 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
4168
4169 // Replace this with a libcall that will check overflow.
4170 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4171 if (VT == MVT::i32)
4172 LC = RTLIB::MULO_I32;
4173 else if (VT == MVT::i64)
4174 LC = RTLIB::MULO_I64;
4175 else if (VT == MVT::i128)
4176 LC = RTLIB::MULO_I128;
4177
4178 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
4179 // FIXME: This is not an optimal expansion, but better than crashing.
4180 EVT WideVT =
4181 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
4182 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0));
4183 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1));
4184 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
4185 SDValue MulLo, MulHi;
4186 SplitInteger(Mul, MulLo, MulHi);
4187 SDValue SRA =
4188 DAG.getNode(ISD::SRA, dl, VT, MulLo,
4189 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT));
4190 SDValue Overflow =
4191 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
4192 SplitInteger(MulLo, Lo, Hi);
4193 ReplaceValueWith(SDValue(N, 1), Overflow);
4194 return;
4195 }
4196
4197 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
4198 // Temporary for the overflow value, default it to zero.
4199 SDValue Chain =
4200 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
4201 MachinePointerInfo());
4202
4203 TargetLowering::ArgListTy Args;
4204 TargetLowering::ArgListEntry Entry;
4205 for (const SDValue &Op : N->op_values()) {
4206 EVT ArgVT = Op.getValueType();
4207 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
4208 Entry.Node = Op;
4209 Entry.Ty = ArgTy;
4210 Entry.IsSExt = true;
4211 Entry.IsZExt = false;
4212 Args.push_back(Entry);
4213 }
4214
4215 // Also pass the address of the overflow check.
4216 Entry.Node = Temp;
4217 Entry.Ty = PtrTy->getPointerTo();
4218 Entry.IsSExt = true;
4219 Entry.IsZExt = false;
4220 Args.push_back(Entry);
4221
4222 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
4223
4224 TargetLowering::CallLoweringInfo CLI(DAG);
4225 CLI.setDebugLoc(dl)
4226 .setChain(Chain)
4227 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
4228 .setSExtResult();
4229
4230 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
4231
4232 SplitInteger(CallInfo.first, Lo, Hi);
4233 SDValue Temp2 =
4234 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
4235 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
4236 DAG.getConstant(0, dl, PtrVT),
4237 ISD::SETNE);
4238 // Use the overflow from the libcall everywhere.
4239 ReplaceValueWith(SDValue(N, 1), Ofl);
4240}
4241
4242void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
4243 SDValue &Lo, SDValue &Hi) {
4244 EVT VT = N->getValueType(0);
4245 SDLoc dl(N);
4246 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4247
4248 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4249 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4250 SplitInteger(Res.getValue(0), Lo, Hi);
4251 return;
4252 }
4253
4254 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4255 if (VT == MVT::i16)
4256 LC = RTLIB::UDIV_I16;
4257 else if (VT == MVT::i32)
4258 LC = RTLIB::UDIV_I32;
4259 else if (VT == MVT::i64)
4260 LC = RTLIB::UDIV_I64;
4261 else if (VT == MVT::i128)
4262 LC = RTLIB::UDIV_I128;
4263 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported UDIV!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4263, __extension__ __PRETTY_FUNCTION__))
;
4264
4265 TargetLowering::MakeLibCallOptions CallOptions;
4266 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4267}
4268
4269void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
4270 SDValue &Lo, SDValue &Hi) {
4271 EVT VT = N->getValueType(0);
4272 SDLoc dl(N);
4273 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4274
4275 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4276 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4277 SplitInteger(Res.getValue(1), Lo, Hi);
4278 return;
4279 }
4280
4281 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4282 if (VT == MVT::i16)
4283 LC = RTLIB::UREM_I16;
4284 else if (VT == MVT::i32)
4285 LC = RTLIB::UREM_I32;
4286 else if (VT == MVT::i64)
4287 LC = RTLIB::UREM_I64;
4288 else if (VT == MVT::i128)
4289 LC = RTLIB::UREM_I128;
4290 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported UREM!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4290, __extension__ __PRETTY_FUNCTION__))
;
4291
4292 TargetLowering::MakeLibCallOptions CallOptions;
4293 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4294}
4295
4296void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
4297 SDValue &Lo, SDValue &Hi) {
4298 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4299 SDLoc dl(N);
4300 SDValue Op = N->getOperand(0);
4301 if (Op.getValueType().bitsLE(NVT)) {
4302 // The low part is zero extension of the input (degenerates to a copy).
4303 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
4304 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
4305 } else {
4306 // For example, extension of an i48 to an i64. The operand type necessarily
4307 // promotes to the result type, so will end up being expanded too.
4308 assert(getTypeAction(Op.getValueType()) ==(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4310, __extension__ __PRETTY_FUNCTION__))
4309 TargetLowering::TypePromoteInteger &&(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4310, __extension__ __PRETTY_FUNCTION__))
4310 "Only know how to promote this result!")(static_cast <bool> (getTypeAction(Op.getValueType()) ==
TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4310, __extension__ __PRETTY_FUNCTION__))
;
4311 SDValue Res = GetPromotedInteger(Op);
4312 assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4313, __extension__ __PRETTY_FUNCTION__))
4313 "Operand over promoted?")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4313, __extension__ __PRETTY_FUNCTION__))
;
4314 // Split the promoted operand. This will simplify when it is expanded.
4315 SplitInteger(Res, Lo, Hi);
4316 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4317 Hi = DAG.getZeroExtendInReg(Hi, dl,
4318 EVT::getIntegerVT(*DAG.getContext(),
4319 ExcessBits));
4320 }
4321}
4322
4323void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
4324 SDValue &Lo, SDValue &Hi) {
4325 SDLoc dl(N);
4326 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
4327 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
4328 SDValue Zero = DAG.getConstant(0, dl, VT);
4329 SDValue Swap = DAG.getAtomicCmpSwap(
4330 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
4331 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
4332 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
4333
4334 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
4335 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
4336}
4337
4338void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
4339 SDValue &Lo, SDValue &Hi) {
4340 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
4341 // both halves independently.
4342 SDValue Res = TLI.expandVecReduce(N, DAG);
4343 SplitInteger(Res, Lo, Hi);
4344}
4345
4346void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
4347 SDValue &Lo, SDValue &Hi) {
4348 // Lower the rotate to shifts and ORs which can be expanded.
4349 SDValue Res;
4350 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
4351 SplitInteger(Res, Lo, Hi);
4352}
4353
4354void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N,
4355 SDValue &Lo, SDValue &Hi) {
4356 // Lower the funnel shift to shifts and ORs which can be expanded.
4357 SDValue Res;
4358 TLI.expandFunnelShift(N, Res, DAG);
4359 SplitInteger(Res, Lo, Hi);
4360}
4361
4362void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
4363 SDValue &Hi) {
4364 EVT VT = N->getValueType(0);
4365 EVT HalfVT =
4366 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
4367 SDLoc dl(N);
4368
4369 // We assume VSCALE(1) fits into a legal integer.
4370 APInt One(HalfVT.getSizeInBits(), 1);
4371 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
4372 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
4373 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
4374 SplitInteger(Res, Lo, Hi);
4375}
4376
4377//===----------------------------------------------------------------------===//
4378// Integer Operand Expansion
4379//===----------------------------------------------------------------------===//
4380
4381/// ExpandIntegerOperand - This method is called when the specified operand of
4382/// the specified node is found to need expansion. At this point, all of the
4383/// result types of the node are known to be legal, but other operands of the
4384/// node may need promotion or expansion as well as the specified one.
4385bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
4386 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
4387 dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
;
4388 SDValue Res = SDValue();
4389
4390 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4391 return false;
4392
4393 switch (N->getOpcode()) {
4394 default:
4395 #ifndef NDEBUG
4396 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
4397 N->dump(&DAG); dbgs() << "\n";
4398 #endif
4399 report_fatal_error("Do not know how to expand this operator's operand!");
4400
4401 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
4402 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
4403 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
4404 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
4405 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
4406 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
4407 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
4408 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
4409 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
4410 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
4411 case ISD::STRICT_SINT_TO_FP:
4412 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
4413 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
4414 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
4415 case ISD::STRICT_UINT_TO_FP:
4416 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
4417
4418 case ISD::SHL:
4419 case ISD::SRA:
4420 case ISD::SRL:
4421 case ISD::ROTL:
4422 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
4423 case ISD::RETURNADDR:
4424 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
4425
4426 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
4427 }
4428
4429 // If the result is null, the sub-method took care of registering results etc.
4430 if (!Res.getNode()) return false;
4431
4432 // If the result is N, the sub-method updated N in place. Tell the legalizer
4433 // core about this.
4434 if (Res.getNode() == N)
4435 return true;
4436
4437 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == 1 && "Invalid operand expansion"
) ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && \"Invalid operand expansion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4438, __extension__ __PRETTY_FUNCTION__))
4438 "Invalid operand expansion")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == 1 && "Invalid operand expansion"
) ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && \"Invalid operand expansion\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4438, __extension__ __PRETTY_FUNCTION__))
;
4439
4440 ReplaceValueWith(SDValue(N, 0), Res);
4441 return false;
4442}
4443
4444/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
4445/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
4446void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
4447 SDValue &NewRHS,
4448 ISD::CondCode &CCCode,
4449 const SDLoc &dl) {
4450 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4451 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
4452 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
4453
4454 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
4455 if (RHSLo == RHSHi) {
4456 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
4457 if (RHSCST->isAllOnes()) {
4458 // Equality comparison to -1.
4459 NewLHS = DAG.getNode(ISD::AND, dl,
4460 LHSLo.getValueType(), LHSLo, LHSHi);
4461 NewRHS = RHSLo;
4462 return;
4463 }
4464 }
4465 }
4466
4467 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
4468 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
4469 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
4470 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
4471 return;
4472 }
4473
4474 // If this is a comparison of the sign bit, just look at the top part.
4475 // X > -1, x < 0
4476 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
4477 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
4478 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
4479 NewLHS = LHSHi;
4480 NewRHS = RHSHi;
4481 return;
4482 }
4483
4484 // FIXME: This generated code sucks.
4485 ISD::CondCode LowCC;
4486 switch (CCCode) {
4487 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4487)
;
4488 case ISD::SETLT:
4489 case ISD::SETULT: LowCC = ISD::SETULT; break;
4490 case ISD::SETGT:
4491 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4492 case ISD::SETLE:
4493 case ISD::SETULE: LowCC = ISD::SETULE; break;
4494 case ISD::SETGE:
4495 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4496 }
4497
4498 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
4499 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
4500 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
4501
4502 // NOTE: on targets without efficient SELECT of bools, we can always use
4503 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4504 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
4505 nullptr);
4506 SDValue LoCmp, HiCmp;
4507 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
4508 TLI.isTypeLegal(RHSLo.getValueType()))
4509 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
4510 RHSLo, LowCC, false, DagCombineInfo, dl);
4511 if (!LoCmp.getNode())
4512 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
4513 RHSLo, LowCC);
4514 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
4515 TLI.isTypeLegal(RHSHi.getValueType()))
4516 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
4517 RHSHi, CCCode, false, DagCombineInfo, dl);
4518 if (!HiCmp.getNode())
4519 HiCmp =
4520 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
4521 LHSHi, RHSHi, DAG.getCondCode(CCCode));
4522
4523 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
4524 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
4525
4526 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4527 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
4528
4529 // FIXME: Is the HiCmpC->isOne() here correct for
4530 // ZeroOrNegativeOneBooleanContent.
4531 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
4532 (!EqAllowed &&
4533 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
4534 // For LE / GE, if high part is known false, ignore the low part.
4535 // For LT / GT: if low part is known false, return the high part.
4536 // if high part is known true, ignore the low part.
4537 NewLHS = HiCmp;
4538 NewRHS = SDValue();
4539 return;
4540 }
4541
4542 if (LHSHi == RHSHi) {
4543 // Comparing the low bits is enough.
4544 NewLHS = LoCmp;
4545 NewRHS = SDValue();
4546 return;
4547 }
4548
4549 // Lower with SETCCCARRY if the target supports it.
4550 EVT HiVT = LHSHi.getValueType();
4551 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
4552 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
4553
4554 // FIXME: Make all targets support this, then remove the other lowering.
4555 if (HasSETCCCARRY) {
4556 // SETCCCARRY can detect < and >= directly. For > and <=, flip
4557 // operands and condition code.
4558 bool FlipOperands = false;
4559 switch (CCCode) {
4560 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
4561 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
4562 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
4563 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
4564 default: break;
4565 }
4566 if (FlipOperands) {
4567 std::swap(LHSLo, RHSLo);
4568 std::swap(LHSHi, RHSHi);
4569 }
4570 // Perform a wide subtraction, feeding the carry from the low part into
4571 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
4572 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
4573 // zero or positive iff LHS >= RHS.
4574 EVT LoVT = LHSLo.getValueType();
4575 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
4576 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
4577 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
4578 LHSHi, RHSHi, LowCmp.getValue(1),
4579 DAG.getCondCode(CCCode));
4580 NewLHS = Res;
4581 NewRHS = SDValue();
4582 return;
4583 }
4584
4585 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
4586 false, DagCombineInfo, dl);
4587 if (!NewLHS.getNode())
4588 NewLHS =
4589 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
4590 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
4591 NewRHS = SDValue();
4592}
4593
4594SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
4595 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
4596 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
4597 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4598
4599 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4600 // against zero to select between true and false values.
4601 if (!NewRHS.getNode()) {
4602 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4603 CCCode = ISD::SETNE;
4604 }
4605
4606 // Update N to have the operands specified.
4607 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
4608 DAG.getCondCode(CCCode), NewLHS, NewRHS,
4609 N->getOperand(4)), 0);
4610}
4611
4612SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
4613 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4614 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
4615 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4616
4617 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4618 // against zero to select between true and false values.
4619 if (!NewRHS.getNode()) {
4620 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4621 CCCode = ISD::SETNE;
4622 }
4623
4624 // Update N to have the operands specified.
4625 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
4626 N->getOperand(2), N->getOperand(3),
4627 DAG.getCondCode(CCCode)), 0);
4628}
4629
4630SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
4631 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4632 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
4633 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4634
4635 // If ExpandSetCCOperands returned a scalar, use it.
4636 if (!NewRHS.getNode()) {
4637 assert(NewLHS.getValueType() == N->getValueType(0) &&(static_cast <bool> (NewLHS.getValueType() == N->getValueType
(0) && "Unexpected setcc expansion!") ? void (0) : __assert_fail
("NewLHS.getValueType() == N->getValueType(0) && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4638, __extension__ __PRETTY_FUNCTION__))
4638 "Unexpected setcc expansion!")(static_cast <bool> (NewLHS.getValueType() == N->getValueType
(0) && "Unexpected setcc expansion!") ? void (0) : __assert_fail
("NewLHS.getValueType() == N->getValueType(0) && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4638, __extension__ __PRETTY_FUNCTION__))
;
4639 return NewLHS;
4640 }
4641
4642 // Otherwise, update N to have the operands specified.
4643 return SDValue(
4644 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4645}
4646
4647SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
4648 SDValue LHS = N->getOperand(0);
4649 SDValue RHS = N->getOperand(1);
4650 SDValue Carry = N->getOperand(2);
4651 SDValue Cond = N->getOperand(3);
4652 SDLoc dl = SDLoc(N);
4653
4654 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4655 GetExpandedInteger(LHS, LHSLo, LHSHi);
4656 GetExpandedInteger(RHS, RHSLo, RHSHi);
4657
4658 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
4659 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
4660 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
4661 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
4662 LowCmp.getValue(1), Cond);
4663}
4664
4665SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
4666 // Split the operand and replace with SPLAT_VECTOR_PARTS.
4667 SDValue Lo, Hi;
4668 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4669 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
4670 Hi);
4671}
4672
4673SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
4674 // The value being shifted is legal, but the shift amount is too big.
4675 // It follows that either the result of the shift is undefined, or the
4676 // upper half of the shift amount is zero. Just use the lower half.
4677 SDValue Lo, Hi;
4678 GetExpandedInteger(N->getOperand(1), Lo, Hi);
4679 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4680}
4681
4682SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
4683 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
4684 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
4685 // constant to valid type.
4686 SDValue Lo, Hi;
4687 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4688 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4689}
4690
4691SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
4692 bool IsStrict = N->isStrictFPOpcode();
4693 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4694 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4695 EVT DstVT = N->getValueType(0);
4696 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
4697 assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Don't know how to expand this SINT_TO_FP!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this SINT_TO_FP!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4698, __extension__ __PRETTY_FUNCTION__))
4698 "Don't know how to expand this SINT_TO_FP!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Don't know how to expand this SINT_TO_FP!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this SINT_TO_FP!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4698, __extension__ __PRETTY_FUNCTION__))
;
4699 TargetLowering::MakeLibCallOptions CallOptions;
4700 CallOptions.setSExt(true);
4701 std::pair<SDValue, SDValue> Tmp =
4702 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4703
4704 if (!IsStrict)
4705 return Tmp.first;
4706
4707 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4708 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4709 return SDValue();
4710}
4711
4712SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
4713 if (N->isAtomic()) {
4714 // It's typical to have larger CAS than atomic store instructions.
4715 SDLoc dl(N);
4716 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4717 N->getMemoryVT(),
4718 N->getOperand(0), N->getOperand(2),
4719 N->getOperand(1),
4720 N->getMemOperand());
4721 return Swap.getValue(1);
4722 }
4723 if (ISD::isNormalStore(N))
4724 return ExpandOp_NormalStore(N, OpNo);
4725
4726 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDStore(N) &&
"Indexed store during type legalization!") ? void (0) : __assert_fail
("ISD::isUNINDEXEDStore(N) && \"Indexed store during type legalization!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4726, __extension__ __PRETTY_FUNCTION__))
;
4727 assert(OpNo == 1 && "Can only expand the stored value so far")(static_cast <bool> (OpNo == 1 && "Can only expand the stored value so far"
) ? void (0) : __assert_fail ("OpNo == 1 && \"Can only expand the stored value so far\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4727, __extension__ __PRETTY_FUNCTION__))
;
4728
4729 EVT VT = N->getOperand(1).getValueType();
4730 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4731 SDValue Ch = N->getChain();
4732 SDValue Ptr = N->getBasePtr();
4733 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4734 AAMDNodes AAInfo = N->getAAInfo();
4735 SDLoc dl(N);
4736 SDValue Lo, Hi;
4737
4738 assert(NVT.isByteSized() && "Expanded type not byte sized!")(static_cast <bool> (NVT.isByteSized() && "Expanded type not byte sized!"
) ? void (0) : __assert_fail ("NVT.isByteSized() && \"Expanded type not byte sized!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4738, __extension__ __PRETTY_FUNCTION__))
;
4739
4740 if (N->getMemoryVT().bitsLE(NVT)) {
4741 GetExpandedInteger(N->getValue(), Lo, Hi);
4742 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4743 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
4744 AAInfo);
4745 }
4746
4747 if (DAG.getDataLayout().isLittleEndian()) {
4748 // Little-endian - low bits are at low addresses.
4749 GetExpandedInteger(N->getValue(), Lo, Hi);
4750
4751 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4752 N->getOriginalAlign(), MMOFlags, AAInfo);
4753
4754 unsigned ExcessBits =
4755 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4756 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4757
4758 // Increment the pointer to the other half.
4759 unsigned IncrementSize = NVT.getSizeInBits()/8;
4760 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4761 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
4762 N->getPointerInfo().getWithOffset(IncrementSize),
4763 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
4764 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4765 }
4766
4767 // Big-endian - high bits are at low addresses. Favor aligned stores at
4768 // the cost of some bit-fiddling.
4769 GetExpandedInteger(N->getValue(), Lo, Hi);
4770
4771 EVT ExtVT = N->getMemoryVT();
4772 unsigned EBytes = ExtVT.getStoreSize();
4773 unsigned IncrementSize = NVT.getSizeInBits()/8;
4774 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4775 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
4776 ExtVT.getSizeInBits() - ExcessBits);
4777
4778 if (ExcessBits < NVT.getSizeInBits()) {
4779 // Transfer high bits from the top of Lo to the bottom of Hi.
4780 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
4781 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4782 TLI.getPointerTy(DAG.getDataLayout())));
4783 Hi = DAG.getNode(
4784 ISD::OR, dl, NVT, Hi,
4785 DAG.getNode(ISD::SRL, dl, NVT, Lo,
4786 DAG.getConstant(ExcessBits, dl,
4787 TLI.getPointerTy(DAG.getDataLayout()))));
4788 }
4789
4790 // Store both the high bits and maybe some of the low bits.
4791 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
4792 N->getOriginalAlign(), MMOFlags, AAInfo);
4793
4794 // Increment the pointer to the other half.
4795 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4796 // Store the lowest ExcessBits bits in the second half.
4797 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
4798 N->getPointerInfo().getWithOffset(IncrementSize),
4799 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4800 N->getOriginalAlign(), MMOFlags, AAInfo);
4801 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4802}
4803
4804SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
4805 SDValue InL, InH;
4806 GetExpandedInteger(N->getOperand(0), InL, InH);
4807 // Just truncate the low part of the source.
4808 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4809}
4810
4811SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
4812 bool IsStrict = N->isStrictFPOpcode();
4813 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4814 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4815 EVT DstVT = N->getValueType(0);
4816 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
4817 assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Don't know how to expand this UINT_TO_FP!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this UINT_TO_FP!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4818, __extension__ __PRETTY_FUNCTION__))
4818 "Don't know how to expand this UINT_TO_FP!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Don't know how to expand this UINT_TO_FP!") ? void (0) : __assert_fail
("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this UINT_TO_FP!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4818, __extension__ __PRETTY_FUNCTION__))
;
4819 TargetLowering::MakeLibCallOptions CallOptions;
4820 CallOptions.setSExt(true);
4821 std::pair<SDValue, SDValue> Tmp =
4822 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4823
4824 if (!IsStrict)
4825 return Tmp.first;
4826
4827 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4828 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4829 return SDValue();
4830}
4831
4832SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4833 SDLoc dl(N);
4834 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4835 cast<AtomicSDNode>(N)->getMemoryVT(),
4836 N->getOperand(0),
4837 N->getOperand(1), N->getOperand(2),
4838 cast<AtomicSDNode>(N)->getMemOperand());
4839 return Swap.getValue(1);
4840}
4841
4842SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
4843 SDLoc dl(N);
4844
4845 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4846 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4847 EVT OutVT = V0.getValueType();
4848
4849 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
4850}
4851
4852SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4853
4854 EVT OutVT = N->getValueType(0);
4855 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4856 assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4856, __extension__ __PRETTY_FUNCTION__))
;
4857 EVT NOutVTElem = NOutVT.getVectorElementType();
4858
4859 SDLoc dl(N);
4860 SDValue BaseIdx = N->getOperand(1);
4861
4862 // TODO: We may be able to use this for types other than scalable
4863 // vectors and fix those tests that expect BUILD_VECTOR to be used
4864 if (OutVT.isScalableVector()) {
4865 SDValue InOp0 = N->getOperand(0);
4866 EVT InVT = InOp0.getValueType();
4867
4868 // Try and extract from a smaller type so that it eventually falls
4869 // into the promotion code below.
4870 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
4871 getTypeAction(InVT) == TargetLowering::TypeLegal) {
4872 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
4873 unsigned NElts = NInVT.getVectorMinNumElements();
4874 uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue();
4875
4876 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
4877 DAG.getConstant(alignDown(IdxVal, NElts), dl,
4878 BaseIdx.getValueType()));
4879 SDValue Step2 = DAG.getNode(
4880 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
4881 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
4882 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
4883 }
4884
4885 // Try and extract from a widened type.
4886 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4887 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
4888 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
4889 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4890 }
4891
4892 // Promote operands and see if this is handled by target lowering,
4893 // Otherwise, use the BUILD_VECTOR approach below
4894 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
4895 // Collect the (promoted) operands
4896 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
4897
4898 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
4899 assert(PromEltVT.bitsLE(NOutVTElem) &&(static_cast <bool> (PromEltVT.bitsLE(NOutVTElem) &&
"Promoted operand has an element type greater than result") ?
void (0) : __assert_fail ("PromEltVT.bitsLE(NOutVTElem) && \"Promoted operand has an element type greater than result\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4900, __extension__ __PRETTY_FUNCTION__))
4900 "Promoted operand has an element type greater than result")(static_cast <bool> (PromEltVT.bitsLE(NOutVTElem) &&
"Promoted operand has an element type greater than result") ?
void (0) : __assert_fail ("PromEltVT.bitsLE(NOutVTElem) && \"Promoted operand has an element type greater than result\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4900, __extension__ __PRETTY_FUNCTION__))
;
4901
4902 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
4903 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
4904 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4905 }
4906 }
4907
4908 if (OutVT.isScalableVector())
4909 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
4910
4911 SDValue InOp0 = N->getOperand(0);
4912 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4913 InOp0 = GetPromotedInteger(N->getOperand(0));
4914
4915 EVT InVT = InOp0.getValueType();
4916
4917 unsigned OutNumElems = OutVT.getVectorNumElements();
4918 SmallVector<SDValue, 8> Ops;
4919 Ops.reserve(OutNumElems);
4920 for (unsigned i = 0; i != OutNumElems; ++i) {
4921
4922 // Extract the element from the original vector.
4923 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4924 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4925 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4926 InVT.getVectorElementType(), N->getOperand(0), Index);
4927
4928 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4929 // Insert the converted element to the new vector.
4930 Ops.push_back(Op);
4931 }
4932
4933 return DAG.getBuildVector(NOutVT, dl, Ops);
4934}
4935
4936SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
4937 EVT OutVT = N->getValueType(0);
4938 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4939 assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4939, __extension__ __PRETTY_FUNCTION__))
;
4940
4941 SDLoc dl(N);
4942 SDValue Vec = N->getOperand(0);
4943 SDValue SubVec = N->getOperand(1);
4944 SDValue Idx = N->getOperand(2);
4945
4946 EVT SubVecVT = SubVec.getValueType();
4947 EVT NSubVT =
4948 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(),
4949 SubVecVT.getVectorElementCount());
4950
4951 Vec = GetPromotedInteger(Vec);
4952 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);
4953
4954 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
4955}
4956
4957SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
4958 SDLoc dl(N);
4959
4960 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4961 EVT OutVT = V0.getValueType();
4962
4963 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
4964}
4965
4966SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4967 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4968 EVT VT = N->getValueType(0);
4969 SDLoc dl(N);
4970
4971 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4972
4973 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4974 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4975 EVT OutVT = V0.getValueType();
4976
4977 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4978}
4979
4980
4981SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4982 EVT OutVT = N->getValueType(0);
4983 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4984 assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4984, __extension__ __PRETTY_FUNCTION__))
;
4985 unsigned NumElems = N->getNumOperands();
4986 EVT NOutVTElem = NOutVT.getVectorElementType();
4987
4988 SDLoc dl(N);
4989
4990 SmallVector<SDValue, 8> Ops;
4991 Ops.reserve(NumElems);
4992 for (unsigned i = 0; i != NumElems; ++i) {
4993 SDValue Op;
4994 // BUILD_VECTOR integer operand types are allowed to be larger than the
4995 // result's element type. This may still be true after the promotion. For
4996 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4997 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4998 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4999 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
5000 else
5001 Op = N->getOperand(i);
5002 Ops.push_back(Op);
5003 }
5004
5005 return DAG.getBuildVector(NOutVT, dl, Ops);
5006}
5007
5008SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
5009
5010 SDLoc dl(N);
5011
5012 assert(!N->getOperand(0).getValueType().isVector() &&(static_cast <bool> (!N->getOperand(0).getValueType(
).isVector() && "Input must be a scalar") ? void (0) :
__assert_fail ("!N->getOperand(0).getValueType().isVector() && \"Input must be a scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5013, __extension__ __PRETTY_FUNCTION__))
5013 "Input must be a scalar")(static_cast <bool> (!N->getOperand(0).getValueType(
).isVector() && "Input must be a scalar") ? void (0) :
__assert_fail ("!N->getOperand(0).getValueType().isVector() && \"Input must be a scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5013, __extension__ __PRETTY_FUNCTION__))
;
5014
5015 EVT OutVT = N->getValueType(0);
5016 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5017 assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5017, __extension__ __PRETTY_FUNCTION__))
;
5018 EVT NOutVTElem = NOutVT.getVectorElementType();
5019
5020 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
5021
5022 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
5023}
5024
5025SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
5026 SDLoc dl(N);
5027
5028 SDValue SplatVal = N->getOperand(0);
5029
5030 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar")(static_cast <bool> (!SplatVal.getValueType().isVector(
) && "Input must be a scalar") ? void (0) : __assert_fail
("!SplatVal.getValueType().isVector() && \"Input must be a scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5030, __extension__ __PRETTY_FUNCTION__))
;
5031
5032 EVT OutVT = N->getValueType(0);
5033 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5034 assert(NOutVT.isVector() && "Type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "Type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"Type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5034, __extension__ __PRETTY_FUNCTION__))
;
5035 EVT NOutElemVT = NOutVT.getVectorElementType();
5036
5037 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
5038
5039 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
5040}
5041
5042SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
5043 SDLoc dl(N);
5044 EVT OutVT = N->getValueType(0);
5045 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5046 assert(NOutVT.isVector() && "Type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "Type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"Type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5046, __extension__ __PRETTY_FUNCTION__))
;
5047 APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
5048 return DAG.getStepVector(dl, NOutVT,
5049 StepVal.sext(NOutVT.getScalarSizeInBits()));
5050}
5051
5052SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
5053 SDLoc dl(N);
5054
5055 EVT OutVT = N->getValueType(0);
5056 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5057 assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5057, __extension__ __PRETTY_FUNCTION__))
;
5058
5059 EVT OutElemTy = NOutVT.getVectorElementType();
5060
5061 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
5062 unsigned NumOutElem = NOutVT.getVectorNumElements();
5063 unsigned NumOperands = N->getNumOperands();
5064 assert(NumElem * NumOperands == NumOutElem &&(static_cast <bool> (NumElem * NumOperands == NumOutElem
&& "Unexpected number of elements") ? void (0) : __assert_fail
("NumElem * NumOperands == NumOutElem && \"Unexpected number of elements\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))
5065 "Unexpected number of elements")(static_cast <bool> (NumElem * NumOperands == NumOutElem
&& "Unexpected number of elements") ? void (0) : __assert_fail
("NumElem * NumOperands == NumOutElem && \"Unexpected number of elements\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))
;
5066
5067 // Take the elements from the first vector.
5068 SmallVector<SDValue, 8> Ops(NumOutElem);
5069 for (unsigned i = 0; i < NumOperands; ++i) {
5070 SDValue Op = N->getOperand(i);
5071 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
5072 Op = GetPromotedInteger(Op);
5073 EVT SclrTy = Op.getValueType().getVectorElementType();
5074 assert(NumElem == Op.getValueType().getVectorNumElements() &&(static_cast <bool> (NumElem == Op.getValueType().getVectorNumElements
() && "Unexpected number of elements") ? void (0) : __assert_fail
("NumElem == Op.getValueType().getVectorNumElements() && \"Unexpected number of elements\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5075, __extension__ __PRETTY_FUNCTION__))
5075 "Unexpected number of elements")(static_cast <bool> (NumElem == Op.getValueType().getVectorNumElements
() && "Unexpected number of elements") ? void (0) : __assert_fail
("NumElem == Op.getValueType().getVectorNumElements() && \"Unexpected number of elements\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5075, __extension__ __PRETTY_FUNCTION__))
;
5076
5077 for (unsigned j = 0; j < NumElem; ++j) {
5078 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
5079 DAG.getVectorIdxConstant(j, dl));
5080 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
5081 }
5082 }
5083
5084 return DAG.getBuildVector(NOutVT, dl, Ops);
5085}
5086
5087SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
5088 EVT VT = N->getValueType(0);
5089 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5090 assert(NVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5090, __extension__ __PRETTY_FUNCTION__))
;
5091
5092 SDLoc dl(N);
5093
5094 // For operands whose TypeAction is to promote, extend the promoted node
5095 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
5096 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
5097 // type..
5098 if (getTypeAction(N->getOperand(0).getValueType())
5099 == TargetLowering::TypePromoteInteger) {
5100 SDValue Promoted;
5101
5102 switch(N->getOpcode()) {
5103 case ISD::SIGN_EXTEND_VECTOR_INREG:
5104 Promoted = SExtPromotedInteger(N->getOperand(0));
5105 break;
5106 case ISD::ZERO_EXTEND_VECTOR_INREG:
5107 Promoted = ZExtPromotedInteger(N->getOperand(0));
5108 break;
5109 case ISD::ANY_EXTEND_VECTOR_INREG:
5110 Promoted = GetPromotedInteger(N->getOperand(0));
5111 break;
5112 default:
5113 llvm_unreachable("Node has unexpected Opcode")::llvm::llvm_unreachable_internal("Node has unexpected Opcode"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5113)
;
5114 }
5115 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
5116 }
5117
5118 // Directly extend to the appropriate transform-to type.
5119 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
5120}
5121
5122SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
5123 EVT OutVT = N->getValueType(0);
5124 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5125 assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 5125, __extension__ __PRETTY_FUNCTION__))
;
5126
5127 EVT NOutVTElem = NOutVT.getVectorElementType();
5128
5129 SDLoc dl(N);
5130 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5131
5132 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
5133 NOutVTElem, N->getOperand(1));
5134 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
5135 V0, ConvElem, N->getOperand(2));
5136}
5137
5138SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
5139 // The VECREDUCE result size may be larger than the element size, so
5140 // we can simply change the result type.
5141 SDLoc dl(N);
5142 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5143 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
5144}
5145
5146SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
5147 // The VP_REDUCE result size may be larger than the element size, so we can
5148 // simply change the result type. However the start value and result must be
5149 // the same.
5150 SDLoc DL(N);
5151 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
5152 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
5153 N->getOperand(1), N->getOperand(2), N->getOperand(3));
5154}
5155
5156SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
5157 SDLoc dl(N);
5158 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5159 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
5160 TLI.getVectorIdxTy(DAG.getDataLayout()));
5161 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
5162 V0->getValueType(0).getScalarType(), V0, V1);
5163
5164 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
5165 // element types. If this is the case then we need to expand the outgoing
5166 // value and not truncate it.
5167 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5168}
5169
5170SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
5171 SDLoc dl(N);
5172 // The result type is equal to the first input operand's type, so the
5173 // type that needs promoting must be the second source vector.
5174 SDValue V0 = N->getOperand(0);
5175 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5176 SDValue Idx = N->getOperand(2);
5177 EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
5178 V1.getValueType().getVectorElementType(),
5179 V0.getValueType().getVectorElementCount());
5180 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
5181 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
5182 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5183}
5184
5185SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
5186 SDLoc dl(N);
5187
5188 EVT ResVT = N->getValueType(0);
5189 unsigned NumElems = N->getNumOperands();
5190
5191 if (ResVT.isScalableVector()) {
5192 SDValue ResVec = DAG.getUNDEF(ResVT);
5193
5194 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5195 SDValue Op = N->getOperand(OpIdx);
5196 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
5197 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
5198 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
5199 }
5200
5201 return ResVec;
5202 }
5203
5204 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
5205
5206 SmallVector<SDValue, 8> NewOps;
5207 NewOps.reserve(NumElems);
5208
5209 // For each incoming vector
5210 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5211 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
5212 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
5213 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
5214
5215 for (unsigned i=0; i<NumElem; ++i) {
5216 // Extract element from incoming vector
5217 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
5218 DAG.getVectorIdxConstant(i, dl));
5219 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
5220 NewOps.push_back(Tr);
5221 }
5222 }
5223
5224 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
5225}

/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class VPLoadStoreSDNode;
513 friend class MaskedLoadStoreSDNode;
514 friend class MaskedGatherScatterSDNode;
515 friend class VPGatherScatterSDNode;
516
517 uint16_t : NumMemSDNodeBits;
518
519 // This storage is shared between disparate class hierarchies to hold an
520 // enumeration specific to the class hierarchy in use.
521 // LSBaseSDNode => enum ISD::MemIndexedMode
522 // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
523 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
524 // VPGatherScatterSDNode => enum ISD::MemIndexType
525 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
526 uint16_t AddressingMode : 3;
527 };
528 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
529
530 class LoadSDNodeBitfields {
531 friend class LoadSDNode;
532 friend class VPLoadSDNode;
533 friend class MaskedLoadSDNode;
534 friend class MaskedGatherSDNode;
535 friend class VPGatherSDNode;
536
537 uint16_t : NumLSBaseSDNodeBits;
538
539 uint16_t ExtTy : 2; // enum ISD::LoadExtType
540 uint16_t IsExpanding : 1;
541 };
542
543 class StoreSDNodeBitfields {
544 friend class StoreSDNode;
545 friend class VPStoreSDNode;
546 friend class MaskedStoreSDNode;
547 friend class MaskedScatterSDNode;
548 friend class VPScatterSDNode;
549
550 uint16_t : NumLSBaseSDNodeBits;
551
552 uint16_t IsTruncating : 1;
553 uint16_t IsCompressing : 1;
554 };
555
556 union {
557 char RawSDNodeBits[sizeof(uint16_t)];
558 SDNodeBitfields SDNodeBits;
559 ConstantSDNodeBitfields ConstantSDNodeBits;
560 MemSDNodeBitfields MemSDNodeBits;
561 LSBaseSDNodeBitfields LSBaseSDNodeBits;
562 LoadSDNodeBitfields LoadSDNodeBits;
563 StoreSDNodeBitfields StoreSDNodeBits;
564 };
565END_TWO_BYTE_PACK()
566#undef BEGIN_TWO_BYTE_PACK
567#undef END_TWO_BYTE_PACK
568
569 // RawSDNodeBits must cover the entirety of the union. This means that all of
570 // the union's members must have size <= RawSDNodeBits. We write the RHS as
571 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
572 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
573 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
574 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
575 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
576 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
577 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
578
579private:
580 friend class SelectionDAG;