Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1158, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/build-llvm -ferror-limit 19 -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-26-234817-15343-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/FoldingSet.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Triple.h"
26#include "llvm/ADT/Twine.h"
27#include "llvm/Analysis/BlockFrequencyInfo.h"
28#include "llvm/Analysis/MemoryLocation.h"
29#include "llvm/Analysis/ProfileSummaryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/CodeGen/FunctionLoweringInfo.h"
32#include "llvm/CodeGen/ISDOpcodes.h"
33#include "llvm/CodeGen/MachineBasicBlock.h"
34#include "llvm/CodeGen/MachineConstantPool.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/RuntimeLibcalls.h"
39#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
40#include "llvm/CodeGen/SelectionDAGNodes.h"
41#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
42#include "llvm/CodeGen/TargetFrameLowering.h"
43#include "llvm/CodeGen/TargetLowering.h"
44#include "llvm/CodeGen/TargetRegisterInfo.h"
45#include "llvm/CodeGen/TargetSubtargetInfo.h"
46#include "llvm/CodeGen/ValueTypes.h"
47#include "llvm/IR/Constant.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/DebugLoc.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/GlobalValue.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Type.h"
57#include "llvm/IR/Value.h"
58#include "llvm/Support/Casting.h"
59#include "llvm/Support/CodeGen.h"
60#include "llvm/Support/Compiler.h"
61#include "llvm/Support/Debug.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/KnownBits.h"
64#include "llvm/Support/MachineValueType.h"
65#include "llvm/Support/ManagedStatic.h"
66#include "llvm/Support/MathExtras.h"
67#include "llvm/Support/Mutex.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/Transforms/Utils/SizeOpts.h"
72#include <algorithm>
73#include <cassert>
74#include <cstdint>
75#include <cstdlib>
76#include <limits>
77#include <set>
78#include <string>
79#include <utility>
80#include <vector>
81
82using namespace llvm;
83
84/// makeVTList - Return an instance of the SDVTList struct initialized with the
85/// specified members.
86static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
87 SDVTList Res = {VTs, NumVTs};
88 return Res;
89}
90
91// Default null implementations of the callbacks.
92void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
93void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
94void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
95
96void SelectionDAG::DAGNodeDeletedListener::anchor() {}
97
98#define DEBUG_TYPE"selectiondag" "selectiondag"
99
100static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
101 cl::Hidden, cl::init(true),
102 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
103
104static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
105 cl::desc("Number limit for gluing ld/st of memcpy."),
106 cl::Hidden, cl::init(0));
107
108static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
109 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("selectiondag")) { dbgs() << Msg; V.getNode()->dump
(G);; } } while (false)
;
110}
111
112//===----------------------------------------------------------------------===//
113// ConstantFPSDNode Class
114//===----------------------------------------------------------------------===//
115
116/// isExactlyValue - We don't rely on operator== working on double values, as
117/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
118/// As such, this method can be used to do an exact bit-for-bit comparison of
119/// two floating point values.
120bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
121 return getValueAPF().bitwiseIsEqual(V);
122}
123
124bool ConstantFPSDNode::isValueValidForType(EVT VT,
125 const APFloat& Val) {
126 assert(VT.isFloatingPoint() && "Can only convert between FP types")(static_cast <bool> (VT.isFloatingPoint() && "Can only convert between FP types"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Can only convert between FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 126, __extension__ __PRETTY_FUNCTION__))
;
127
128 // convert modifies in place, so make a copy.
129 APFloat Val2 = APFloat(Val);
130 bool losesInfo;
131 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
132 APFloat::rmNearestTiesToEven,
133 &losesInfo);
134 return !losesInfo;
135}
136
137//===----------------------------------------------------------------------===//
138// ISD Namespace
139//===----------------------------------------------------------------------===//
140
141bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
142 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
143 unsigned EltSize =
144 N->getValueType(0).getVectorElementType().getSizeInBits();
145 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
146 SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize);
147 return true;
148 }
149 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
150 SplatVal = Op0->getValueAPF().bitcastToAPInt().truncOrSelf(EltSize);
151 return true;
152 }
153 }
154
155 auto *BV = dyn_cast<BuildVectorSDNode>(N);
156 if (!BV)
157 return false;
158
159 APInt SplatUndef;
160 unsigned SplatBitSize;
161 bool HasUndefs;
162 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
163 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
164 EltSize) &&
165 EltSize == SplatBitSize;
166}
167
168// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
169// specializations of the more general isConstantSplatVector()?
170
171bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
172 // Look through a bit convert.
173 while (N->getOpcode() == ISD::BITCAST)
174 N = N->getOperand(0).getNode();
175
176 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
177 APInt SplatVal;
178 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue();
179 }
180
181 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
182
183 unsigned i = 0, e = N->getNumOperands();
184
185 // Skip over all of the undef values.
186 while (i != e && N->getOperand(i).isUndef())
187 ++i;
188
189 // Do not accept an all-undef vector.
190 if (i == e) return false;
191
192 // Do not accept build_vectors that aren't all constants or which have non-~0
193 // elements. We have to be a bit careful here, as the type of the constant
194 // may not be the same as the type of the vector elements due to type
195 // legalization (the elements are promoted to a legal type for the target and
196 // a vector of a type may be legal when the base element type is not).
197 // We only want to check enough bits to cover the vector elements, because
198 // we care if the resultant vector is all ones, not whether the individual
199 // constants are.
200 SDValue NotZero = N->getOperand(i);
201 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
202 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
203 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
204 return false;
205 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
206 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
207 return false;
208 } else
209 return false;
210
211 // Okay, we have at least one ~0 value, check to see if the rest match or are
212 // undefs. Even with the above element type twiddling, this should be OK, as
213 // the same type legalization should have applied to all the elements.
214 for (++i; i != e; ++i)
215 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
216 return false;
217 return true;
218}
219
220bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
221 // Look through a bit convert.
222 while (N->getOpcode() == ISD::BITCAST)
223 N = N->getOperand(0).getNode();
224
225 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
226 APInt SplatVal;
227 return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue();
228 }
229
230 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
231
232 bool IsAllUndef = true;
233 for (const SDValue &Op : N->op_values()) {
234 if (Op.isUndef())
235 continue;
236 IsAllUndef = false;
237 // Do not accept build_vectors that aren't all constants or which have non-0
238 // elements. We have to be a bit careful here, as the type of the constant
239 // may not be the same as the type of the vector elements due to type
240 // legalization (the elements are promoted to a legal type for the target
241 // and a vector of a type may be legal when the base element type is not).
242 // We only want to check enough bits to cover the vector elements, because
243 // we care if the resultant vector is all zeros, not whether the individual
244 // constants are.
245 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
246 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
247 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
248 return false;
249 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
250 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
251 return false;
252 } else
253 return false;
254 }
255
256 // Do not accept an all-undef vector.
257 if (IsAllUndef)
258 return false;
259 return true;
260}
261
262bool ISD::isBuildVectorAllOnes(const SDNode *N) {
263 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
264}
265
266bool ISD::isBuildVectorAllZeros(const SDNode *N) {
267 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
268}
269
270bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
271 if (N->getOpcode() != ISD::BUILD_VECTOR)
272 return false;
273
274 for (const SDValue &Op : N->op_values()) {
275 if (Op.isUndef())
276 continue;
277 if (!isa<ConstantSDNode>(Op))
278 return false;
279 }
280 return true;
281}
282
283bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
284 if (N->getOpcode() != ISD::BUILD_VECTOR)
285 return false;
286
287 for (const SDValue &Op : N->op_values()) {
288 if (Op.isUndef())
289 continue;
290 if (!isa<ConstantFPSDNode>(Op))
291 return false;
292 }
293 return true;
294}
295
296bool ISD::allOperandsUndef(const SDNode *N) {
297 // Return false if the node has no operands.
298 // This is "logically inconsistent" with the definition of "all" but
299 // is probably the desired behavior.
300 if (N->getNumOperands() == 0)
301 return false;
302 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
303}
304
305bool ISD::matchUnaryPredicate(SDValue Op,
306 std::function<bool(ConstantSDNode *)> Match,
307 bool AllowUndefs) {
308 // FIXME: Add support for scalar UNDEF cases?
309 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
310 return Match(Cst);
311
312 // FIXME: Add support for vector UNDEF cases?
313 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
314 ISD::SPLAT_VECTOR != Op.getOpcode())
315 return false;
316
317 EVT SVT = Op.getValueType().getScalarType();
318 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
319 if (AllowUndefs && Op.getOperand(i).isUndef()) {
320 if (!Match(nullptr))
321 return false;
322 continue;
323 }
324
325 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
326 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
327 return false;
328 }
329 return true;
330}
331
332bool ISD::matchBinaryPredicate(
333 SDValue LHS, SDValue RHS,
334 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
335 bool AllowUndefs, bool AllowTypeMismatch) {
336 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
337 return false;
338
339 // TODO: Add support for scalar UNDEF cases?
340 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
341 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
342 return Match(LHSCst, RHSCst);
343
344 // TODO: Add support for vector UNDEF cases?
345 if (LHS.getOpcode() != RHS.getOpcode() ||
346 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
347 LHS.getOpcode() != ISD::SPLAT_VECTOR))
348 return false;
349
350 EVT SVT = LHS.getValueType().getScalarType();
351 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
352 SDValue LHSOp = LHS.getOperand(i);
353 SDValue RHSOp = RHS.getOperand(i);
354 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
355 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
356 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
357 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
358 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
359 return false;
360 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
361 LHSOp.getValueType() != RHSOp.getValueType()))
362 return false;
363 if (!Match(LHSCst, RHSCst))
364 return false;
365 }
366 return true;
367}
368
369ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
370 switch (VecReduceOpcode) {
371 default:
372 llvm_unreachable("Expected VECREDUCE opcode")::llvm::llvm_unreachable_internal("Expected VECREDUCE opcode"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 372)
;
373 case ISD::VECREDUCE_FADD:
374 case ISD::VECREDUCE_SEQ_FADD:
375 return ISD::FADD;
376 case ISD::VECREDUCE_FMUL:
377 case ISD::VECREDUCE_SEQ_FMUL:
378 return ISD::FMUL;
379 case ISD::VECREDUCE_ADD:
380 return ISD::ADD;
381 case ISD::VECREDUCE_MUL:
382 return ISD::MUL;
383 case ISD::VECREDUCE_AND:
384 return ISD::AND;
385 case ISD::VECREDUCE_OR:
386 return ISD::OR;
387 case ISD::VECREDUCE_XOR:
388 return ISD::XOR;
389 case ISD::VECREDUCE_SMAX:
390 return ISD::SMAX;
391 case ISD::VECREDUCE_SMIN:
392 return ISD::SMIN;
393 case ISD::VECREDUCE_UMAX:
394 return ISD::UMAX;
395 case ISD::VECREDUCE_UMIN:
396 return ISD::UMIN;
397 case ISD::VECREDUCE_FMAX:
398 return ISD::FMAXNUM;
399 case ISD::VECREDUCE_FMIN:
400 return ISD::FMINNUM;
401 }
402}
403
404bool ISD::isVPOpcode(unsigned Opcode) {
405 switch (Opcode) {
406 default:
407 return false;
408#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
409 case ISD::SDOPC: \
410 return true;
411#include "llvm/IR/VPIntrinsics.def"
412 }
413}
414
415/// The operand position of the vector mask.
416Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
417 switch (Opcode) {
418 default:
419 return None;
420#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \
421 case ISD::SDOPC: \
422 return MASKPOS;
423#include "llvm/IR/VPIntrinsics.def"
424 }
425}
426
427/// The operand position of the explicit vector length parameter.
428Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
429 switch (Opcode) {
430 default:
431 return None;
432#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
433 case ISD::SDOPC: \
434 return EVLPOS;
435#include "llvm/IR/VPIntrinsics.def"
436 }
437}
438
439ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
440 switch (ExtType) {
441 case ISD::EXTLOAD:
442 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
443 case ISD::SEXTLOAD:
444 return ISD::SIGN_EXTEND;
445 case ISD::ZEXTLOAD:
446 return ISD::ZERO_EXTEND;
447 default:
448 break;
449 }
450
451 llvm_unreachable("Invalid LoadExtType")::llvm::llvm_unreachable_internal("Invalid LoadExtType", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 451)
;
452}
453
454ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
455 // To perform this operation, we just need to swap the L and G bits of the
456 // operation.
457 unsigned OldL = (Operation >> 2) & 1;
458 unsigned OldG = (Operation >> 1) & 1;
459 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
460 (OldL << 1) | // New G bit
461 (OldG << 2)); // New L bit.
462}
463
464static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
465 unsigned Operation = Op;
466 if (isIntegerLike)
467 Operation ^= 7; // Flip L, G, E bits, but not U.
468 else
469 Operation ^= 15; // Flip all of the condition bits.
470
471 if (Operation > ISD::SETTRUE2)
472 Operation &= ~8; // Don't let N and U bits get set.
473
474 return ISD::CondCode(Operation);
475}
476
477ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
478 return getSetCCInverseImpl(Op, Type.isInteger());
479}
480
481ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
482 bool isIntegerLike) {
483 return getSetCCInverseImpl(Op, isIntegerLike);
484}
485
486/// For an integer comparison, return 1 if the comparison is a signed operation
487/// and 2 if the result is an unsigned comparison. Return zero if the operation
488/// does not depend on the sign of the input (setne and seteq).
489static int isSignedOp(ISD::CondCode Opcode) {
490 switch (Opcode) {
491 default: llvm_unreachable("Illegal integer setcc operation!")::llvm::llvm_unreachable_internal("Illegal integer setcc operation!"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 491)
;
492 case ISD::SETEQ:
493 case ISD::SETNE: return 0;
494 case ISD::SETLT:
495 case ISD::SETLE:
496 case ISD::SETGT:
497 case ISD::SETGE: return 1;
498 case ISD::SETULT:
499 case ISD::SETULE:
500 case ISD::SETUGT:
501 case ISD::SETUGE: return 2;
502 }
503}
504
505ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
506 EVT Type) {
507 bool IsInteger = Type.isInteger();
508 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
509 // Cannot fold a signed integer setcc with an unsigned integer setcc.
510 return ISD::SETCC_INVALID;
511
512 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
513
514 // If the N and U bits get set, then the resultant comparison DOES suddenly
515 // care about orderedness, and it is true when ordered.
516 if (Op > ISD::SETTRUE2)
517 Op &= ~16; // Clear the U bit if the N bit is set.
518
519 // Canonicalize illegal integer setcc's.
520 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
521 Op = ISD::SETNE;
522
523 return ISD::CondCode(Op);
524}
525
526ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
527 EVT Type) {
528 bool IsInteger = Type.isInteger();
529 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
530 // Cannot fold a signed setcc with an unsigned setcc.
531 return ISD::SETCC_INVALID;
532
533 // Combine all of the condition bits.
534 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
535
536 // Canonicalize illegal integer setcc's.
537 if (IsInteger) {
538 switch (Result) {
539 default: break;
540 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
541 case ISD::SETOEQ: // SETEQ & SETU[LG]E
542 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
543 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
544 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
545 }
546 }
547
548 return Result;
549}
550
551//===----------------------------------------------------------------------===//
552// SDNode Profile Support
553//===----------------------------------------------------------------------===//
554
555/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
556static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
557 ID.AddInteger(OpC);
558}
559
560/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
561/// solely with their pointer.
562static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
563 ID.AddPointer(VTList.VTs);
564}
565
566/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
567static void AddNodeIDOperands(FoldingSetNodeID &ID,
568 ArrayRef<SDValue> Ops) {
569 for (auto& Op : Ops) {
570 ID.AddPointer(Op.getNode());
571 ID.AddInteger(Op.getResNo());
572 }
573}
574
575/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
576static void AddNodeIDOperands(FoldingSetNodeID &ID,
577 ArrayRef<SDUse> Ops) {
578 for (auto& Op : Ops) {
579 ID.AddPointer(Op.getNode());
580 ID.AddInteger(Op.getResNo());
581 }
582}
583
584static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
585 SDVTList VTList, ArrayRef<SDValue> OpList) {
586 AddNodeIDOpcode(ID, OpC);
587 AddNodeIDValueTypes(ID, VTList);
588 AddNodeIDOperands(ID, OpList);
589}
590
591/// If this is an SDNode with special info, add this info to the NodeID data.
592static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
593 switch (N->getOpcode()) {
594 case ISD::TargetExternalSymbol:
595 case ISD::ExternalSymbol:
596 case ISD::MCSymbol:
597 llvm_unreachable("Should only be used on nodes with operands")::llvm::llvm_unreachable_internal("Should only be used on nodes with operands"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 597)
;
598 default: break; // Normal nodes don't need extra info.
599 case ISD::TargetConstant:
600 case ISD::Constant: {
601 const ConstantSDNode *C = cast<ConstantSDNode>(N);
602 ID.AddPointer(C->getConstantIntValue());
603 ID.AddBoolean(C->isOpaque());
604 break;
605 }
606 case ISD::TargetConstantFP:
607 case ISD::ConstantFP:
608 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
609 break;
610 case ISD::TargetGlobalAddress:
611 case ISD::GlobalAddress:
612 case ISD::TargetGlobalTLSAddress:
613 case ISD::GlobalTLSAddress: {
614 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
615 ID.AddPointer(GA->getGlobal());
616 ID.AddInteger(GA->getOffset());
617 ID.AddInteger(GA->getTargetFlags());
618 break;
619 }
620 case ISD::BasicBlock:
621 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
622 break;
623 case ISD::Register:
624 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
625 break;
626 case ISD::RegisterMask:
627 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
628 break;
629 case ISD::SRCVALUE:
630 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
631 break;
632 case ISD::FrameIndex:
633 case ISD::TargetFrameIndex:
634 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
635 break;
636 case ISD::LIFETIME_START:
637 case ISD::LIFETIME_END:
638 if (cast<LifetimeSDNode>(N)->hasOffset()) {
639 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
640 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
641 }
642 break;
643 case ISD::PSEUDO_PROBE:
644 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
645 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
646 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
647 break;
648 case ISD::JumpTable:
649 case ISD::TargetJumpTable:
650 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
651 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
652 break;
653 case ISD::ConstantPool:
654 case ISD::TargetConstantPool: {
655 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
656 ID.AddInteger(CP->getAlign().value());
657 ID.AddInteger(CP->getOffset());
658 if (CP->isMachineConstantPoolEntry())
659 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
660 else
661 ID.AddPointer(CP->getConstVal());
662 ID.AddInteger(CP->getTargetFlags());
663 break;
664 }
665 case ISD::TargetIndex: {
666 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
667 ID.AddInteger(TI->getIndex());
668 ID.AddInteger(TI->getOffset());
669 ID.AddInteger(TI->getTargetFlags());
670 break;
671 }
672 case ISD::LOAD: {
673 const LoadSDNode *LD = cast<LoadSDNode>(N);
674 ID.AddInteger(LD->getMemoryVT().getRawBits());
675 ID.AddInteger(LD->getRawSubclassData());
676 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
677 break;
678 }
679 case ISD::STORE: {
680 const StoreSDNode *ST = cast<StoreSDNode>(N);
681 ID.AddInteger(ST->getMemoryVT().getRawBits());
682 ID.AddInteger(ST->getRawSubclassData());
683 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
684 break;
685 }
686 case ISD::VP_LOAD: {
687 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
688 ID.AddInteger(ELD->getMemoryVT().getRawBits());
689 ID.AddInteger(ELD->getRawSubclassData());
690 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
691 break;
692 }
693 case ISD::VP_STORE: {
694 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
695 ID.AddInteger(EST->getMemoryVT().getRawBits());
696 ID.AddInteger(EST->getRawSubclassData());
697 ID.AddInteger(EST->getPointerInfo().getAddrSpace());
698 break;
699 }
700 case ISD::VP_GATHER: {
701 const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);
702 ID.AddInteger(EG->getMemoryVT().getRawBits());
703 ID.AddInteger(EG->getRawSubclassData());
704 ID.AddInteger(EG->getPointerInfo().getAddrSpace());
705 break;
706 }
707 case ISD::VP_SCATTER: {
708 const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);
709 ID.AddInteger(ES->getMemoryVT().getRawBits());
710 ID.AddInteger(ES->getRawSubclassData());
711 ID.AddInteger(ES->getPointerInfo().getAddrSpace());
712 break;
713 }
714 case ISD::MLOAD: {
715 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
716 ID.AddInteger(MLD->getMemoryVT().getRawBits());
717 ID.AddInteger(MLD->getRawSubclassData());
718 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
719 break;
720 }
721 case ISD::MSTORE: {
722 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
723 ID.AddInteger(MST->getMemoryVT().getRawBits());
724 ID.AddInteger(MST->getRawSubclassData());
725 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
726 break;
727 }
728 case ISD::MGATHER: {
729 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
730 ID.AddInteger(MG->getMemoryVT().getRawBits());
731 ID.AddInteger(MG->getRawSubclassData());
732 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
733 break;
734 }
735 case ISD::MSCATTER: {
736 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
737 ID.AddInteger(MS->getMemoryVT().getRawBits());
738 ID.AddInteger(MS->getRawSubclassData());
739 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
740 break;
741 }
742 case ISD::ATOMIC_CMP_SWAP:
743 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
744 case ISD::ATOMIC_SWAP:
745 case ISD::ATOMIC_LOAD_ADD:
746 case ISD::ATOMIC_LOAD_SUB:
747 case ISD::ATOMIC_LOAD_AND:
748 case ISD::ATOMIC_LOAD_CLR:
749 case ISD::ATOMIC_LOAD_OR:
750 case ISD::ATOMIC_LOAD_XOR:
751 case ISD::ATOMIC_LOAD_NAND:
752 case ISD::ATOMIC_LOAD_MIN:
753 case ISD::ATOMIC_LOAD_MAX:
754 case ISD::ATOMIC_LOAD_UMIN:
755 case ISD::ATOMIC_LOAD_UMAX:
756 case ISD::ATOMIC_LOAD:
757 case ISD::ATOMIC_STORE: {
758 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
759 ID.AddInteger(AT->getMemoryVT().getRawBits());
760 ID.AddInteger(AT->getRawSubclassData());
761 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
762 break;
763 }
764 case ISD::PREFETCH: {
765 const MemSDNode *PF = cast<MemSDNode>(N);
766 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
767 break;
768 }
769 case ISD::VECTOR_SHUFFLE: {
770 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
771 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
772 i != e; ++i)
773 ID.AddInteger(SVN->getMaskElt(i));
774 break;
775 }
776 case ISD::TargetBlockAddress:
777 case ISD::BlockAddress: {
778 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
779 ID.AddPointer(BA->getBlockAddress());
780 ID.AddInteger(BA->getOffset());
781 ID.AddInteger(BA->getTargetFlags());
782 break;
783 }
784 } // end switch (N->getOpcode())
785
786 // Target specific memory nodes could also have address spaces to check.
787 if (N->isTargetMemoryOpcode())
788 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
789}
790
791/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
792/// data.
793static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
794 AddNodeIDOpcode(ID, N->getOpcode());
795 // Add the return value info.
796 AddNodeIDValueTypes(ID, N->getVTList());
797 // Add the operand info.
798 AddNodeIDOperands(ID, N->ops());
799
800 // Handle SDNode leafs with special info.
801 AddNodeIDCustom(ID, N);
802}
803
804//===----------------------------------------------------------------------===//
805// SelectionDAG Class
806//===----------------------------------------------------------------------===//
807
808/// doNotCSE - Return true if CSE should not be performed for this node.
809static bool doNotCSE(SDNode *N) {
810 if (N->getValueType(0) == MVT::Glue)
811 return true; // Never CSE anything that produces a flag.
812
813 switch (N->getOpcode()) {
814 default: break;
815 case ISD::HANDLENODE:
816 case ISD::EH_LABEL:
817 return true; // Never CSE these nodes.
818 }
819
820 // Check that remaining values produced are not flags.
821 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
822 if (N->getValueType(i) == MVT::Glue)
823 return true; // Never CSE anything that produces a flag.
824
825 return false;
826}
827
828/// RemoveDeadNodes - This method deletes all unreachable nodes in the
829/// SelectionDAG.
830void SelectionDAG::RemoveDeadNodes() {
831 // Create a dummy node (which is not added to allnodes), that adds a reference
832 // to the root node, preventing it from being deleted.
833 HandleSDNode Dummy(getRoot());
834
835 SmallVector<SDNode*, 128> DeadNodes;
836
837 // Add all obviously-dead nodes to the DeadNodes worklist.
838 for (SDNode &Node : allnodes())
839 if (Node.use_empty())
840 DeadNodes.push_back(&Node);
841
842 RemoveDeadNodes(DeadNodes);
843
844 // If the root changed (e.g. it was a dead load, update the root).
845 setRoot(Dummy.getValue());
846}
847
848/// RemoveDeadNodes - This method deletes the unreachable nodes in the
849/// given list, and any nodes that become unreachable as a result.
850void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
851
852 // Process the worklist, deleting the nodes and adding their uses to the
853 // worklist.
854 while (!DeadNodes.empty()) {
855 SDNode *N = DeadNodes.pop_back_val();
856 // Skip to next node if we've already managed to delete the node. This could
857 // happen if replacing a node causes a node previously added to the node to
858 // be deleted.
859 if (N->getOpcode() == ISD::DELETED_NODE)
860 continue;
861
862 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
863 DUL->NodeDeleted(N, nullptr);
864
865 // Take the node out of the appropriate CSE map.
866 RemoveNodeFromCSEMaps(N);
867
868 // Next, brutally remove the operand list. This is safe to do, as there are
869 // no cycles in the graph.
870 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
871 SDUse &Use = *I++;
872 SDNode *Operand = Use.getNode();
873 Use.set(SDValue());
874
875 // Now that we removed this operand, see if there are no uses of it left.
876 if (Operand->use_empty())
877 DeadNodes.push_back(Operand);
878 }
879
880 DeallocateNode(N);
881 }
882}
883
884void SelectionDAG::RemoveDeadNode(SDNode *N){
885 SmallVector<SDNode*, 16> DeadNodes(1, N);
886
887 // Create a dummy node that adds a reference to the root node, preventing
888 // it from being deleted. (This matters if the root is an operand of the
889 // dead node.)
890 HandleSDNode Dummy(getRoot());
891
892 RemoveDeadNodes(DeadNodes);
893}
894
895void SelectionDAG::DeleteNode(SDNode *N) {
896 // First take this out of the appropriate CSE map.
897 RemoveNodeFromCSEMaps(N);
898
899 // Finally, remove uses due to operands of this node, remove from the
900 // AllNodes list, and delete the node.
901 DeleteNodeNotInCSEMaps(N);
902}
903
904void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
905 assert(N->getIterator() != AllNodes.begin() &&(static_cast <bool> (N->getIterator() != AllNodes.begin
() && "Cannot delete the entry node!") ? void (0) : __assert_fail
("N->getIterator() != AllNodes.begin() && \"Cannot delete the entry node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 906, __extension__ __PRETTY_FUNCTION__))
906 "Cannot delete the entry node!")(static_cast <bool> (N->getIterator() != AllNodes.begin
() && "Cannot delete the entry node!") ? void (0) : __assert_fail
("N->getIterator() != AllNodes.begin() && \"Cannot delete the entry node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 906, __extension__ __PRETTY_FUNCTION__))
;
907 assert(N->use_empty() && "Cannot delete a node that is not dead!")(static_cast <bool> (N->use_empty() && "Cannot delete a node that is not dead!"
) ? void (0) : __assert_fail ("N->use_empty() && \"Cannot delete a node that is not dead!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))
;
908
909 // Drop all of the operands and decrement used node's use counts.
910 N->DropOperands();
911
912 DeallocateNode(N);
913}
914
915void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
916 assert(!(V->isVariadic() && isParameter))(static_cast <bool> (!(V->isVariadic() && isParameter
)) ? void (0) : __assert_fail ("!(V->isVariadic() && isParameter)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 916, __extension__ __PRETTY_FUNCTION__))
;
917 if (isParameter)
918 ByvalParmDbgValues.push_back(V);
919 else
920 DbgValues.push_back(V);
921 for (const SDNode *Node : V->getSDNodes())
922 if (Node)
923 DbgValMap[Node].push_back(V);
924}
925
926void SDDbgInfo::erase(const SDNode *Node) {
927 DbgValMapType::iterator I = DbgValMap.find(Node);
928 if (I == DbgValMap.end())
929 return;
930 for (auto &Val: I->second)
931 Val->setIsInvalidated();
932 DbgValMap.erase(I);
933}
934
935void SelectionDAG::DeallocateNode(SDNode *N) {
936 // If we have operands, deallocate them.
937 removeOperands(N);
938
939 NodeAllocator.Deallocate(AllNodes.remove(N));
940
941 // Set the opcode to DELETED_NODE to help catch bugs when node
942 // memory is reallocated.
943 // FIXME: There are places in SDag that have grown a dependency on the opcode
944 // value in the released node.
945 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
946 N->NodeType = ISD::DELETED_NODE;
947
948 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
949 // them and forget about that node.
950 DbgInfo->erase(N);
951}
952
953#ifndef NDEBUG
954/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
955static void VerifySDNode(SDNode *N) {
956 switch (N->getOpcode()) {
957 default:
958 break;
959 case ISD::BUILD_PAIR: {
960 EVT VT = N->getValueType(0);
961 assert(N->getNumValues() == 1 && "Too many results!")(static_cast <bool> (N->getNumValues() == 1 &&
"Too many results!") ? void (0) : __assert_fail ("N->getNumValues() == 1 && \"Too many results!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 961, __extension__ __PRETTY_FUNCTION__))
;
962 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&(static_cast <bool> (!VT.isVector() && (VT.isInteger
() || VT.isFloatingPoint()) && "Wrong return type!") ?
void (0) : __assert_fail ("!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 963, __extension__ __PRETTY_FUNCTION__))
963 "Wrong return type!")(static_cast <bool> (!VT.isVector() && (VT.isInteger
() || VT.isFloatingPoint()) && "Wrong return type!") ?
void (0) : __assert_fail ("!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 963, __extension__ __PRETTY_FUNCTION__))
;
964 assert(N->getNumOperands() == 2 && "Wrong number of operands!")(static_cast <bool> (N->getNumOperands() == 2 &&
"Wrong number of operands!") ? void (0) : __assert_fail ("N->getNumOperands() == 2 && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 964, __extension__ __PRETTY_FUNCTION__))
;
965 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&(static_cast <bool> (N->getOperand(0).getValueType()
== N->getOperand(1).getValueType() && "Mismatched operand types!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && \"Mismatched operand types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 966, __extension__ __PRETTY_FUNCTION__))
966 "Mismatched operand types!")(static_cast <bool> (N->getOperand(0).getValueType()
== N->getOperand(1).getValueType() && "Mismatched operand types!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && \"Mismatched operand types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 966, __extension__ __PRETTY_FUNCTION__))
;
967 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&(static_cast <bool> (N->getOperand(0).getValueType()
.isInteger() == VT.isInteger() && "Wrong operand type!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType().isInteger() == VT.isInteger() && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 968, __extension__ __PRETTY_FUNCTION__))
968 "Wrong operand type!")(static_cast <bool> (N->getOperand(0).getValueType()
.isInteger() == VT.isInteger() && "Wrong operand type!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType().isInteger() == VT.isInteger() && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 968, __extension__ __PRETTY_FUNCTION__))
;
969 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&(static_cast <bool> (VT.getSizeInBits() == 2 * N->getOperand
(0).getValueSizeInBits() && "Wrong return type size")
? void (0) : __assert_fail ("VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && \"Wrong return type size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 970, __extension__ __PRETTY_FUNCTION__))
970 "Wrong return type size")(static_cast <bool> (VT.getSizeInBits() == 2 * N->getOperand
(0).getValueSizeInBits() && "Wrong return type size")
? void (0) : __assert_fail ("VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && \"Wrong return type size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 970, __extension__ __PRETTY_FUNCTION__))
;
971 break;
972 }
973 case ISD::BUILD_VECTOR: {
974 assert(N->getNumValues() == 1 && "Too many results!")(static_cast <bool> (N->getNumValues() == 1 &&
"Too many results!") ? void (0) : __assert_fail ("N->getNumValues() == 1 && \"Too many results!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 974, __extension__ __PRETTY_FUNCTION__))
;
975 assert(N->getValueType(0).isVector() && "Wrong return type!")(static_cast <bool> (N->getValueType(0).isVector() &&
"Wrong return type!") ? void (0) : __assert_fail ("N->getValueType(0).isVector() && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 975, __extension__ __PRETTY_FUNCTION__))
;
976 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&(static_cast <bool> (N->getNumOperands() == N->getValueType
(0).getVectorNumElements() && "Wrong number of operands!"
) ? void (0) : __assert_fail ("N->getNumOperands() == N->getValueType(0).getVectorNumElements() && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 977, __extension__ __PRETTY_FUNCTION__))
977 "Wrong number of operands!")(static_cast <bool> (N->getNumOperands() == N->getValueType
(0).getVectorNumElements() && "Wrong number of operands!"
) ? void (0) : __assert_fail ("N->getNumOperands() == N->getValueType(0).getVectorNumElements() && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 977, __extension__ __PRETTY_FUNCTION__))
;
978 EVT EltVT = N->getValueType(0).getVectorElementType();
979 for (const SDUse &Op : N->ops()) {
980 assert((Op.getValueType() == EltVT ||(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 983, __extension__ __PRETTY_FUNCTION__))
981 (EltVT.isInteger() && Op.getValueType().isInteger() &&(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 983, __extension__ __PRETTY_FUNCTION__))
982 EltVT.bitsLE(Op.getValueType()))) &&(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 983, __extension__ __PRETTY_FUNCTION__))
983 "Wrong operand type!")(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 983, __extension__ __PRETTY_FUNCTION__))
;
984 assert(Op.getValueType() == N->getOperand(0).getValueType() &&(static_cast <bool> (Op.getValueType() == N->getOperand
(0).getValueType() && "Operands must all have the same type"
) ? void (0) : __assert_fail ("Op.getValueType() == N->getOperand(0).getValueType() && \"Operands must all have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 985, __extension__ __PRETTY_FUNCTION__))
985 "Operands must all have the same type")(static_cast <bool> (Op.getValueType() == N->getOperand
(0).getValueType() && "Operands must all have the same type"
) ? void (0) : __assert_fail ("Op.getValueType() == N->getOperand(0).getValueType() && \"Operands must all have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 985, __extension__ __PRETTY_FUNCTION__))
;
986 }
987 break;
988 }
989 }
990}
991#endif // NDEBUG
992
993/// Insert a newly allocated node into the DAG.
994///
995/// Handles insertion into the all nodes list and CSE map, as well as
996/// verification and other common operations when a new node is allocated.
997void SelectionDAG::InsertNode(SDNode *N) {
998 AllNodes.push_back(N);
999#ifndef NDEBUG
1000 N->PersistentId = NextPersistentId++;
1001 VerifySDNode(N);
1002#endif
1003 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1004 DUL->NodeInserted(N);
1005}
1006
1007/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1008/// correspond to it. This is useful when we're about to delete or repurpose
1009/// the node. We don't want future request for structurally identical nodes
1010/// to return N anymore.
1011bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1012 bool Erased = false;
1013 switch (N->getOpcode()) {
1014 case ISD::HANDLENODE: return false; // noop.
1015 case ISD::CONDCODE:
1016 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&(static_cast <bool> (CondCodeNodes[cast<CondCodeSDNode
>(N)->get()] && "Cond code doesn't exist!") ? void
(0) : __assert_fail ("CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && \"Cond code doesn't exist!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1017, __extension__ __PRETTY_FUNCTION__))
1017 "Cond code doesn't exist!")(static_cast <bool> (CondCodeNodes[cast<CondCodeSDNode
>(N)->get()] && "Cond code doesn't exist!") ? void
(0) : __assert_fail ("CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && \"Cond code doesn't exist!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1017, __extension__ __PRETTY_FUNCTION__))
;
1018 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1019 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1020 break;
1021 case ISD::ExternalSymbol:
1022 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1023 break;
1024 case ISD::TargetExternalSymbol: {
1025 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1026 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1027 ESN->getSymbol(), ESN->getTargetFlags()));
1028 break;
1029 }
1030 case ISD::MCSymbol: {
1031 auto *MCSN = cast<MCSymbolSDNode>(N);
1032 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1033 break;
1034 }
1035 case ISD::VALUETYPE: {
1036 EVT VT = cast<VTSDNode>(N)->getVT();
1037 if (VT.isExtended()) {
1038 Erased = ExtendedValueTypeNodes.erase(VT);
1039 } else {
1040 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1041 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1042 }
1043 break;
1044 }
1045 default:
1046 // Remove it from the CSE Map.
1047 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "DELETED_NODE in CSEMap!") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"DELETED_NODE in CSEMap!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1047, __extension__ __PRETTY_FUNCTION__))
;
1048 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!")(static_cast <bool> (N->getOpcode() != ISD::EntryToken
&& "EntryToken in CSEMap!") ? void (0) : __assert_fail
("N->getOpcode() != ISD::EntryToken && \"EntryToken in CSEMap!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1048, __extension__ __PRETTY_FUNCTION__))
;
1049 Erased = CSEMap.RemoveNode(N);
1050 break;
1051 }
1052#ifndef NDEBUG
1053 // Verify that the node was actually in one of the CSE maps, unless it has a
1054 // flag result (which cannot be CSE'd) or is one of the special cases that are
1055 // not subject to CSE.
1056 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1057 !N->isMachineOpcode() && !doNotCSE(N)) {
1058 N->dump(this);
1059 dbgs() << "\n";
1060 llvm_unreachable("Node is not in map!")::llvm::llvm_unreachable_internal("Node is not in map!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1060)
;
1061 }
1062#endif
1063 return Erased;
1064}
1065
1066/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1067/// maps and modified in place. Add it back to the CSE maps, unless an identical
1068/// node already exists, in which case transfer all its users to the existing
1069/// node. This transfer can potentially trigger recursive merging.
1070void
1071SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1072 // For node types that aren't CSE'd, just act as if no identical node
1073 // already exists.
1074 if (!doNotCSE(N)) {
1075 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1076 if (Existing != N) {
1077 // If there was already an existing matching node, use ReplaceAllUsesWith
1078 // to replace the dead one with the existing one. This can cause
1079 // recursive merging of other unrelated nodes down the line.
1080 ReplaceAllUsesWith(N, Existing);
1081
1082 // N is now dead. Inform the listeners and delete it.
1083 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1084 DUL->NodeDeleted(N, Existing);
1085 DeleteNodeNotInCSEMaps(N);
1086 return;
1087 }
1088 }
1089
1090 // If the node doesn't already exist, we updated it. Inform listeners.
1091 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1092 DUL->NodeUpdated(N);
1093}
1094
1095/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1096/// were replaced with those specified. If this node is never memoized,
1097/// return null, otherwise return a pointer to the slot it would take. If a
1098/// node already exists with these operands, the slot will be non-null.
1099SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1100 void *&InsertPos) {
1101 if (doNotCSE(N))
1102 return nullptr;
1103
1104 SDValue Ops[] = { Op };
1105 FoldingSetNodeID ID;
1106 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1107 AddNodeIDCustom(ID, N);
1108 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1109 if (Node)
1110 Node->intersectFlagsWith(N->getFlags());
1111 return Node;
1112}
1113
1114/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1115/// were replaced with those specified. If this node is never memoized,
1116/// return null, otherwise return a pointer to the slot it would take. If a
1117/// node already exists with these operands, the slot will be non-null.
1118SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1119 SDValue Op1, SDValue Op2,
1120 void *&InsertPos) {
1121 if (doNotCSE(N))
1122 return nullptr;
1123
1124 SDValue Ops[] = { Op1, Op2 };
1125 FoldingSetNodeID ID;
1126 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1127 AddNodeIDCustom(ID, N);
1128 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1129 if (Node)
1130 Node->intersectFlagsWith(N->getFlags());
1131 return Node;
1132}
1133
1134/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1135/// were replaced with those specified. If this node is never memoized,
1136/// return null, otherwise return a pointer to the slot it would take. If a
1137/// node already exists with these operands, the slot will be non-null.
1138SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1139 void *&InsertPos) {
1140 if (doNotCSE(N))
1141 return nullptr;
1142
1143 FoldingSetNodeID ID;
1144 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1145 AddNodeIDCustom(ID, N);
1146 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1147 if (Node)
1148 Node->intersectFlagsWith(N->getFlags());
1149 return Node;
1150}
1151
1152Align SelectionDAG::getEVTAlign(EVT VT) const {
1153 Type *Ty = VT == MVT::iPTR ?
1154 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
1155 VT.getTypeForEVT(*getContext());
1156
1157 return getDataLayout().getABITypeAlign(Ty);
1158}
1159
1160// EntryNode could meaningfully have debug info if we can find it...
1161SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
1162 : TM(tm), OptLevel(OL),
1163 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1164 Root(getEntryNode()) {
1165 InsertNode(&EntryNode);
1166 DbgInfo = new SDDbgInfo();
1167}
1168
1169void SelectionDAG::init(MachineFunction &NewMF,
1170 OptimizationRemarkEmitter &NewORE,
1171 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1172 LegacyDivergenceAnalysis * Divergence,
1173 ProfileSummaryInfo *PSIin,
1174 BlockFrequencyInfo *BFIin) {
1175 MF = &NewMF;
1176 SDAGISelPass = PassPtr;
1177 ORE = &NewORE;
1178 TLI = getSubtarget().getTargetLowering();
1179 TSI = getSubtarget().getSelectionDAGInfo();
1180 LibInfo = LibraryInfo;
1181 Context = &MF->getFunction().getContext();
1182 DA = Divergence;
1183 PSI = PSIin;
1184 BFI = BFIin;
1185}
1186
1187SelectionDAG::~SelectionDAG() {
1188 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners")(static_cast <bool> (!UpdateListeners && "Dangling registered DAGUpdateListeners"
) ? void (0) : __assert_fail ("!UpdateListeners && \"Dangling registered DAGUpdateListeners\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1188, __extension__ __PRETTY_FUNCTION__))
;
1189 allnodes_clear();
1190 OperandRecycler.clear(OperandAllocator);
1191 delete DbgInfo;
1192}
1193
1194bool SelectionDAG::shouldOptForSize() const {
1195 return MF->getFunction().hasOptSize() ||
1196 llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1197}
1198
1199void SelectionDAG::allnodes_clear() {
1200 assert(&*AllNodes.begin() == &EntryNode)(static_cast <bool> (&*AllNodes.begin() == &EntryNode
) ? void (0) : __assert_fail ("&*AllNodes.begin() == &EntryNode"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1200, __extension__ __PRETTY_FUNCTION__))
;
1201 AllNodes.remove(AllNodes.begin());
1202 while (!AllNodes.empty())
1203 DeallocateNode(&AllNodes.front());
1204#ifndef NDEBUG
1205 NextPersistentId = 0;
1206#endif
1207}
1208
1209SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1210 void *&InsertPos) {
1211 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1212 if (N) {
1213 switch (N->getOpcode()) {
1214 default: break;
1215 case ISD::Constant:
1216 case ISD::ConstantFP:
1217 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "::llvm::llvm_unreachable_internal("Querying for Constant and ConstantFP nodes requires "
"debug location. Use another overload.", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1218)
1218 "debug location. Use another overload.")::llvm::llvm_unreachable_internal("Querying for Constant and ConstantFP nodes requires "
"debug location. Use another overload.", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1218)
;
1219 }
1220 }
1221 return N;
1222}
1223
1224SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1225 const SDLoc &DL, void *&InsertPos) {
1226 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1227 if (N) {
1228 switch (N->getOpcode()) {
1229 case ISD::Constant:
1230 case ISD::ConstantFP:
1231 // Erase debug location from the node if the node is used at several
1232 // different places. Do not propagate one location to all uses as it
1233 // will cause a worse single stepping debugging experience.
1234 if (N->getDebugLoc() != DL.getDebugLoc())
1235 N->setDebugLoc(DebugLoc());
1236 break;
1237 default:
1238 // When the node's point of use is located earlier in the instruction
1239 // sequence than its prior point of use, update its debug info to the
1240 // earlier location.
1241 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1242 N->setDebugLoc(DL.getDebugLoc());
1243 break;
1244 }
1245 }
1246 return N;
1247}
1248
1249void SelectionDAG::clear() {
1250 allnodes_clear();
1251 OperandRecycler.clear(OperandAllocator);
1252 OperandAllocator.Reset();
1253 CSEMap.clear();
1254
1255 ExtendedValueTypeNodes.clear();
1256 ExternalSymbols.clear();
1257 TargetExternalSymbols.clear();
1258 MCSymbols.clear();
1259 SDCallSiteDbgInfo.clear();
1260 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1261 static_cast<CondCodeSDNode*>(nullptr));
1262 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1263 static_cast<SDNode*>(nullptr));
1264
1265 EntryNode.UseList = nullptr;
1266 InsertNode(&EntryNode);
1267 Root = getEntryNode();
1268 DbgInfo->clear();
1269}
1270
1271SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1272 return VT.bitsGT(Op.getValueType())
1273 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1274 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1275}
1276
1277std::pair<SDValue, SDValue>
1278SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1279 const SDLoc &DL, EVT VT) {
1280 assert(!VT.bitsEq(Op.getValueType()) &&(static_cast <bool> (!VT.bitsEq(Op.getValueType()) &&
"Strict no-op FP extend/round not allowed.") ? void (0) : __assert_fail
("!VT.bitsEq(Op.getValueType()) && \"Strict no-op FP extend/round not allowed.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1281, __extension__ __PRETTY_FUNCTION__))
1281 "Strict no-op FP extend/round not allowed.")(static_cast <bool> (!VT.bitsEq(Op.getValueType()) &&
"Strict no-op FP extend/round not allowed.") ? void (0) : __assert_fail
("!VT.bitsEq(Op.getValueType()) && \"Strict no-op FP extend/round not allowed.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1281, __extension__ __PRETTY_FUNCTION__))
;
1282 SDValue Res =
1283 VT.bitsGT(Op.getValueType())
1284 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1285 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1286 {Chain, Op, getIntPtrConstant(0, DL)});
1287
1288 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1289}
1290
1291SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1292 return VT.bitsGT(Op.getValueType()) ?
1293 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1294 getNode(ISD::TRUNCATE, DL, VT, Op);
1295}
1296
1297SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1298 return VT.bitsGT(Op.getValueType()) ?
1299 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1300 getNode(ISD::TRUNCATE, DL, VT, Op);
1301}
1302
1303SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1304 return VT.bitsGT(Op.getValueType()) ?
1305 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1306 getNode(ISD::TRUNCATE, DL, VT, Op);
1307}
1308
1309SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1310 EVT OpVT) {
1311 if (VT.bitsLE(Op.getValueType()))
1312 return getNode(ISD::TRUNCATE, SL, VT, Op);
1313
1314 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1315 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1316}
1317
1318SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1319 EVT OpVT = Op.getValueType();
1320 assert(VT.isInteger() && OpVT.isInteger() &&(static_cast <bool> (VT.isInteger() && OpVT.isInteger
() && "Cannot getZeroExtendInReg FP types") ? void (0
) : __assert_fail ("VT.isInteger() && OpVT.isInteger() && \"Cannot getZeroExtendInReg FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1321, __extension__ __PRETTY_FUNCTION__))
1321 "Cannot getZeroExtendInReg FP types")(static_cast <bool> (VT.isInteger() && OpVT.isInteger
() && "Cannot getZeroExtendInReg FP types") ? void (0
) : __assert_fail ("VT.isInteger() && OpVT.isInteger() && \"Cannot getZeroExtendInReg FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1321, __extension__ __PRETTY_FUNCTION__))
;
1322 assert(VT.isVector() == OpVT.isVector() &&(static_cast <bool> (VT.isVector() == OpVT.isVector() &&
"getZeroExtendInReg type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == OpVT.isVector() && \"getZeroExtendInReg type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1324, __extension__ __PRETTY_FUNCTION__))
1323 "getZeroExtendInReg type should be vector iff the operand "(static_cast <bool> (VT.isVector() == OpVT.isVector() &&
"getZeroExtendInReg type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == OpVT.isVector() && \"getZeroExtendInReg type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1324, __extension__ __PRETTY_FUNCTION__))
1324 "type is vector!")(static_cast <bool> (VT.isVector() == OpVT.isVector() &&
"getZeroExtendInReg type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == OpVT.isVector() && \"getZeroExtendInReg type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1324, __extension__ __PRETTY_FUNCTION__))
;
1325 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == OpVT.getVectorElementCount()) && "Vector element counts must match in getZeroExtendInReg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == OpVT.getVectorElementCount()) && \"Vector element counts must match in getZeroExtendInReg\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1327, __extension__ __PRETTY_FUNCTION__))
1326 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == OpVT.getVectorElementCount()) && "Vector element counts must match in getZeroExtendInReg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == OpVT.getVectorElementCount()) && \"Vector element counts must match in getZeroExtendInReg\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1327, __extension__ __PRETTY_FUNCTION__))
1327 "Vector element counts must match in getZeroExtendInReg")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == OpVT.getVectorElementCount()) && "Vector element counts must match in getZeroExtendInReg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == OpVT.getVectorElementCount()) && \"Vector element counts must match in getZeroExtendInReg\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1327, __extension__ __PRETTY_FUNCTION__))
;
1328 assert(VT.bitsLE(OpVT) && "Not extending!")(static_cast <bool> (VT.bitsLE(OpVT) && "Not extending!"
) ? void (0) : __assert_fail ("VT.bitsLE(OpVT) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1328, __extension__ __PRETTY_FUNCTION__))
;
1329 if (OpVT == VT)
1330 return Op;
1331 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),
1332 VT.getScalarSizeInBits());
1333 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1334}
1335
1336SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1337 // Only unsigned pointer semantics are supported right now. In the future this
1338 // might delegate to TLI to check pointer signedness.
1339 return getZExtOrTrunc(Op, DL, VT);
1340}
1341
1342SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1343 // Only unsigned pointer semantics are supported right now. In the future this
1344 // might delegate to TLI to check pointer signedness.
1345 return getZeroExtendInReg(Op, DL, VT);
1346}
1347
1348/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1349SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1350 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1351}
1352
1353SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1354 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1355 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1356}
1357
1358SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1359 EVT OpVT) {
1360 if (!V)
1361 return getConstant(0, DL, VT);
1362
1363 switch (TLI->getBooleanContents(OpVT)) {
1364 case TargetLowering::ZeroOrOneBooleanContent:
1365 case TargetLowering::UndefinedBooleanContent:
1366 return getConstant(1, DL, VT);
1367 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1368 return getAllOnesConstant(DL, VT);
1369 }
1370 llvm_unreachable("Unexpected boolean content enum!")::llvm::llvm_unreachable_internal("Unexpected boolean content enum!"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1370)
;
1371}
1372
1373SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1374 bool isT, bool isO) {
1375 EVT EltVT = VT.getScalarType();
1376 assert((EltVT.getSizeInBits() >= 64 ||(static_cast <bool> ((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 <
2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? void (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1378, __extension__ __PRETTY_FUNCTION__))
1377 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&(static_cast <bool> ((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 <
2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? void (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1378, __extension__ __PRETTY_FUNCTION__))
1378 "getConstant with a uint64_t value that doesn't fit in the type!")(static_cast <bool> ((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 <
2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? void (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1378, __extension__ __PRETTY_FUNCTION__))
;
1379 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1380}
1381
1382SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1383 bool isT, bool isO) {
1384 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1385}
1386
1387SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1388 EVT VT, bool isT, bool isO) {
1389 assert(VT.isInteger() && "Cannot create FP integer constant!")(static_cast <bool> (VT.isInteger() && "Cannot create FP integer constant!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Cannot create FP integer constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1389, __extension__ __PRETTY_FUNCTION__))
;
1390
1391 EVT EltVT = VT.getScalarType();
1392 const ConstantInt *Elt = &Val;
1393
1394 // In some cases the vector type is legal but the element type is illegal and
1395 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1396 // inserted value (the type does not need to match the vector element type).
1397 // Any extra bits introduced will be truncated away.
1398 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1399 TargetLowering::TypePromoteInteger) {
1400 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1401 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1402 Elt = ConstantInt::get(*getContext(), NewVal);
1403 }
1404 // In other cases the element type is illegal and needs to be expanded, for
1405 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1406 // the value into n parts and use a vector type with n-times the elements.
1407 // Then bitcast to the type requested.
1408 // Legalizing constants too early makes the DAGCombiner's job harder so we
1409 // only legalize if the DAG tells us we must produce legal types.
1410 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1411 TLI->getTypeAction(*getContext(), EltVT) ==
1412 TargetLowering::TypeExpandInteger) {
1413 const APInt &NewVal = Elt->getValue();
1414 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1415 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1416
1417 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1418 if (VT.isScalableVector()) {
1419 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&(static_cast <bool> (EltVT.getSizeInBits() % ViaEltSizeInBits
== 0 && "Can only handle an even split!") ? void (0)
: __assert_fail ("EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && \"Can only handle an even split!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1420, __extension__ __PRETTY_FUNCTION__))
1420 "Can only handle an even split!")(static_cast <bool> (EltVT.getSizeInBits() % ViaEltSizeInBits
== 0 && "Can only handle an even split!") ? void (0)
: __assert_fail ("EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && \"Can only handle an even split!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1420, __extension__ __PRETTY_FUNCTION__))
;
1421 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1422
1423 SmallVector<SDValue, 2> ScalarParts;
1424 for (unsigned i = 0; i != Parts; ++i)
1425 ScalarParts.push_back(getConstant(
1426 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1427 ViaEltVT, isT, isO));
1428
1429 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1430 }
1431
1432 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1433 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1434
1435 // Check the temporary vector is the correct size. If this fails then
1436 // getTypeToTransformTo() probably returned a type whose size (in bits)
1437 // isn't a power-of-2 factor of the requested type size.
1438 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits())(static_cast <bool> (ViaVecVT.getSizeInBits() == VT.getSizeInBits
()) ? void (0) : __assert_fail ("ViaVecVT.getSizeInBits() == VT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1438, __extension__ __PRETTY_FUNCTION__))
;
1439
1440 SmallVector<SDValue, 2> EltParts;
1441 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1442 EltParts.push_back(getConstant(
1443 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1444 ViaEltVT, isT, isO));
1445
1446 // EltParts is currently in little endian order. If we actually want
1447 // big-endian order then reverse it now.
1448 if (getDataLayout().isBigEndian())
1449 std::reverse(EltParts.begin(), EltParts.end());
1450
1451 // The elements must be reversed when the element order is different
1452 // to the endianness of the elements (because the BITCAST is itself a
1453 // vector shuffle in this situation). However, we do not need any code to
1454 // perform this reversal because getConstant() is producing a vector
1455 // splat.
1456 // This situation occurs in MIPS MSA.
1457
1458 SmallVector<SDValue, 8> Ops;
1459 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1460 llvm::append_range(Ops, EltParts);
1461
1462 SDValue V =
1463 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1464 return V;
1465 }
1466
1467 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&(static_cast <bool> (Elt->getBitWidth() == EltVT.getSizeInBits
() && "APInt size does not match type size!") ? void (
0) : __assert_fail ("Elt->getBitWidth() == EltVT.getSizeInBits() && \"APInt size does not match type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1468, __extension__ __PRETTY_FUNCTION__))
1468 "APInt size does not match type size!")(static_cast <bool> (Elt->getBitWidth() == EltVT.getSizeInBits
() && "APInt size does not match type size!") ? void (
0) : __assert_fail ("Elt->getBitWidth() == EltVT.getSizeInBits() && \"APInt size does not match type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1468, __extension__ __PRETTY_FUNCTION__))
;
1469 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1470 FoldingSetNodeID ID;
1471 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1472 ID.AddPointer(Elt);
1473 ID.AddBoolean(isO);
1474 void *IP = nullptr;
1475 SDNode *N = nullptr;
1476 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1477 if (!VT.isVector())
1478 return SDValue(N, 0);
1479
1480 if (!N) {
1481 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1482 CSEMap.InsertNode(N, IP);
1483 InsertNode(N);
1484 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1485 }
1486
1487 SDValue Result(N, 0);
1488 if (VT.isScalableVector())
1489 Result = getSplatVector(VT, DL, Result);
1490 else if (VT.isVector())
1491 Result = getSplatBuildVector(VT, DL, Result);
1492
1493 return Result;
1494}
1495
1496SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1497 bool isTarget) {
1498 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1499}
1500
1501SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1502 const SDLoc &DL, bool LegalTypes) {
1503 assert(VT.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (VT.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1503, __extension__ __PRETTY_FUNCTION__))
;
1504 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1505 return getConstant(Val, DL, ShiftVT);
1506}
1507
1508SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1509 bool isTarget) {
1510 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1511}
1512
1513SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1514 bool isTarget) {
1515 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1516}
1517
1518SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1519 EVT VT, bool isTarget) {
1520 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!")(static_cast <bool> (VT.isFloatingPoint() && "Cannot create integer FP constant!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Cannot create integer FP constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1520, __extension__ __PRETTY_FUNCTION__))
;
1521
1522 EVT EltVT = VT.getScalarType();
1523
1524 // Do the map lookup using the actual bit pattern for the floating point
1525 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1526 // we don't have issues with SNANs.
1527 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1528 FoldingSetNodeID ID;
1529 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1530 ID.AddPointer(&V);
1531 void *IP = nullptr;
1532 SDNode *N = nullptr;
1533 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1534 if (!VT.isVector())
1535 return SDValue(N, 0);
1536
1537 if (!N) {
1538 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1539 CSEMap.InsertNode(N, IP);
1540 InsertNode(N);
1541 }
1542
1543 SDValue Result(N, 0);
1544 if (VT.isScalableVector())
1545 Result = getSplatVector(VT, DL, Result);
1546 else if (VT.isVector())
1547 Result = getSplatBuildVector(VT, DL, Result);
1548 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1549 return Result;
1550}
1551
1552SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1553 bool isTarget) {
1554 EVT EltVT = VT.getScalarType();
1555 if (EltVT == MVT::f32)
1556 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1557 if (EltVT == MVT::f64)
1558 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1559 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1560 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1561 bool Ignored;
1562 APFloat APF = APFloat(Val);
1563 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1564 &Ignored);
1565 return getConstantFP(APF, DL, VT, isTarget);
1566 }
1567 llvm_unreachable("Unsupported type in getConstantFP")::llvm::llvm_unreachable_internal("Unsupported type in getConstantFP"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1567)
;
1568}
1569
1570SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1571 EVT VT, int64_t Offset, bool isTargetGA,
1572 unsigned TargetFlags) {
1573 assert((TargetFlags == 0 || isTargetGA) &&(static_cast <bool> ((TargetFlags == 0 || isTargetGA) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTargetGA) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1574, __extension__ __PRETTY_FUNCTION__))
1574 "Cannot set target flags on target-independent globals")(static_cast <bool> ((TargetFlags == 0 || isTargetGA) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTargetGA) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1574, __extension__ __PRETTY_FUNCTION__))
;
1575
1576 // Truncate (with sign-extension) the offset value to the pointer size.
1577 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1578 if (BitWidth < 64)
1579 Offset = SignExtend64(Offset, BitWidth);
1580
1581 unsigned Opc;
1582 if (GV->isThreadLocal())
1583 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1584 else
1585 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1586
1587 FoldingSetNodeID ID;
1588 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1589 ID.AddPointer(GV);
1590 ID.AddInteger(Offset);
1591 ID.AddInteger(TargetFlags);
1592 void *IP = nullptr;
1593 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1594 return SDValue(E, 0);
1595
1596 auto *N = newSDNode<GlobalAddressSDNode>(
1597 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1598 CSEMap.InsertNode(N, IP);
1599 InsertNode(N);
1600 return SDValue(N, 0);
1601}
1602
1603SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1604 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1605 FoldingSetNodeID ID;
1606 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1607 ID.AddInteger(FI);
1608 void *IP = nullptr;
1609 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1610 return SDValue(E, 0);
1611
1612 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1613 CSEMap.InsertNode(N, IP);
1614 InsertNode(N);
1615 return SDValue(N, 0);
1616}
1617
1618SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1619 unsigned TargetFlags) {
1620 assert((TargetFlags == 0 || isTarget) &&(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent jump tables")
? void (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent jump tables\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1621, __extension__ __PRETTY_FUNCTION__))
1621 "Cannot set target flags on target-independent jump tables")(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent jump tables")
? void (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent jump tables\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1621, __extension__ __PRETTY_FUNCTION__))
;
1622 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1623 FoldingSetNodeID ID;
1624 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1625 ID.AddInteger(JTI);
1626 ID.AddInteger(TargetFlags);
1627 void *IP = nullptr;
1628 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1629 return SDValue(E, 0);
1630
1631 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1632 CSEMap.InsertNode(N, IP);
1633 InsertNode(N);
1634 return SDValue(N, 0);
1635}
1636
1637SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1638 MaybeAlign Alignment, int Offset,
1639 bool isTarget, unsigned TargetFlags) {
1640 assert((TargetFlags == 0 || isTarget) &&(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1641, __extension__ __PRETTY_FUNCTION__))
1641 "Cannot set target flags on target-independent globals")(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1641, __extension__ __PRETTY_FUNCTION__))
;
1642 if (!Alignment)
1643 Alignment = shouldOptForSize()
1644 ? getDataLayout().getABITypeAlign(C->getType())
1645 : getDataLayout().getPrefTypeAlign(C->getType());
1646 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1647 FoldingSetNodeID ID;
1648 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1649 ID.AddInteger(Alignment->value());
1650 ID.AddInteger(Offset);
1651 ID.AddPointer(C);
1652 ID.AddInteger(TargetFlags);
1653 void *IP = nullptr;
1654 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1655 return SDValue(E, 0);
1656
1657 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1658 TargetFlags);
1659 CSEMap.InsertNode(N, IP);
1660 InsertNode(N);
1661 SDValue V = SDValue(N, 0);
1662 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1663 return V;
1664}
1665
1666SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1667 MaybeAlign Alignment, int Offset,
1668 bool isTarget, unsigned TargetFlags) {
1669 assert((TargetFlags == 0 || isTarget) &&(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1670, __extension__ __PRETTY_FUNCTION__))
1670 "Cannot set target flags on target-independent globals")(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1670, __extension__ __PRETTY_FUNCTION__))
;
1671 if (!Alignment)
1672 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1673 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1674 FoldingSetNodeID ID;
1675 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1676 ID.AddInteger(Alignment->value());
1677 ID.AddInteger(Offset);
1678 C->addSelectionDAGCSEId(ID);
1679 ID.AddInteger(TargetFlags);
1680 void *IP = nullptr;
1681 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1682 return SDValue(E, 0);
1683
1684 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1685 TargetFlags);
1686 CSEMap.InsertNode(N, IP);
1687 InsertNode(N);
1688 return SDValue(N, 0);
1689}
1690
1691SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1692 unsigned TargetFlags) {
1693 FoldingSetNodeID ID;
1694 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1695 ID.AddInteger(Index);
1696 ID.AddInteger(Offset);
1697 ID.AddInteger(TargetFlags);
1698 void *IP = nullptr;
1699 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1700 return SDValue(E, 0);
1701
1702 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1703 CSEMap.InsertNode(N, IP);
1704 InsertNode(N);
1705 return SDValue(N, 0);
1706}
1707
1708SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1709 FoldingSetNodeID ID;
1710 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1711 ID.AddPointer(MBB);
1712 void *IP = nullptr;
1713 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1714 return SDValue(E, 0);
1715
1716 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1717 CSEMap.InsertNode(N, IP);
1718 InsertNode(N);
1719 return SDValue(N, 0);
1720}
1721
1722SDValue SelectionDAG::getValueType(EVT VT) {
1723 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1724 ValueTypeNodes.size())
1725 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1726
1727 SDNode *&N = VT.isExtended() ?
1728 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1729
1730 if (N) return SDValue(N, 0);
1731 N = newSDNode<VTSDNode>(VT);
1732 InsertNode(N);
1733 return SDValue(N, 0);
1734}
1735
1736SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1737 SDNode *&N = ExternalSymbols[Sym];
1738 if (N) return SDValue(N, 0);
1739 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1740 InsertNode(N);
1741 return SDValue(N, 0);
1742}
1743
1744SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1745 SDNode *&N = MCSymbols[Sym];
1746 if (N)
1747 return SDValue(N, 0);
1748 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1749 InsertNode(N);
1750 return SDValue(N, 0);
1751}
1752
1753SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1754 unsigned TargetFlags) {
1755 SDNode *&N =
1756 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1757 if (N) return SDValue(N, 0);
1758 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1759 InsertNode(N);
1760 return SDValue(N, 0);
1761}
1762
1763SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1764 if ((unsigned)Cond >= CondCodeNodes.size())
1765 CondCodeNodes.resize(Cond+1);
1766
1767 if (!CondCodeNodes[Cond]) {
1768 auto *N = newSDNode<CondCodeSDNode>(Cond);
1769 CondCodeNodes[Cond] = N;
1770 InsertNode(N);
1771 }
1772
1773 return SDValue(CondCodeNodes[Cond], 0);
1774}
1775
1776SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
1777 APInt One(ResVT.getScalarSizeInBits(), 1);
1778 return getStepVector(DL, ResVT, One);
1779}
1780
1781SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
1782 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth())(static_cast <bool> (ResVT.getScalarSizeInBits() == StepVal
.getBitWidth()) ? void (0) : __assert_fail ("ResVT.getScalarSizeInBits() == StepVal.getBitWidth()"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1782, __extension__ __PRETTY_FUNCTION__))
;
1783 if (ResVT.isScalableVector())
1784 return getNode(
1785 ISD::STEP_VECTOR, DL, ResVT,
1786 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
1787
1788 SmallVector<SDValue, 16> OpsStepConstants;
1789 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
1790 OpsStepConstants.push_back(
1791 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
1792 return getBuildVector(ResVT, DL, OpsStepConstants);
1793}
1794
1795/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1796/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1797static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1798 std::swap(N1, N2);
1799 ShuffleVectorSDNode::commuteMask(M);
1800}
1801
1802SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1803 SDValue N2, ArrayRef<int> Mask) {
1804 assert(VT.getVectorNumElements() == Mask.size() &&(static_cast <bool> (VT.getVectorNumElements() == Mask.
size() && "Must have the same number of vector elements as mask elements!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == Mask.size() && \"Must have the same number of vector elements as mask elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1805, __extension__ __PRETTY_FUNCTION__))
1805 "Must have the same number of vector elements as mask elements!")(static_cast <bool> (VT.getVectorNumElements() == Mask.
size() && "Must have the same number of vector elements as mask elements!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == Mask.size() && \"Must have the same number of vector elements as mask elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1805, __extension__ __PRETTY_FUNCTION__))
;
1806 assert(VT == N1.getValueType() && VT == N2.getValueType() &&(static_cast <bool> (VT == N1.getValueType() &&
VT == N2.getValueType() && "Invalid VECTOR_SHUFFLE")
? void (0) : __assert_fail ("VT == N1.getValueType() && VT == N2.getValueType() && \"Invalid VECTOR_SHUFFLE\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1807, __extension__ __PRETTY_FUNCTION__))
1807 "Invalid VECTOR_SHUFFLE")(static_cast <bool> (VT == N1.getValueType() &&
VT == N2.getValueType() && "Invalid VECTOR_SHUFFLE")
? void (0) : __assert_fail ("VT == N1.getValueType() && VT == N2.getValueType() && \"Invalid VECTOR_SHUFFLE\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1807, __extension__ __PRETTY_FUNCTION__))
;
1808
1809 // Canonicalize shuffle undef, undef -> undef
1810 if (N1.isUndef() && N2.isUndef())
1811 return getUNDEF(VT);
1812
1813 // Validate that all indices in Mask are within the range of the elements
1814 // input to the shuffle.
1815 int NElts = Mask.size();
1816 assert(llvm::all_of(Mask,(static_cast <bool> (llvm::all_of(Mask, [&](int M) {
return M < (NElts * 2) && M >= -1; }) &&
"Index out of range") ? void (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1818, __extension__ __PRETTY_FUNCTION__))
1817 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&(static_cast <bool> (llvm::all_of(Mask, [&](int M) {
return M < (NElts * 2) && M >= -1; }) &&
"Index out of range") ? void (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1818, __extension__ __PRETTY_FUNCTION__))
1818 "Index out of range")(static_cast <bool> (llvm::all_of(Mask, [&](int M) {
return M < (NElts * 2) && M >= -1; }) &&
"Index out of range") ? void (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1818, __extension__ __PRETTY_FUNCTION__))
;
1819
1820 // Copy the mask so we can do any needed cleanup.
1821 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1822
1823 // Canonicalize shuffle v, v -> v, undef
1824 if (N1 == N2) {
1825 N2 = getUNDEF(VT);
1826 for (int i = 0; i != NElts; ++i)
1827 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1828 }
1829
1830 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1831 if (N1.isUndef())
1832 commuteShuffle(N1, N2, MaskVec);
1833
1834 if (TLI->hasVectorBlend()) {
1835 // If shuffling a splat, try to blend the splat instead. We do this here so
1836 // that even when this arises during lowering we don't have to re-handle it.
1837 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1838 BitVector UndefElements;
1839 SDValue Splat = BV->getSplatValue(&UndefElements);
1840 if (!Splat)
1841 return;
1842
1843 for (int i = 0; i < NElts; ++i) {
1844 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1845 continue;
1846
1847 // If this input comes from undef, mark it as such.
1848 if (UndefElements[MaskVec[i] - Offset]) {
1849 MaskVec[i] = -1;
1850 continue;
1851 }
1852
1853 // If we can blend a non-undef lane, use that instead.
1854 if (!UndefElements[i])
1855 MaskVec[i] = i + Offset;
1856 }
1857 };
1858 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1859 BlendSplat(N1BV, 0);
1860 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1861 BlendSplat(N2BV, NElts);
1862 }
1863
1864 // Canonicalize all index into lhs, -> shuffle lhs, undef
1865 // Canonicalize all index into rhs, -> shuffle rhs, undef
1866 bool AllLHS = true, AllRHS = true;
1867 bool N2Undef = N2.isUndef();
1868 for (int i = 0; i != NElts; ++i) {
1869 if (MaskVec[i] >= NElts) {
1870 if (N2Undef)
1871 MaskVec[i] = -1;
1872 else
1873 AllLHS = false;
1874 } else if (MaskVec[i] >= 0) {
1875 AllRHS = false;
1876 }
1877 }
1878 if (AllLHS && AllRHS)
1879 return getUNDEF(VT);
1880 if (AllLHS && !N2Undef)
1881 N2 = getUNDEF(VT);
1882 if (AllRHS) {
1883 N1 = getUNDEF(VT);
1884 commuteShuffle(N1, N2, MaskVec);
1885 }
1886 // Reset our undef status after accounting for the mask.
1887 N2Undef = N2.isUndef();
1888 // Re-check whether both sides ended up undef.
1889 if (N1.isUndef() && N2Undef)
1890 return getUNDEF(VT);
1891
1892 // If Identity shuffle return that node.
1893 bool Identity = true, AllSame = true;
1894 for (int i = 0; i != NElts; ++i) {
1895 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1896 if (MaskVec[i] != MaskVec[0]) AllSame = false;
1897 }
1898 if (Identity && NElts)
1899 return N1;
1900
1901 // Shuffling a constant splat doesn't change the result.
1902 if (N2Undef) {
1903 SDValue V = N1;
1904
1905 // Look through any bitcasts. We check that these don't change the number
1906 // (and size) of elements and just changes their types.
1907 while (V.getOpcode() == ISD::BITCAST)
1908 V = V->getOperand(0);
1909
1910 // A splat should always show up as a build vector node.
1911 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1912 BitVector UndefElements;
1913 SDValue Splat = BV->getSplatValue(&UndefElements);
1914 // If this is a splat of an undef, shuffling it is also undef.
1915 if (Splat && Splat.isUndef())
1916 return getUNDEF(VT);
1917
1918 bool SameNumElts =
1919 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1920
1921 // We only have a splat which can skip shuffles if there is a splatted
1922 // value and no undef lanes rearranged by the shuffle.
1923 if (Splat && UndefElements.none()) {
1924 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1925 // number of elements match or the value splatted is a zero constant.
1926 if (SameNumElts)
1927 return N1;
1928 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1929 if (C->isZero())
1930 return N1;
1931 }
1932
1933 // If the shuffle itself creates a splat, build the vector directly.
1934 if (AllSame && SameNumElts) {
1935 EVT BuildVT = BV->getValueType(0);
1936 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1937 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1938
1939 // We may have jumped through bitcasts, so the type of the
1940 // BUILD_VECTOR may not match the type of the shuffle.
1941 if (BuildVT != VT)
1942 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1943 return NewBV;
1944 }
1945 }
1946 }
1947
1948 FoldingSetNodeID ID;
1949 SDValue Ops[2] = { N1, N2 };
1950 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1951 for (int i = 0; i != NElts; ++i)
1952 ID.AddInteger(MaskVec[i]);
1953
1954 void* IP = nullptr;
1955 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1956 return SDValue(E, 0);
1957
1958 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1959 // SDNode doesn't have access to it. This memory will be "leaked" when
1960 // the node is deallocated, but recovered when the NodeAllocator is released.
1961 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1962 llvm::copy(MaskVec, MaskAlloc);
1963
1964 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1965 dl.getDebugLoc(), MaskAlloc);
1966 createOperands(N, Ops);
1967
1968 CSEMap.InsertNode(N, IP);
1969 InsertNode(N);
1970 SDValue V = SDValue(N, 0);
1971 NewSDValueDbgMsg(V, "Creating new node: ", this);
1972 return V;
1973}
1974
1975SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1976 EVT VT = SV.getValueType(0);
1977 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1978 ShuffleVectorSDNode::commuteMask(MaskVec);
1979
1980 SDValue Op0 = SV.getOperand(0);
1981 SDValue Op1 = SV.getOperand(1);
1982 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1983}
1984
1985SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1986 FoldingSetNodeID ID;
1987 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1988 ID.AddInteger(RegNo);
1989 void *IP = nullptr;
1990 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1991 return SDValue(E, 0);
1992
1993 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1994 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1995 CSEMap.InsertNode(N, IP);
1996 InsertNode(N);
1997 return SDValue(N, 0);
1998}
1999
2000SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2001 FoldingSetNodeID ID;
2002 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
2003 ID.AddPointer(RegMask);
2004 void *IP = nullptr;
2005 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2006 return SDValue(E, 0);
2007
2008 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2009 CSEMap.InsertNode(N, IP);
2010 InsertNode(N);
2011 return SDValue(N, 0);
2012}
2013
2014SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2015 MCSymbol *Label) {
2016 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2017}
2018
2019SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2020 SDValue Root, MCSymbol *Label) {
2021 FoldingSetNodeID ID;
2022 SDValue Ops[] = { Root };
2023 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2024 ID.AddPointer(Label);
2025 void *IP = nullptr;
2026 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2027 return SDValue(E, 0);
2028
2029 auto *N =
2030 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2031 createOperands(N, Ops);
2032
2033 CSEMap.InsertNode(N, IP);
2034 InsertNode(N);
2035 return SDValue(N, 0);
2036}
2037
2038SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2039 int64_t Offset, bool isTarget,
2040 unsigned TargetFlags) {
2041 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2042
2043 FoldingSetNodeID ID;
2044 AddNodeIDNode(ID, Opc, getVTList(VT), None);
2045 ID.AddPointer(BA);
2046 ID.AddInteger(Offset);
2047 ID.AddInteger(TargetFlags);
2048 void *IP = nullptr;
2049 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2050 return SDValue(E, 0);
2051
2052 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2053 CSEMap.InsertNode(N, IP);
2054 InsertNode(N);
2055 return SDValue(N, 0);
2056}
2057
2058SDValue SelectionDAG::getSrcValue(const Value *V) {
2059 FoldingSetNodeID ID;
2060 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
2061 ID.AddPointer(V);
2062
2063 void *IP = nullptr;
2064 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2065 return SDValue(E, 0);
2066
2067 auto *N = newSDNode<SrcValueSDNode>(V);
2068 CSEMap.InsertNode(N, IP);
2069 InsertNode(N);
2070 return SDValue(N, 0);
2071}
2072
2073SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2074 FoldingSetNodeID ID;
2075 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
2076 ID.AddPointer(MD);
2077
2078 void *IP = nullptr;
2079 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2080 return SDValue(E, 0);
2081
2082 auto *N = newSDNode<MDNodeSDNode>(MD);
2083 CSEMap.InsertNode(N, IP);
2084 InsertNode(N);
2085 return SDValue(N, 0);
2086}
2087
2088SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2089 if (VT == V.getValueType())
2090 return V;
2091
2092 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2093}
2094
2095SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2096 unsigned SrcAS, unsigned DestAS) {
2097 SDValue Ops[] = {Ptr};
2098 FoldingSetNodeID ID;
2099 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
2100 ID.AddInteger(SrcAS);
2101 ID.AddInteger(DestAS);
2102
2103 void *IP = nullptr;
2104 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2105 return SDValue(E, 0);
2106
2107 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2108 VT, SrcAS, DestAS);
2109 createOperands(N, Ops);
2110
2111 CSEMap.InsertNode(N, IP);
2112 InsertNode(N);
2113 return SDValue(N, 0);
2114}
2115
2116SDValue SelectionDAG::getFreeze(SDValue V) {
2117 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2118}
2119
2120/// getShiftAmountOperand - Return the specified value casted to
2121/// the target's desired shift amount type.
2122SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2123 EVT OpTy = Op.getValueType();
2124 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2125 if (OpTy == ShTy || OpTy.isVector()) return Op;
2126
2127 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2128}
2129
2130SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2131 SDLoc dl(Node);
2132 const TargetLowering &TLI = getTargetLoweringInfo();
2133 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2134 EVT VT = Node->getValueType(0);
2135 SDValue Tmp1 = Node->getOperand(0);
2136 SDValue Tmp2 = Node->getOperand(1);
2137 const MaybeAlign MA(Node->getConstantOperandVal(3));
2138
2139 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2140 Tmp2, MachinePointerInfo(V));
2141 SDValue VAList = VAListLoad;
2142
2143 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2144 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2145 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2146
2147 VAList =
2148 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2149 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2150 }
2151
2152 // Increment the pointer, VAList, to the next vaarg
2153 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2154 getConstant(getDataLayout().getTypeAllocSize(
2155 VT.getTypeForEVT(*getContext())),
2156 dl, VAList.getValueType()));
2157 // Store the incremented VAList to the legalized pointer
2158 Tmp1 =
2159 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2160 // Load the actual argument out of the pointer VAList
2161 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2162}
2163
2164SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2165 SDLoc dl(Node);
2166 const TargetLowering &TLI = getTargetLoweringInfo();
2167 // This defaults to loading a pointer from the input and storing it to the
2168 // output, returning the chain.
2169 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2170 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2171 SDValue Tmp1 =
2172 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2173 Node->getOperand(2), MachinePointerInfo(VS));
2174 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2175 MachinePointerInfo(VD));
2176}
2177
2178Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2179 const DataLayout &DL = getDataLayout();
2180 Type *Ty = VT.getTypeForEVT(*getContext());
2181 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2182
2183 if (TLI->isTypeLegal(VT) || !VT.isVector())
2184 return RedAlign;
2185
2186 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2187 const Align StackAlign = TFI->getStackAlign();
2188
2189 // See if we can choose a smaller ABI alignment in cases where it's an
2190 // illegal vector type that will get broken down.
2191 if (RedAlign > StackAlign) {
2192 EVT IntermediateVT;
2193 MVT RegisterVT;
2194 unsigned NumIntermediates;
2195 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2196 NumIntermediates, RegisterVT);
2197 Ty = IntermediateVT.getTypeForEVT(*getContext());
2198 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2199 if (RedAlign2 < RedAlign)
2200 RedAlign = RedAlign2;
2201 }
2202
2203 return RedAlign;
2204}
2205
2206SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2207 MachineFrameInfo &MFI = MF->getFrameInfo();
2208 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2209 int StackID = 0;
2210 if (Bytes.isScalable())
2211 StackID = TFI->getStackIDForScalableVectors();
2212 // The stack id gives an indication of whether the object is scalable or
2213 // not, so it's safe to pass in the minimum size here.
2214 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment,
2215 false, nullptr, StackID);
2216 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2217}
2218
2219SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2220 Type *Ty = VT.getTypeForEVT(*getContext());
2221 Align StackAlign =
2222 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2223 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2224}
2225
2226SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2227 TypeSize VT1Size = VT1.getStoreSize();
2228 TypeSize VT2Size = VT2.getStoreSize();
2229 assert(VT1Size.isScalable() == VT2Size.isScalable() &&(static_cast <bool> (VT1Size.isScalable() == VT2Size.isScalable
() && "Don't know how to choose the maximum size when creating a stack "
"temporary") ? void (0) : __assert_fail ("VT1Size.isScalable() == VT2Size.isScalable() && \"Don't know how to choose the maximum size when creating a stack \" \"temporary\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2231, __extension__ __PRETTY_FUNCTION__))
2230 "Don't know how to choose the maximum size when creating a stack "(static_cast <bool> (VT1Size.isScalable() == VT2Size.isScalable
() && "Don't know how to choose the maximum size when creating a stack "
"temporary") ? void (0) : __assert_fail ("VT1Size.isScalable() == VT2Size.isScalable() && \"Don't know how to choose the maximum size when creating a stack \" \"temporary\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2231, __extension__ __PRETTY_FUNCTION__))
2231 "temporary")(static_cast <bool> (VT1Size.isScalable() == VT2Size.isScalable
() && "Don't know how to choose the maximum size when creating a stack "
"temporary") ? void (0) : __assert_fail ("VT1Size.isScalable() == VT2Size.isScalable() && \"Don't know how to choose the maximum size when creating a stack \" \"temporary\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2231, __extension__ __PRETTY_FUNCTION__))
;
2232 TypeSize Bytes =
2233 VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size;
2234
2235 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2236 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2237 const DataLayout &DL = getDataLayout();
2238 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2239 return CreateStackTemporary(Bytes, Align);
2240}
2241
2242SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2243 ISD::CondCode Cond, const SDLoc &dl) {
2244 EVT OpVT = N1.getValueType();
2245
2246 // These setcc operations always fold.
2247 switch (Cond) {
2248 default: break;
2249 case ISD::SETFALSE:
2250 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2251 case ISD::SETTRUE:
2252 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2253
2254 case ISD::SETOEQ:
2255 case ISD::SETOGT:
2256 case ISD::SETOGE:
2257 case ISD::SETOLT:
2258 case ISD::SETOLE:
2259 case ISD::SETONE:
2260 case ISD::SETO:
2261 case ISD::SETUO:
2262 case ISD::SETUEQ:
2263 case ISD::SETUNE:
2264 assert(!OpVT.isInteger() && "Illegal setcc for integer!")(static_cast <bool> (!OpVT.isInteger() && "Illegal setcc for integer!"
) ? void (0) : __assert_fail ("!OpVT.isInteger() && \"Illegal setcc for integer!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2264, __extension__ __PRETTY_FUNCTION__))
;
2265 break;
2266 }
2267
2268 if (OpVT.isInteger()) {
2269 // For EQ and NE, we can always pick a value for the undef to make the
2270 // predicate pass or fail, so we can return undef.
2271 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2272 // icmp eq/ne X, undef -> undef.
2273 if ((N1.isUndef() || N2.isUndef()) &&
2274 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2275 return getUNDEF(VT);
2276
2277 // If both operands are undef, we can return undef for int comparison.
2278 // icmp undef, undef -> undef.
2279 if (N1.isUndef() && N2.isUndef())
2280 return getUNDEF(VT);
2281
2282 // icmp X, X -> true/false
2283 // icmp X, undef -> true/false because undef could be X.
2284 if (N1 == N2)
2285 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2286 }
2287
2288 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2289 const APInt &C2 = N2C->getAPIntValue();
2290 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2291 const APInt &C1 = N1C->getAPIntValue();
2292
2293 switch (Cond) {
2294 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2294)
;
2295 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT);
2296 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT);
2297 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
2298 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
2299 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
2300 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2301 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2302 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2303 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2304 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2305 }
2306 }
2307 }
2308
2309 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2310 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2311
2312 if (N1CFP && N2CFP) {
2313 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2314 switch (Cond) {
2315 default: break;
2316 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2317 return getUNDEF(VT);
2318 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2319 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2320 OpVT);
2321 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2322 return getUNDEF(VT);
2323 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2324 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2325 R==APFloat::cmpLessThan, dl, VT,
2326 OpVT);
2327 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2328 return getUNDEF(VT);
2329 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2330 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2331 OpVT);
2332 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2333 return getUNDEF(VT);
2334 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2335 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2336 VT, OpVT);
2337 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2338 return getUNDEF(VT);
2339 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2340 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2341 R==APFloat::cmpEqual, dl, VT,
2342 OpVT);
2343 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2344 return getUNDEF(VT);
2345 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2346 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2347 R==APFloat::cmpEqual, dl, VT, OpVT);
2348 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2349 OpVT);
2350 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2351 OpVT);
2352 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2353 R==APFloat::cmpEqual, dl, VT,
2354 OpVT);
2355 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2356 OpVT);
2357 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2358 R==APFloat::cmpLessThan, dl, VT,
2359 OpVT);
2360 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2361 R==APFloat::cmpUnordered, dl, VT,
2362 OpVT);
2363 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2364 VT, OpVT);
2365 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2366 OpVT);
2367 }
2368 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2369 // Ensure that the constant occurs on the RHS.
2370 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2371 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2372 return SDValue();
2373 return getSetCC(dl, VT, N2, N1, SwappedCond);
2374 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2375 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2376 // If an operand is known to be a nan (or undef that could be a nan), we can
2377 // fold it.
2378 // Choosing NaN for the undef will always make unordered comparison succeed
2379 // and ordered comparison fails.
2380 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2381 switch (ISD::getUnorderedFlavor(Cond)) {
2382 default:
2383 llvm_unreachable("Unknown flavor!")::llvm::llvm_unreachable_internal("Unknown flavor!", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2383)
;
2384 case 0: // Known false.
2385 return getBoolConstant(false, dl, VT, OpVT);
2386 case 1: // Known true.
2387 return getBoolConstant(true, dl, VT, OpVT);
2388 case 2: // Undefined.
2389 return getUNDEF(VT);
2390 }
2391 }
2392
2393 // Could not fold it.
2394 return SDValue();
2395}
2396
2397/// See if the specified operand can be simplified with the knowledge that only
2398/// the bits specified by DemandedBits are used.
2399/// TODO: really we should be making this into the DAG equivalent of
2400/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2401SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2402 EVT VT = V.getValueType();
2403
2404 if (VT.isScalableVector())
2405 return SDValue();
2406
2407 APInt DemandedElts = VT.isVector()
2408 ? APInt::getAllOnes(VT.getVectorNumElements())
2409 : APInt(1, 1);
2410 return GetDemandedBits(V, DemandedBits, DemandedElts);
2411}
2412
2413/// See if the specified operand can be simplified with the knowledge that only
2414/// the bits specified by DemandedBits are used in the elements specified by
2415/// DemandedElts.
2416/// TODO: really we should be making this into the DAG equivalent of
2417/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2418SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2419 const APInt &DemandedElts) {
2420 switch (V.getOpcode()) {
2421 default:
2422 return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts,
2423 *this, 0);
2424 case ISD::Constant: {
2425 const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue();
2426 APInt NewVal = CVal & DemandedBits;
2427 if (NewVal != CVal)
2428 return getConstant(NewVal, SDLoc(V), V.getValueType());
2429 break;
2430 }
2431 case ISD::SRL:
2432 // Only look at single-use SRLs.
2433 if (!V.getNode()->hasOneUse())
2434 break;
2435 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2436 // See if we can recursively simplify the LHS.
2437 unsigned Amt = RHSC->getZExtValue();
2438
2439 // Watch out for shift count overflow though.
2440 if (Amt >= DemandedBits.getBitWidth())
2441 break;
2442 APInt SrcDemandedBits = DemandedBits << Amt;
2443 if (SDValue SimplifyLHS =
2444 GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2445 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2446 V.getOperand(1));
2447 }
2448 break;
2449 }
2450 return SDValue();
2451}
2452
2453/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2454/// use this predicate to simplify operations downstream.
2455bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2456 unsigned BitWidth = Op.getScalarValueSizeInBits();
2457 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2458}
2459
2460/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2461/// this predicate to simplify operations downstream. Mask is known to be zero
2462/// for bits that V cannot have.
2463bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2464 unsigned Depth) const {
2465 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2466}
2467
2468/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2469/// DemandedElts. We use this predicate to simplify operations downstream.
2470/// Mask is known to be zero for bits that V cannot have.
2471bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2472 const APInt &DemandedElts,
2473 unsigned Depth) const {
2474 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2475}
2476
2477/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2478bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2479 unsigned Depth) const {
2480 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2481}
2482
2483/// isSplatValue - Return true if the vector V has the same value
2484/// across all DemandedElts. For scalable vectors it does not make
2485/// sense to specify which elements are demanded or undefined, therefore
2486/// they are simply ignored.
2487bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2488 APInt &UndefElts, unsigned Depth) {
2489 EVT VT = V.getValueType();
2490 assert(VT.isVector() && "Vector type expected")(static_cast <bool> (VT.isVector() && "Vector type expected"
) ? void (0) : __assert_fail ("VT.isVector() && \"Vector type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2490, __extension__ __PRETTY_FUNCTION__))
;
2491
2492 if (!VT.isScalableVector() && !DemandedElts)
2493 return false; // No demanded elts, better to assume we don't know anything.
2494
2495 if (Depth >= MaxRecursionDepth)
2496 return false; // Limit search depth.
2497
2498 // Deal with some common cases here that work for both fixed and scalable
2499 // vector types.
2500 switch (V.getOpcode()) {
2501 case ISD::SPLAT_VECTOR:
2502 UndefElts = V.getOperand(0).isUndef()
2503 ? APInt::getAllOnes(DemandedElts.getBitWidth())
2504 : APInt(DemandedElts.getBitWidth(), 0);
2505 return true;
2506 case ISD::ADD:
2507 case ISD::SUB:
2508 case ISD::AND:
2509 case ISD::XOR:
2510 case ISD::OR: {
2511 APInt UndefLHS, UndefRHS;
2512 SDValue LHS = V.getOperand(0);
2513 SDValue RHS = V.getOperand(1);
2514 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2515 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2516 UndefElts = UndefLHS | UndefRHS;
2517 return true;
2518 }
2519 return false;
2520 }
2521 case ISD::ABS:
2522 case ISD::TRUNCATE:
2523 case ISD::SIGN_EXTEND:
2524 case ISD::ZERO_EXTEND:
2525 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2526 }
2527
2528 // We don't support other cases than those above for scalable vectors at
2529 // the moment.
2530 if (VT.isScalableVector())
2531 return false;
2532
2533 unsigned NumElts = VT.getVectorNumElements();
2534 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch")(static_cast <bool> (NumElts == DemandedElts.getBitWidth
() && "Vector size mismatch") ? void (0) : __assert_fail
("NumElts == DemandedElts.getBitWidth() && \"Vector size mismatch\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2534, __extension__ __PRETTY_FUNCTION__))
;
2535 UndefElts = APInt::getZero(NumElts);
2536
2537 switch (V.getOpcode()) {
2538 case ISD::BUILD_VECTOR: {
2539 SDValue Scl;
2540 for (unsigned i = 0; i != NumElts; ++i) {
2541 SDValue Op = V.getOperand(i);
2542 if (Op.isUndef()) {
2543 UndefElts.setBit(i);
2544 continue;
2545 }
2546 if (!DemandedElts[i])
2547 continue;
2548 if (Scl && Scl != Op)
2549 return false;
2550 Scl = Op;
2551 }
2552 return true;
2553 }
2554 case ISD::VECTOR_SHUFFLE: {
2555 // Check if this is a shuffle node doing a splat.
2556 // TODO: Do we need to handle shuffle(splat, undef, mask)?
2557 int SplatIndex = -1;
2558 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2559 for (int i = 0; i != (int)NumElts; ++i) {
2560 int M = Mask[i];
2561 if (M < 0) {
2562 UndefElts.setBit(i);
2563 continue;
2564 }
2565 if (!DemandedElts[i])
2566 continue;
2567 if (0 <= SplatIndex && SplatIndex != M)
2568 return false;
2569 SplatIndex = M;
2570 }
2571 return true;
2572 }
2573 case ISD::EXTRACT_SUBVECTOR: {
2574 // Offset the demanded elts by the subvector index.
2575 SDValue Src = V.getOperand(0);
2576 // We don't support scalable vectors at the moment.
2577 if (Src.getValueType().isScalableVector())
2578 return false;
2579 uint64_t Idx = V.getConstantOperandVal(1);
2580 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2581 APInt UndefSrcElts;
2582 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2583 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2584 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2585 return true;
2586 }
2587 break;
2588 }
2589 }
2590
2591 return false;
2592}
2593
2594/// Helper wrapper to main isSplatValue function.
2595bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2596 EVT VT = V.getValueType();
2597 assert(VT.isVector() && "Vector type expected")(static_cast <bool> (VT.isVector() && "Vector type expected"
) ? void (0) : __assert_fail ("VT.isVector() && \"Vector type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2597, __extension__ __PRETTY_FUNCTION__))
;
2598
2599 APInt UndefElts;
2600 APInt DemandedElts;
2601
2602 // For now we don't support this with scalable vectors.
2603 if (!VT.isScalableVector())
2604 DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
2605 return isSplatValue(V, DemandedElts, UndefElts) &&
2606 (AllowUndefs || !UndefElts);
2607}
2608
2609SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2610 V = peekThroughExtractSubvectors(V);
2611
2612 EVT VT = V.getValueType();
2613 unsigned Opcode = V.getOpcode();
2614 switch (Opcode) {
2615 default: {
2616 APInt UndefElts;
2617 APInt DemandedElts;
2618
2619 if (!VT.isScalableVector())
2620 DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
2621
2622 if (isSplatValue(V, DemandedElts, UndefElts)) {
2623 if (VT.isScalableVector()) {
2624 // DemandedElts and UndefElts are ignored for scalable vectors, since
2625 // the only supported cases are SPLAT_VECTOR nodes.
2626 SplatIdx = 0;
2627 } else {
2628 // Handle case where all demanded elements are UNDEF.
2629 if (DemandedElts.isSubsetOf(UndefElts)) {
2630 SplatIdx = 0;
2631 return getUNDEF(VT);
2632 }
2633 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2634 }
2635 return V;
2636 }
2637 break;
2638 }
2639 case ISD::SPLAT_VECTOR:
2640 SplatIdx = 0;
2641 return V;
2642 case ISD::VECTOR_SHUFFLE: {
2643 if (VT.isScalableVector())
2644 return SDValue();
2645
2646 // Check if this is a shuffle node doing a splat.
2647 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2648 // getTargetVShiftNode currently struggles without the splat source.
2649 auto *SVN = cast<ShuffleVectorSDNode>(V);
2650 if (!SVN->isSplat())
2651 break;
2652 int Idx = SVN->getSplatIndex();
2653 int NumElts = V.getValueType().getVectorNumElements();
2654 SplatIdx = Idx % NumElts;
2655 return V.getOperand(Idx / NumElts);
2656 }
2657 }
2658
2659 return SDValue();
2660}
2661
2662SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2663 int SplatIdx;
2664 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2665 EVT SVT = SrcVector.getValueType().getScalarType();
2666 EVT LegalSVT = SVT;
2667 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2668 if (!SVT.isInteger())
2669 return SDValue();
2670 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2671 if (LegalSVT.bitsLT(SVT))
2672 return SDValue();
2673 }
2674 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2675 getVectorIdxConstant(SplatIdx, SDLoc(V)));
2676 }
2677 return SDValue();
2678}
2679
2680const APInt *
2681SelectionDAG::getValidShiftAmountConstant(SDValue V,
2682 const APInt &DemandedElts) const {
2683 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2685, __extension__ __PRETTY_FUNCTION__))
2684 V.getOpcode() == ISD::SRA) &&(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2685, __extension__ __PRETTY_FUNCTION__))
2685 "Unknown shift node")(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2685, __extension__ __PRETTY_FUNCTION__))
;
2686 unsigned BitWidth = V.getScalarValueSizeInBits();
2687 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2688 // Shifting more than the bitwidth is not valid.
2689 const APInt &ShAmt = SA->getAPIntValue();
2690 if (ShAmt.ult(BitWidth))
2691 return &ShAmt;
2692 }
2693 return nullptr;
2694}
2695
2696const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2697 SDValue V, const APInt &DemandedElts) const {
2698 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2700, __extension__ __PRETTY_FUNCTION__))
2699 V.getOpcode() == ISD::SRA) &&(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2700, __extension__ __PRETTY_FUNCTION__))
2700 "Unknown shift node")(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2700, __extension__ __PRETTY_FUNCTION__))
;
2701 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2702 return ValidAmt;
2703 unsigned BitWidth = V.getScalarValueSizeInBits();
2704 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2705 if (!BV)
2706 return nullptr;
2707 const APInt *MinShAmt = nullptr;
2708 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2709 if (!DemandedElts[i])
2710 continue;
2711 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2712 if (!SA)
2713 return nullptr;
2714 // Shifting more than the bitwidth is not valid.
2715 const APInt &ShAmt = SA->getAPIntValue();
2716 if (ShAmt.uge(BitWidth))
2717 return nullptr;
2718 if (MinShAmt && MinShAmt->ule(ShAmt))
2719 continue;
2720 MinShAmt = &ShAmt;
2721 }
2722 return MinShAmt;
2723}
2724
2725const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
2726 SDValue V, const APInt &DemandedElts) const {
2727 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2729, __extension__ __PRETTY_FUNCTION__))
2728 V.getOpcode() == ISD::SRA) &&(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2729, __extension__ __PRETTY_FUNCTION__))
2729 "Unknown shift node")(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2729, __extension__ __PRETTY_FUNCTION__))
;
2730 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2731 return ValidAmt;
2732 unsigned BitWidth = V.getScalarValueSizeInBits();
2733 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2734 if (!BV)
2735 return nullptr;
2736 const APInt *MaxShAmt = nullptr;
2737 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2738 if (!DemandedElts[i])
2739 continue;
2740 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2741 if (!SA)
2742 return nullptr;
2743 // Shifting more than the bitwidth is not valid.
2744 const APInt &ShAmt = SA->getAPIntValue();
2745 if (ShAmt.uge(BitWidth))
2746 return nullptr;
2747 if (MaxShAmt && MaxShAmt->uge(ShAmt))
2748 continue;
2749 MaxShAmt = &ShAmt;
2750 }
2751 return MaxShAmt;
2752}
2753
2754/// Determine which bits of Op are known to be either zero or one and return
2755/// them in Known. For vectors, the known bits are those that are shared by
2756/// every vector element.
2757KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2758 EVT VT = Op.getValueType();
2759
2760 // TOOD: Until we have a plan for how to represent demanded elements for
2761 // scalable vectors, we can just bail out for now.
2762 if (Op.getValueType().isScalableVector()) {
2763 unsigned BitWidth = Op.getScalarValueSizeInBits();
2764 return KnownBits(BitWidth);
2765 }
2766
2767 APInt DemandedElts = VT.isVector()
2768 ? APInt::getAllOnes(VT.getVectorNumElements())
2769 : APInt(1, 1);
2770 return computeKnownBits(Op, DemandedElts, Depth);
2771}
2772
2773/// Determine which bits of Op are known to be either zero or one and return
2774/// them in Known. The DemandedElts argument allows us to only collect the known
2775/// bits that are shared by the requested vector elements.
2776KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2777 unsigned Depth) const {
2778 unsigned BitWidth = Op.getScalarValueSizeInBits();
2779
2780 KnownBits Known(BitWidth); // Don't know anything.
2781
2782 // TOOD: Until we have a plan for how to represent demanded elements for
2783 // scalable vectors, we can just bail out for now.
2784 if (Op.getValueType().isScalableVector())
2785 return Known;
2786
2787 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2788 // We know all of the bits for a constant!
2789 return KnownBits::makeConstant(C->getAPIntValue());
2790 }
2791 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2792 // We know all of the bits for a constant fp!
2793 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
2794 }
2795
2796 if (Depth >= MaxRecursionDepth)
2797 return Known; // Limit search depth.
2798
2799 KnownBits Known2;
2800 unsigned NumElts = DemandedElts.getBitWidth();
2801 assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2803, __extension__ __PRETTY_FUNCTION__))
2802 NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2803, __extension__ __PRETTY_FUNCTION__))
2803 "Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2803, __extension__ __PRETTY_FUNCTION__))
;
2804
2805 if (!DemandedElts)
2806 return Known; // No demanded elts, better to assume we don't know anything.
2807
2808 unsigned Opcode = Op.getOpcode();
2809 switch (Opcode) {
2810 case ISD::BUILD_VECTOR:
2811 // Collect the known bits that are shared by every demanded vector element.
2812 Known.Zero.setAllBits(); Known.One.setAllBits();
2813 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2814 if (!DemandedElts[i])
2815 continue;
2816
2817 SDValue SrcOp = Op.getOperand(i);
2818 Known2 = computeKnownBits(SrcOp, Depth + 1);
2819
2820 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2821 if (SrcOp.getValueSizeInBits() != BitWidth) {
2822 assert(SrcOp.getValueSizeInBits() > BitWidth &&(static_cast <bool> (SrcOp.getValueSizeInBits() > BitWidth
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > BitWidth && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2823, __extension__ __PRETTY_FUNCTION__))
2823 "Expected BUILD_VECTOR implicit truncation")(static_cast <bool> (SrcOp.getValueSizeInBits() > BitWidth
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > BitWidth && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2823, __extension__ __PRETTY_FUNCTION__))
;
2824 Known2 = Known2.trunc(BitWidth);
2825 }
2826
2827 // Known bits are the values that are shared by every demanded element.
2828 Known = KnownBits::commonBits(Known, Known2);
2829
2830 // If we don't know any bits, early out.
2831 if (Known.isUnknown())
2832 break;
2833 }
2834 break;
2835 case ISD::VECTOR_SHUFFLE: {
2836 // Collect the known bits that are shared by every vector element referenced
2837 // by the shuffle.
2838 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2839 Known.Zero.setAllBits(); Known.One.setAllBits();
2840 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2841 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")(static_cast <bool> (NumElts == SVN->getMask().size(
) && "Unexpected vector size") ? void (0) : __assert_fail
("NumElts == SVN->getMask().size() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2841, __extension__ __PRETTY_FUNCTION__))
;
2842 for (unsigned i = 0; i != NumElts; ++i) {
2843 if (!DemandedElts[i])
2844 continue;
2845
2846 int M = SVN->getMaskElt(i);
2847 if (M < 0) {
2848 // For UNDEF elements, we don't know anything about the common state of
2849 // the shuffle result.
2850 Known.resetAll();
2851 DemandedLHS.clearAllBits();
2852 DemandedRHS.clearAllBits();
2853 break;
2854 }
2855
2856 if ((unsigned)M < NumElts)
2857 DemandedLHS.setBit((unsigned)M % NumElts);
2858 else
2859 DemandedRHS.setBit((unsigned)M % NumElts);
2860 }
2861 // Known bits are the values that are shared by every demanded element.
2862 if (!!DemandedLHS) {
2863 SDValue LHS = Op.getOperand(0);
2864 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2865 Known = KnownBits::commonBits(Known, Known2);
2866 }
2867 // If we don't know any bits, early out.
2868 if (Known.isUnknown())
2869 break;
2870 if (!!DemandedRHS) {
2871 SDValue RHS = Op.getOperand(1);
2872 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2873 Known = KnownBits::commonBits(Known, Known2);
2874 }
2875 break;
2876 }
2877 case ISD::CONCAT_VECTORS: {
2878 // Split DemandedElts and test each of the demanded subvectors.
2879 Known.Zero.setAllBits(); Known.One.setAllBits();
2880 EVT SubVectorVT = Op.getOperand(0).getValueType();
2881 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2882 unsigned NumSubVectors = Op.getNumOperands();
2883 for (unsigned i = 0; i != NumSubVectors; ++i) {
2884 APInt DemandedSub =
2885 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
2886 if (!!DemandedSub) {
2887 SDValue Sub = Op.getOperand(i);
2888 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2889 Known = KnownBits::commonBits(Known, Known2);
2890 }
2891 // If we don't know any bits, early out.
2892 if (Known.isUnknown())
2893 break;
2894 }
2895 break;
2896 }
2897 case ISD::INSERT_SUBVECTOR: {
2898 // Demand any elements from the subvector and the remainder from the src its
2899 // inserted into.
2900 SDValue Src = Op.getOperand(0);
2901 SDValue Sub = Op.getOperand(1);
2902 uint64_t Idx = Op.getConstantOperandVal(2);
2903 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2904 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2905 APInt DemandedSrcElts = DemandedElts;
2906 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
2907
2908 Known.One.setAllBits();
2909 Known.Zero.setAllBits();
2910 if (!!DemandedSubElts) {
2911 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2912 if (Known.isUnknown())
2913 break; // early-out.
2914 }
2915 if (!!DemandedSrcElts) {
2916 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2917 Known = KnownBits::commonBits(Known, Known2);
2918 }
2919 break;
2920 }
2921 case ISD::EXTRACT_SUBVECTOR: {
2922 // Offset the demanded elts by the subvector index.
2923 SDValue Src = Op.getOperand(0);
2924 // Bail until we can represent demanded elements for scalable vectors.
2925 if (Src.getValueType().isScalableVector())
2926 break;
2927 uint64_t Idx = Op.getConstantOperandVal(1);
2928 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2929 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2930 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2931 break;
2932 }
2933 case ISD::SCALAR_TO_VECTOR: {
2934 // We know about scalar_to_vector as much as we know about it source,
2935 // which becomes the first element of otherwise unknown vector.
2936 if (DemandedElts != 1)
2937 break;
2938
2939 SDValue N0 = Op.getOperand(0);
2940 Known = computeKnownBits(N0, Depth + 1);
2941 if (N0.getValueSizeInBits() != BitWidth)
2942 Known = Known.trunc(BitWidth);
2943
2944 break;
2945 }
2946 case ISD::BITCAST: {
2947 SDValue N0 = Op.getOperand(0);
2948 EVT SubVT = N0.getValueType();
2949 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2950
2951 // Ignore bitcasts from unsupported types.
2952 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2953 break;
2954
2955 // Fast handling of 'identity' bitcasts.
2956 if (BitWidth == SubBitWidth) {
2957 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2958 break;
2959 }
2960
2961 bool IsLE = getDataLayout().isLittleEndian();
2962
2963 // Bitcast 'small element' vector to 'large element' scalar/vector.
2964 if ((BitWidth % SubBitWidth) == 0) {
2965 assert(N0.getValueType().isVector() && "Expected bitcast from vector")(static_cast <bool> (N0.getValueType().isVector() &&
"Expected bitcast from vector") ? void (0) : __assert_fail (
"N0.getValueType().isVector() && \"Expected bitcast from vector\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2965, __extension__ __PRETTY_FUNCTION__))
;
2966
2967 // Collect known bits for the (larger) output by collecting the known
2968 // bits from each set of sub elements and shift these into place.
2969 // We need to separately call computeKnownBits for each set of
2970 // sub elements as the knownbits for each is likely to be different.
2971 unsigned SubScale = BitWidth / SubBitWidth;
2972 APInt SubDemandedElts(NumElts * SubScale, 0);
2973 for (unsigned i = 0; i != NumElts; ++i)
2974 if (DemandedElts[i])
2975 SubDemandedElts.setBit(i * SubScale);
2976
2977 for (unsigned i = 0; i != SubScale; ++i) {
2978 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2979 Depth + 1);
2980 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2981 Known.insertBits(Known2, SubBitWidth * Shifts);
2982 }
2983 }
2984
2985 // Bitcast 'large element' scalar/vector to 'small element' vector.
2986 if ((SubBitWidth % BitWidth) == 0) {
2987 assert(Op.getValueType().isVector() && "Expected bitcast to vector")(static_cast <bool> (Op.getValueType().isVector() &&
"Expected bitcast to vector") ? void (0) : __assert_fail ("Op.getValueType().isVector() && \"Expected bitcast to vector\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
;
2988
2989 // Collect known bits for the (smaller) output by collecting the known
2990 // bits from the overlapping larger input elements and extracting the
2991 // sub sections we actually care about.
2992 unsigned SubScale = SubBitWidth / BitWidth;
2993 APInt SubDemandedElts =
2994 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
2995 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2996
2997 Known.Zero.setAllBits(); Known.One.setAllBits();
2998 for (unsigned i = 0; i != NumElts; ++i)
2999 if (DemandedElts[i]) {
3000 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3001 unsigned Offset = (Shifts % SubScale) * BitWidth;
3002 Known = KnownBits::commonBits(Known,
3003 Known2.extractBits(BitWidth, Offset));
3004 // If we don't know any bits, early out.
3005 if (Known.isUnknown())
3006 break;
3007 }
3008 }
3009 break;
3010 }
3011 case ISD::AND:
3012 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3013 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3014
3015 Known &= Known2;
3016 break;
3017 case ISD::OR:
3018 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3019 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3020
3021 Known |= Known2;
3022 break;
3023 case ISD::XOR:
3024 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3025 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3026
3027 Known ^= Known2;
3028 break;
3029 case ISD::MUL: {
3030 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3031 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3032 Known = KnownBits::mul(Known, Known2);
3033 break;
3034 }
3035 case ISD::MULHU: {
3036 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3037 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3038 Known = KnownBits::mulhu(Known, Known2);
3039 break;
3040 }
3041 case ISD::MULHS: {
3042 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3043 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3044 Known = KnownBits::mulhs(Known, Known2);
3045 break;
3046 }
3047 case ISD::UMUL_LOHI: {
3048 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result")(static_cast <bool> ((Op.getResNo() == 0 || Op.getResNo
() == 1) && "Unknown result") ? void (0) : __assert_fail
("(Op.getResNo() == 0 || Op.getResNo() == 1) && \"Unknown result\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3048, __extension__ __PRETTY_FUNCTION__))
;
3049 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3050 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3051 if (Op.getResNo() == 0)
3052 Known = KnownBits::mul(Known, Known2);
3053 else
3054 Known = KnownBits::mulhu(Known, Known2);
3055 break;
3056 }
3057 case ISD::SMUL_LOHI: {
3058 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result")(static_cast <bool> ((Op.getResNo() == 0 || Op.getResNo
() == 1) && "Unknown result") ? void (0) : __assert_fail
("(Op.getResNo() == 0 || Op.getResNo() == 1) && \"Unknown result\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3058, __extension__ __PRETTY_FUNCTION__))
;
3059 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3060 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3061 if (Op.getResNo() == 0)
3062 Known = KnownBits::mul(Known, Known2);
3063 else
3064 Known = KnownBits::mulhs(Known, Known2);
3065 break;
3066 }
3067 case ISD::UDIV: {
3068 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3069 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3070 Known = KnownBits::udiv(Known, Known2);
3071 break;
3072 }
3073 case ISD::SELECT:
3074 case ISD::VSELECT:
3075 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3076 // If we don't know any bits, early out.
3077 if (Known.isUnknown())
3078 break;
3079 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3080
3081 // Only known if known in both the LHS and RHS.
3082 Known = KnownBits::commonBits(Known, Known2);
3083 break;
3084 case ISD::SELECT_CC:
3085 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3086 // If we don't know any bits, early out.
3087 if (Known.isUnknown())
3088 break;
3089 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3090
3091 // Only known if known in both the LHS and RHS.
3092 Known = KnownBits::commonBits(Known, Known2);
3093 break;
3094 case ISD::SMULO:
3095 case ISD::UMULO:
3096 if (Op.getResNo() != 1)
3097 break;
3098 // The boolean result conforms to getBooleanContents.
3099 // If we know the result of a setcc has the top bits zero, use this info.
3100 // We know that we have an integer-based boolean since these operations
3101 // are only available for integer.
3102 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3103 TargetLowering::ZeroOrOneBooleanContent &&
3104 BitWidth > 1)
3105 Known.Zero.setBitsFrom(1);
3106 break;
3107 case ISD::SETCC:
3108 case ISD::STRICT_FSETCC:
3109 case ISD::STRICT_FSETCCS: {
3110 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3111 // If we know the result of a setcc has the top bits zero, use this info.
3112 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3113 TargetLowering::ZeroOrOneBooleanContent &&
3114 BitWidth > 1)
3115 Known.Zero.setBitsFrom(1);
3116 break;
3117 }
3118 case ISD::SHL:
3119 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3120 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3121 Known = KnownBits::shl(Known, Known2);
3122
3123 // Minimum shift low bits are known zero.
3124 if (const APInt *ShMinAmt =
3125 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3126 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3127 break;
3128 case ISD::SRL:
3129 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3130 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3131 Known = KnownBits::lshr(Known, Known2);
3132
3133 // Minimum shift high bits are known zero.
3134 if (const APInt *ShMinAmt =
3135 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3136 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3137 break;
3138 case ISD::SRA:
3139 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3140 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3141 Known = KnownBits::ashr(Known, Known2);
3142 // TODO: Add minimum shift high known sign bits.
3143 break;
3144 case ISD::FSHL:
3145 case ISD::FSHR:
3146 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3147 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3148
3149 // For fshl, 0-shift returns the 1st arg.
3150 // For fshr, 0-shift returns the 2nd arg.
3151 if (Amt == 0) {
3152 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3153 DemandedElts, Depth + 1);
3154 break;
3155 }
3156
3157 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3158 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3159 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3160 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3161 if (Opcode == ISD::FSHL) {
3162 Known.One <<= Amt;
3163 Known.Zero <<= Amt;
3164 Known2.One.lshrInPlace(BitWidth - Amt);
3165 Known2.Zero.lshrInPlace(BitWidth - Amt);
3166 } else {
3167 Known.One <<= BitWidth - Amt;
3168 Known.Zero <<= BitWidth - Amt;
3169 Known2.One.lshrInPlace(Amt);
3170 Known2.Zero.lshrInPlace(Amt);
3171 }
3172 Known.One |= Known2.One;
3173 Known.Zero |= Known2.Zero;
3174 }
3175 break;
3176 case ISD::SIGN_EXTEND_INREG: {
3177 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3178 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3179 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3180 break;
3181 }
3182 case ISD::CTTZ:
3183 case ISD::CTTZ_ZERO_UNDEF: {
3184 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3185 // If we have a known 1, its position is our upper bound.
3186 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3187 unsigned LowBits = Log2_32(PossibleTZ) + 1;
3188 Known.Zero.setBitsFrom(LowBits);
3189 break;
3190 }
3191 case ISD::CTLZ:
3192 case ISD::CTLZ_ZERO_UNDEF: {
3193 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3194 // If we have a known 1, its position is our upper bound.
3195 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3196 unsigned LowBits = Log2_32(PossibleLZ) + 1;
3197 Known.Zero.setBitsFrom(LowBits);
3198 break;
3199 }
3200 case ISD::CTPOP: {
3201 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3202 // If we know some of the bits are zero, they can't be one.
3203 unsigned PossibleOnes = Known2.countMaxPopulation();
3204 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
3205 break;
3206 }
3207 case ISD::PARITY: {
3208 // Parity returns 0 everywhere but the LSB.
3209 Known.Zero.setBitsFrom(1);
3210 break;
3211 }
3212 case ISD::LOAD: {
3213 LoadSDNode *LD = cast<LoadSDNode>(Op);
3214 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3215 if (ISD::isNON_EXTLoad(LD) && Cst) {
3216 // Determine any common known bits from the loaded constant pool value.
3217 Type *CstTy = Cst->getType();
3218 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
3219 // If its a vector splat, then we can (quickly) reuse the scalar path.
3220 // NOTE: We assume all elements match and none are UNDEF.
3221 if (CstTy->isVectorTy()) {
3222 if (const Constant *Splat = Cst->getSplatValue()) {
3223 Cst = Splat;
3224 CstTy = Cst->getType();
3225 }
3226 }
3227 // TODO - do we need to handle different bitwidths?
3228 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3229 // Iterate across all vector elements finding common known bits.
3230 Known.One.setAllBits();
3231 Known.Zero.setAllBits();
3232 for (unsigned i = 0; i != NumElts; ++i) {
3233 if (!DemandedElts[i])
3234 continue;
3235 if (Constant *Elt = Cst->getAggregateElement(i)) {
3236 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3237 const APInt &Value = CInt->getValue();
3238 Known.One &= Value;
3239 Known.Zero &= ~Value;
3240 continue;
3241 }
3242 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3243 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3244 Known.One &= Value;
3245 Known.Zero &= ~Value;
3246 continue;
3247 }
3248 }
3249 Known.One.clearAllBits();
3250 Known.Zero.clearAllBits();
3251 break;
3252 }
3253 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3254 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3255 Known = KnownBits::makeConstant(CInt->getValue());
3256 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3257 Known =
3258 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3259 }
3260 }
3261 }
3262 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3263 // If this is a ZEXTLoad and we are looking at the loaded value.
3264 EVT VT = LD->getMemoryVT();
3265 unsigned MemBits = VT.getScalarSizeInBits();
3266 Known.Zero.setBitsFrom(MemBits);
3267 } else if (const MDNode *Ranges = LD->getRanges()) {
3268 if (LD->getExtensionType() == ISD::NON_EXTLOAD)
3269 computeKnownBitsFromRangeMetadata(*Ranges, Known);
3270 }
3271 break;
3272 }
3273 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3274 EVT InVT = Op.getOperand(0).getValueType();
3275 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3276 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3277 Known = Known.zext(BitWidth);
3278 break;
3279 }
3280 case ISD::ZERO_EXTEND: {
3281 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3282 Known = Known.zext(BitWidth);
3283 break;
3284 }
3285 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3286 EVT InVT = Op.getOperand(0).getValueType();
3287 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3288 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3289 // If the sign bit is known to be zero or one, then sext will extend
3290 // it to the top bits, else it will just zext.
3291 Known = Known.sext(BitWidth);
3292 break;
3293 }
3294 case ISD::SIGN_EXTEND: {
3295 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3296 // If the sign bit is known to be zero or one, then sext will extend
3297 // it to the top bits, else it will just zext.
3298 Known = Known.sext(BitWidth);
3299 break;
3300 }
3301 case ISD::ANY_EXTEND_VECTOR_INREG: {
3302 EVT InVT = Op.getOperand(0).getValueType();
3303 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3304 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3305 Known = Known.anyext(BitWidth);
3306 break;
3307 }
3308 case ISD::ANY_EXTEND: {
3309 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3310 Known = Known.anyext(BitWidth);
3311 break;
3312 }
3313 case ISD::TRUNCATE: {
3314 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3315 Known = Known.trunc(BitWidth);
3316 break;
3317 }
3318 case ISD::AssertZext: {
3319 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3320 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3321 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3322 Known.Zero |= (~InMask);
3323 Known.One &= (~Known.Zero);
3324 break;
3325 }
3326 case ISD::AssertAlign: {
3327 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3328 assert(LogOfAlign != 0)(static_cast <bool> (LogOfAlign != 0) ? void (0) : __assert_fail
("LogOfAlign != 0", "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))
;
3329 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3330 // well as clearing one bits.
3331 Known.Zero.setLowBits(LogOfAlign);
3332 Known.One.clearLowBits(LogOfAlign);
3333 break;
3334 }
3335 case ISD::FGETSIGN:
3336 // All bits are zero except the low bit.
3337 Known.Zero.setBitsFrom(1);
3338 break;
3339 case ISD::USUBO:
3340 case ISD::SSUBO:
3341 if (Op.getResNo() == 1) {
3342 // If we know the result of a setcc has the top bits zero, use this info.
3343 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3344 TargetLowering::ZeroOrOneBooleanContent &&
3345 BitWidth > 1)
3346 Known.Zero.setBitsFrom(1);
3347 break;
3348 }
3349 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3350 case ISD::SUB:
3351 case ISD::SUBC: {
3352 assert(Op.getResNo() == 0 &&(static_cast <bool> (Op.getResNo() == 0 && "We only compute knownbits for the difference here."
) ? void (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the difference here.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3353, __extension__ __PRETTY_FUNCTION__))
3353 "We only compute knownbits for the difference here.")(static_cast <bool> (Op.getResNo() == 0 && "We only compute knownbits for the difference here."
) ? void (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the difference here.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3353, __extension__ __PRETTY_FUNCTION__))
;
3354
3355 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3356 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3357 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3358 Known, Known2);
3359 break;
3360 }
3361 case ISD::UADDO:
3362 case ISD::SADDO:
3363 case ISD::ADDCARRY:
3364 if (Op.getResNo() == 1) {
3365 // If we know the result of a setcc has the top bits zero, use this info.
3366 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3367 TargetLowering::ZeroOrOneBooleanContent &&
3368 BitWidth > 1)
3369 Known.Zero.setBitsFrom(1);
3370 break;
3371 }
3372 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3373 case ISD::ADD:
3374 case ISD::ADDC:
3375 case ISD::ADDE: {
3376 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.")(static_cast <bool> (Op.getResNo() == 0 && "We only compute knownbits for the sum here."
) ? void (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the sum here.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3376, __extension__ __PRETTY_FUNCTION__))
;
3377
3378 // With ADDE and ADDCARRY, a carry bit may be added in.
3379 KnownBits Carry(1);
3380 if (Opcode == ISD::ADDE)
3381 // Can't track carry from glue, set carry to unknown.
3382 Carry.resetAll();
3383 else if (Opcode == ISD::ADDCARRY)
3384 // TODO: Compute known bits for the carry operand. Not sure if it is worth
3385 // the trouble (how often will we find a known carry bit). And I haven't
3386 // tested this very much yet, but something like this might work:
3387 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3388 // Carry = Carry.zextOrTrunc(1, false);
3389 Carry.resetAll();
3390 else
3391 Carry.setAllZero();
3392
3393 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3394 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3395 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3396 break;
3397 }
3398 case ISD::SREM: {
3399 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3400 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3401 Known = KnownBits::srem(Known, Known2);
3402 break;
3403 }
3404 case ISD::UREM: {
3405 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3406 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3407 Known = KnownBits::urem(Known, Known2);
3408 break;
3409 }
3410 case ISD::EXTRACT_ELEMENT: {
3411 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3412 const unsigned Index = Op.getConstantOperandVal(1);
3413 const unsigned EltBitWidth = Op.getValueSizeInBits();
3414
3415 // Remove low part of known bits mask
3416 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3417 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3418
3419 // Remove high part of known bit mask
3420 Known = Known.trunc(EltBitWidth);
3421 break;
3422 }
3423 case ISD::EXTRACT_VECTOR_ELT: {
3424 SDValue InVec = Op.getOperand(0);
3425 SDValue EltNo = Op.getOperand(1);
3426 EVT VecVT = InVec.getValueType();
3427 // computeKnownBits not yet implemented for scalable vectors.
3428 if (VecVT.isScalableVector())
3429 break;
3430 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3431 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3432
3433 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3434 // anything about the extended bits.
3435 if (BitWidth > EltBitWidth)
3436 Known = Known.trunc(EltBitWidth);
3437
3438 // If we know the element index, just demand that vector element, else for
3439 // an unknown element index, ignore DemandedElts and demand them all.
3440 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
3441 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3442 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3443 DemandedSrcElts =
3444 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3445
3446 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3447 if (BitWidth > EltBitWidth)
3448 Known = Known.anyext(BitWidth);
3449 break;
3450 }
3451 case ISD::INSERT_VECTOR_ELT: {
3452 // If we know the element index, split the demand between the
3453 // source vector and the inserted element, otherwise assume we need
3454 // the original demanded vector elements and the value.
3455 SDValue InVec = Op.getOperand(0);
3456 SDValue InVal = Op.getOperand(1);
3457 SDValue EltNo = Op.getOperand(2);
3458 bool DemandedVal = true;
3459 APInt DemandedVecElts = DemandedElts;
3460 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3461 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3462 unsigned EltIdx = CEltNo->getZExtValue();
3463 DemandedVal = !!DemandedElts[EltIdx];
3464 DemandedVecElts.clearBit(EltIdx);
3465 }
3466 Known.One.setAllBits();
3467 Known.Zero.setAllBits();
3468 if (DemandedVal) {
3469 Known2 = computeKnownBits(InVal, Depth + 1);
3470 Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth));
3471 }
3472 if (!!DemandedVecElts) {
3473 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3474 Known = KnownBits::commonBits(Known, Known2);
3475 }
3476 break;
3477 }
3478 case ISD::BITREVERSE: {
3479 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3480 Known = Known2.reverseBits();
3481 break;
3482 }
3483 case ISD::BSWAP: {
3484 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3485 Known = Known2.byteSwap();
3486 break;
3487 }
3488 case ISD::ABS: {
3489 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3490 Known = Known2.abs();
3491 break;
3492 }
3493 case ISD::USUBSAT: {
3494 // The result of usubsat will never be larger than the LHS.
3495 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3496 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
3497 break;
3498 }
3499 case ISD::UMIN: {
3500 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3501 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3502 Known = KnownBits::umin(Known, Known2);
3503 break;
3504 }
3505 case ISD::UMAX: {
3506 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3507 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3508 Known = KnownBits::umax(Known, Known2);
3509 break;
3510 }
3511 case ISD::SMIN:
3512 case ISD::SMAX: {
3513 // If we have a clamp pattern, we know that the number of sign bits will be
3514 // the minimum of the clamp min/max range.
3515 bool IsMax = (Opcode == ISD::SMAX);
3516 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3517 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3518 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3519 CstHigh =
3520 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3521 if (CstLow && CstHigh) {
3522 if (!IsMax)
3523 std::swap(CstLow, CstHigh);
3524
3525 const APInt &ValueLow = CstLow->getAPIntValue();
3526 const APInt &ValueHigh = CstHigh->getAPIntValue();
3527 if (ValueLow.sle(ValueHigh)) {
3528 unsigned LowSignBits = ValueLow.getNumSignBits();
3529 unsigned HighSignBits = ValueHigh.getNumSignBits();
3530 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3531 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3532 Known.One.setHighBits(MinSignBits);
3533 break;
3534 }
3535 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3536 Known.Zero.setHighBits(MinSignBits);
3537 break;
3538 }
3539 }
3540 }
3541
3542 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3543 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3544 if (IsMax)
3545 Known = KnownBits::smax(Known, Known2);
3546 else
3547 Known = KnownBits::smin(Known, Known2);
3548 break;
3549 }
3550 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
3551 if (Op.getResNo() == 1) {
3552 // The boolean result conforms to getBooleanContents.
3553 // If we know the result of a setcc has the top bits zero, use this info.
3554 // We know that we have an integer-based boolean since these operations
3555 // are only available for integer.
3556 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3557 TargetLowering::ZeroOrOneBooleanContent &&
3558 BitWidth > 1)
3559 Known.Zero.setBitsFrom(1);
3560 break;
3561 }
3562 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3563 case ISD::ATOMIC_CMP_SWAP:
3564 case ISD::ATOMIC_SWAP:
3565 case ISD::ATOMIC_LOAD_ADD:
3566 case ISD::ATOMIC_LOAD_SUB:
3567 case ISD::ATOMIC_LOAD_AND:
3568 case ISD::ATOMIC_LOAD_CLR:
3569 case ISD::ATOMIC_LOAD_OR:
3570 case ISD::ATOMIC_LOAD_XOR:
3571 case ISD::ATOMIC_LOAD_NAND:
3572 case ISD::ATOMIC_LOAD_MIN:
3573 case ISD::ATOMIC_LOAD_MAX:
3574 case ISD::ATOMIC_LOAD_UMIN:
3575 case ISD::ATOMIC_LOAD_UMAX:
3576 case ISD::ATOMIC_LOAD: {
3577 unsigned MemBits =
3578 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
3579 // If we are looking at the loaded value.
3580 if (Op.getResNo() == 0) {
3581 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
3582 Known.Zero.setBitsFrom(MemBits);
3583 }
3584 break;
3585 }
3586 case ISD::FrameIndex:
3587 case ISD::TargetFrameIndex:
3588 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
3589 Known, getMachineFunction());
3590 break;
3591
3592 default:
3593 if (Opcode < ISD::BUILTIN_OP_END)
3594 break;
3595 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3596 case ISD::INTRINSIC_WO_CHAIN:
3597 case ISD::INTRINSIC_W_CHAIN:
3598 case ISD::INTRINSIC_VOID:
3599 // Allow the target to implement this method for its nodes.
3600 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3601 break;
3602 }
3603
3604 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3604, __extension__ __PRETTY_FUNCTION__))
;
3605 return Known;
3606}
3607
3608SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3609 SDValue N1) const {
3610 // X + 0 never overflow
3611 if (isNullConstant(N1))
3612 return OFK_Never;
3613
3614 KnownBits N1Known = computeKnownBits(N1);
3615 if (N1Known.Zero.getBoolValue()) {
3616 KnownBits N0Known = computeKnownBits(N0);
3617
3618 bool overflow;
3619 (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow);
3620 if (!overflow)
3621 return OFK_Never;
3622 }
3623
3624 // mulhi + 1 never overflow
3625 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3626 (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue())
3627 return OFK_Never;
3628
3629 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3630 KnownBits N0Known = computeKnownBits(N0);
3631
3632 if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue())
3633 return OFK_Never;
3634 }
3635
3636 return OFK_Sometime;
3637}
3638
3639bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3640 EVT OpVT = Val.getValueType();
3641 unsigned BitWidth = OpVT.getScalarSizeInBits();
3642
3643 // Is the constant a known power of 2?
3644 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3645 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3646
3647 // A left-shift of a constant one will have exactly one bit set because
3648 // shifting the bit off the end is undefined.
3649 if (Val.getOpcode() == ISD::SHL) {
3650 auto *C = isConstOrConstSplat(Val.getOperand(0));
3651 if (C && C->getAPIntValue() == 1)
3652 return true;
3653 }
3654
3655 // Similarly, a logical right-shift of a constant sign-bit will have exactly
3656 // one bit set.
3657 if (Val.getOpcode() == ISD::SRL) {
3658 auto *C = isConstOrConstSplat(Val.getOperand(0));
3659 if (C && C->getAPIntValue().isSignMask())
3660 return true;
3661 }
3662
3663 // Are all operands of a build vector constant powers of two?
3664 if (Val.getOpcode() == ISD::BUILD_VECTOR)
3665 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3666 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3667 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3668 return false;
3669 }))
3670 return true;
3671
3672 // Is the operand of a splat vector a constant power of two?
3673 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
3674 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
3675 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
3676 return true;
3677
3678 // More could be done here, though the above checks are enough
3679 // to handle some common cases.
3680
3681 // Fall back to computeKnownBits to catch other known cases.
3682 KnownBits Known = computeKnownBits(Val);
3683 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3684}
3685
3686unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3687 EVT VT = Op.getValueType();
3688
3689 // TODO: Assume we don't know anything for now.
3690 if (VT.isScalableVector())
3691 return 1;
3692
3693 APInt DemandedElts = VT.isVector()
3694 ? APInt::getAllOnes(VT.getVectorNumElements())
3695 : APInt(1, 1);
3696 return ComputeNumSignBits(Op, DemandedElts, Depth);
3697}
3698
3699unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3700 unsigned Depth) const {
3701 EVT VT = Op.getValueType();
3702 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!")(static_cast <bool> ((VT.isInteger() || VT.isFloatingPoint
()) && "Invalid VT!") ? void (0) : __assert_fail ("(VT.isInteger() || VT.isFloatingPoint()) && \"Invalid VT!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3702, __extension__ __PRETTY_FUNCTION__))
;
3703 unsigned VTBits = VT.getScalarSizeInBits();
3704 unsigned NumElts = DemandedElts.getBitWidth();
3705 unsigned Tmp, Tmp2;
3706 unsigned FirstAnswer = 1;
3707
3708 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3709 const APInt &Val = C->getAPIntValue();
3710 return Val.getNumSignBits();
3711 }
3712
3713 if (Depth >= MaxRecursionDepth)
3714 return 1; // Limit search depth.
3715
3716 if (!DemandedElts || VT.isScalableVector())
3717 return 1; // No demanded elts, better to assume we don't know anything.
3718
3719 unsigned Opcode = Op.getOpcode();
3720 switch (Opcode) {
3721 default: break;
3722 case ISD::AssertSext:
3723 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3724 return VTBits-Tmp+1;
3725 case ISD::AssertZext:
3726 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3727 return VTBits-Tmp;
3728
3729 case ISD::BUILD_VECTOR:
3730 Tmp = VTBits;
3731 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3732 if (!DemandedElts[i])
3733 continue;
3734
3735 SDValue SrcOp = Op.getOperand(i);
3736 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
3737
3738 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3739 if (SrcOp.getValueSizeInBits() != VTBits) {
3740 assert(SrcOp.getValueSizeInBits() > VTBits &&(static_cast <bool> (SrcOp.getValueSizeInBits() > VTBits
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > VTBits && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3741, __extension__ __PRETTY_FUNCTION__))
3741 "Expected BUILD_VECTOR implicit truncation")(static_cast <bool> (SrcOp.getValueSizeInBits() > VTBits
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > VTBits && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3741, __extension__ __PRETTY_FUNCTION__))
;
3742 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3743 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3744 }
3745 Tmp = std::min(Tmp, Tmp2);
3746 }
3747 return Tmp;
3748
3749 case ISD::VECTOR_SHUFFLE: {
3750 // Collect the minimum number of sign bits that are shared by every vector
3751 // element referenced by the shuffle.
3752 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3753 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3754 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")(static_cast <bool> (NumElts == SVN->getMask().size(
) && "Unexpected vector size") ? void (0) : __assert_fail
("NumElts == SVN->getMask().size() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3754, __extension__ __PRETTY_FUNCTION__))
;
3755 for (unsigned i = 0; i != NumElts; ++i) {
3756 int M = SVN->getMaskElt(i);
3757 if (!DemandedElts[i])
3758 continue;
3759 // For UNDEF elements, we don't know anything about the common state of
3760 // the shuffle result.
3761 if (M < 0)
3762 return 1;
3763 if ((unsigned)M < NumElts)
3764 DemandedLHS.setBit((unsigned)M % NumElts);
3765 else
3766 DemandedRHS.setBit((unsigned)M % NumElts);
3767 }
3768 Tmp = std::numeric_limits<unsigned>::max();
3769 if (!!DemandedLHS)
3770 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3771 if (!!DemandedRHS) {
3772 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3773 Tmp = std::min(Tmp, Tmp2);
3774 }
3775 // If we don't know anything, early out and try computeKnownBits fall-back.
3776 if (Tmp == 1)
3777 break;
3778 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3778, __extension__ __PRETTY_FUNCTION__))
;
3779 return Tmp;
3780 }
3781
3782 case ISD::BITCAST: {
3783 SDValue N0 = Op.getOperand(0);
3784 EVT SrcVT = N0.getValueType();
3785 unsigned SrcBits = SrcVT.getScalarSizeInBits();
3786
3787 // Ignore bitcasts from unsupported types..
3788 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3789 break;
3790
3791 // Fast handling of 'identity' bitcasts.
3792 if (VTBits == SrcBits)
3793 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3794
3795 bool IsLE = getDataLayout().isLittleEndian();
3796
3797 // Bitcast 'large element' scalar/vector to 'small element' vector.
3798 if ((SrcBits % VTBits) == 0) {
3799 assert(VT.isVector() && "Expected bitcast to vector")(static_cast <bool> (VT.isVector() && "Expected bitcast to vector"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected bitcast to vector\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3799, __extension__ __PRETTY_FUNCTION__))
;
3800
3801 unsigned Scale = SrcBits / VTBits;
3802 APInt SrcDemandedElts =
3803 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
3804
3805 // Fast case - sign splat can be simply split across the small elements.
3806 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3807 if (Tmp == SrcBits)
3808 return VTBits;
3809
3810 // Slow case - determine how far the sign extends into each sub-element.
3811 Tmp2 = VTBits;
3812 for (unsigned i = 0; i != NumElts; ++i)
3813 if (DemandedElts[i]) {
3814 unsigned SubOffset = i % Scale;
3815 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3816 SubOffset = SubOffset * VTBits;
3817 if (Tmp <= SubOffset)
3818 return 1;
3819 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3820 }
3821 return Tmp2;
3822 }
3823 break;
3824 }
3825
3826 case ISD::SIGN_EXTEND:
3827 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3828 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3829 case ISD::SIGN_EXTEND_INREG:
3830 // Max of the input and what this extends.
3831 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3832 Tmp = VTBits-Tmp+1;
3833 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3834 return std::max(Tmp, Tmp2);
3835 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3836 SDValue Src = Op.getOperand(0);
3837 EVT SrcVT = Src.getValueType();
3838 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3839 Tmp = VTBits - SrcVT.getScalarSizeInBits();
3840 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3841 }
3842 case ISD::SRA:
3843 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3844 // SRA X, C -> adds C sign bits.
3845 if (const APInt *ShAmt =
3846 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3847 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
3848 return Tmp;
3849 case ISD::SHL:
3850 if (const APInt *ShAmt =
3851 getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
3852 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
3853 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3854 if (ShAmt->ult(Tmp))
3855 return Tmp - ShAmt->getZExtValue();
3856 }
3857 break;
3858 case ISD::AND:
3859 case ISD::OR:
3860 case ISD::XOR: // NOT is handled here.
3861 // Logical binary ops preserve the number of sign bits at the worst.
3862 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3863 if (Tmp != 1) {
3864 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3865 FirstAnswer = std::min(Tmp, Tmp2);
3866 // We computed what we know about the sign bits as our first
3867 // answer. Now proceed to the generic code that uses
3868 // computeKnownBits, and pick whichever answer is better.
3869 }
3870 break;
3871
3872 case ISD::SELECT:
3873 case ISD::VSELECT:
3874 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3875 if (Tmp == 1) return 1; // Early out.
3876 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3877 return std::min(Tmp, Tmp2);
3878 case ISD::SELECT_CC:
3879 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3880 if (Tmp == 1) return 1; // Early out.
3881 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3882 return std::min(Tmp, Tmp2);
3883
3884 case ISD::SMIN:
3885 case ISD::SMAX: {
3886 // If we have a clamp pattern, we know that the number of sign bits will be
3887 // the minimum of the clamp min/max range.
3888 bool IsMax = (Opcode == ISD::SMAX);
3889 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3890 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3891 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3892 CstHigh =
3893 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3894 if (CstLow && CstHigh) {
3895 if (!IsMax)
3896 std::swap(CstLow, CstHigh);
3897 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3898 Tmp = CstLow->getAPIntValue().getNumSignBits();
3899 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3900 return std::min(Tmp, Tmp2);
3901 }
3902 }
3903
3904 // Fallback - just get the minimum number of sign bits of the operands.
3905 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3906 if (Tmp == 1)
3907 return 1; // Early out.
3908 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3909 return std::min(Tmp, Tmp2);
3910 }
3911 case ISD::UMIN:
3912 case ISD::UMAX:
3913 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3914 if (Tmp == 1)
3915 return 1; // Early out.
3916 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3917 return std::min(Tmp, Tmp2);
3918 case ISD::SADDO:
3919 case ISD::UADDO:
3920 case ISD::SSUBO:
3921 case ISD::USUBO:
3922 case ISD::SMULO:
3923 case ISD::UMULO:
3924 if (Op.getResNo() != 1)
3925 break;
3926 // The boolean result conforms to getBooleanContents. Fall through.
3927 // If setcc returns 0/-1, all bits are sign bits.
3928 // We know that we have an integer-based boolean since these operations
3929 // are only available for integer.
3930 if (TLI->getBooleanContents(VT.isVector(), false) ==
3931 TargetLowering::ZeroOrNegativeOneBooleanContent)
3932 return VTBits;
3933 break;
3934 case ISD::SETCC:
3935 case ISD::STRICT_FSETCC:
3936 case ISD::STRICT_FSETCCS: {
3937 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3938 // If setcc returns 0/-1, all bits are sign bits.
3939 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3940 TargetLowering::ZeroOrNegativeOneBooleanContent)
3941 return VTBits;
3942 break;
3943 }
3944 case ISD::ROTL:
3945 case ISD::ROTR:
3946 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3947
3948 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
3949 if (Tmp == VTBits)
3950 return VTBits;
3951
3952 if (ConstantSDNode *C =
3953 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3954 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3955
3956 // Handle rotate right by N like a rotate left by 32-N.
3957 if (Opcode == ISD::ROTR)
3958 RotAmt = (VTBits - RotAmt) % VTBits;
3959
3960 // If we aren't rotating out all of the known-in sign bits, return the
3961 // number that are left. This handles rotl(sext(x), 1) for example.
3962 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3963 }
3964 break;
3965 case ISD::ADD:
3966 case ISD::ADDC:
3967 // Add can have at most one carry bit. Thus we know that the output
3968 // is, at worst, one more bit than the inputs.
3969 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3970 if (Tmp == 1) return 1; // Early out.
3971
3972 // Special case decrementing a value (ADD X, -1):
3973 if (ConstantSDNode *CRHS =
3974 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
3975 if (CRHS->isAllOnes()) {
3976 KnownBits Known =
3977 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3978
3979 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3980 // sign bits set.
3981 if ((Known.Zero | 1).isAllOnes())
3982 return VTBits;
3983
3984 // If we are subtracting one from a positive number, there is no carry
3985 // out of the result.
3986 if (Known.isNonNegative())
3987 return Tmp;
3988 }
3989
3990 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3991 if (Tmp2 == 1) return 1; // Early out.
3992 return std::min(Tmp, Tmp2) - 1;
3993 case ISD::SUB:
3994 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3995 if (Tmp2 == 1) return 1; // Early out.
3996
3997 // Handle NEG.
3998 if (ConstantSDNode *CLHS =
3999 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
4000 if (CLHS->isZero()) {
4001 KnownBits Known =
4002 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4003 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4004 // sign bits set.
4005 if ((Known.Zero | 1).isAllOnes())
4006 return VTBits;
4007
4008 // If the input is known to be positive (the sign bit is known clear),
4009 // the output of the NEG has the same number of sign bits as the input.
4010 if (Known.isNonNegative())
4011 return Tmp2;
4012
4013 // Otherwise, we treat this like a SUB.
4014 }
4015
4016 // Sub can have at most one carry bit. Thus we know that the output
4017 // is, at worst, one more bit than the inputs.
4018 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4019 if (Tmp == 1) return 1; // Early out.
4020 return std::min(Tmp, Tmp2) - 1;
4021 case ISD::MUL: {
4022 // The output of the Mul can be at most twice the valid bits in the inputs.
4023 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4024 if (SignBitsOp0 == 1)
4025 break;
4026 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4027 if (SignBitsOp1 == 1)
4028 break;
4029 unsigned OutValidBits =
4030 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4031 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4032 }
4033 case ISD::SREM:
4034 // The sign bit is the LHS's sign bit, except when the result of the
4035 // remainder is zero. The magnitude of the result should be less than or
4036 // equal to the magnitude of the LHS. Therefore, the result should have
4037 // at least as many sign bits as the left hand side.
4038 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4039 case ISD::TRUNCATE: {
4040 // Check if the sign bits of source go down as far as the truncated value.
4041 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4042 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4043 if (NumSrcSignBits > (NumSrcBits - VTBits))
4044 return NumSrcSignBits - (NumSrcBits - VTBits);
4045 break;
4046 }
4047 case ISD::EXTRACT_ELEMENT: {
4048 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4049 const int BitWidth = Op.getValueSizeInBits();
4050 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4051
4052 // Get reverse index (starting from 1), Op1 value indexes elements from
4053 // little end. Sign starts at big end.
4054 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4055
4056 // If the sign portion ends in our element the subtraction gives correct
4057 // result. Otherwise it gives either negative or > bitwidth result
4058 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
4059 }
4060 case ISD::INSERT_VECTOR_ELT: {
4061 // If we know the element index, split the demand between the
4062 // source vector and the inserted element, otherwise assume we need
4063 // the original demanded vector elements and the value.
4064 SDValue InVec = Op.getOperand(0);
4065 SDValue InVal = Op.getOperand(1);
4066 SDValue EltNo = Op.getOperand(2);
4067 bool DemandedVal = true;
4068 APInt DemandedVecElts = DemandedElts;
4069 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4070 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4071 unsigned EltIdx = CEltNo->getZExtValue();
4072 DemandedVal = !!DemandedElts[EltIdx];
4073 DemandedVecElts.clearBit(EltIdx);
4074 }
4075 Tmp = std::numeric_limits<unsigned>::max();
4076 if (DemandedVal) {
4077 // TODO - handle implicit truncation of inserted elements.
4078 if (InVal.getScalarValueSizeInBits() != VTBits)
4079 break;
4080 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4081 Tmp = std::min(Tmp, Tmp2);
4082 }
4083 if (!!DemandedVecElts) {
4084 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4085 Tmp = std::min(Tmp, Tmp2);
4086 }
4087 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4087, __extension__ __PRETTY_FUNCTION__))
;
4088 return Tmp;
4089 }
4090 case ISD::EXTRACT_VECTOR_ELT: {
4091 SDValue InVec = Op.getOperand(0);
4092 SDValue EltNo = Op.getOperand(1);
4093 EVT VecVT = InVec.getValueType();
4094 // ComputeNumSignBits not yet implemented for scalable vectors.
4095 if (VecVT.isScalableVector())
4096 break;
4097 const unsigned BitWidth = Op.getValueSizeInBits();
4098 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4099 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4100
4101 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4102 // anything about sign bits. But if the sizes match we can derive knowledge
4103 // about sign bits from the vector operand.
4104 if (BitWidth != EltBitWidth)
4105 break;
4106
4107 // If we know the element index, just demand that vector element, else for
4108 // an unknown element index, ignore DemandedElts and demand them all.
4109 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4110 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4111 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4112 DemandedSrcElts =
4113 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4114
4115 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4116 }
4117 case ISD::EXTRACT_SUBVECTOR: {
4118 // Offset the demanded elts by the subvector index.
4119 SDValue Src = Op.getOperand(0);
4120 // Bail until we can represent demanded elements for scalable vectors.
4121 if (Src.getValueType().isScalableVector())
4122 break;
4123 uint64_t Idx = Op.getConstantOperandVal(1);
4124 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4125 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
4126 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4127 }
4128 case ISD::CONCAT_VECTORS: {
4129 // Determine the minimum number of sign bits across all demanded
4130 // elts of the input vectors. Early out if the result is already 1.
4131 Tmp = std::numeric_limits<unsigned>::max();
4132 EVT SubVectorVT = Op.getOperand(0).getValueType();
4133 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4134 unsigned NumSubVectors = Op.getNumOperands();
4135 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4136 APInt DemandedSub =
4137 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4138 if (!DemandedSub)
4139 continue;
4140 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4141 Tmp = std::min(Tmp, Tmp2);
4142 }
4143 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4143, __extension__ __PRETTY_FUNCTION__))
;
4144 return Tmp;
4145 }
4146 case ISD::INSERT_SUBVECTOR: {
4147 // Demand any elements from the subvector and the remainder from the src its
4148 // inserted into.
4149 SDValue Src = Op.getOperand(0);
4150 SDValue Sub = Op.getOperand(1);
4151 uint64_t Idx = Op.getConstantOperandVal(2);
4152 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4153 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4154 APInt DemandedSrcElts = DemandedElts;
4155 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
4156
4157 Tmp = std::numeric_limits<unsigned>::max();
4158 if (!!DemandedSubElts) {
4159 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4160 if (Tmp == 1)
4161 return 1; // early-out
4162 }
4163 if (!!DemandedSrcElts) {
4164 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4165 Tmp = std::min(Tmp, Tmp2);
4166 }
4167 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4167, __extension__ __PRETTY_FUNCTION__))
;
4168 return Tmp;
4169 }
4170 case ISD::ATOMIC_CMP_SWAP:
4171 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4172 case ISD::ATOMIC_SWAP:
4173 case ISD::ATOMIC_LOAD_ADD:
4174 case ISD::ATOMIC_LOAD_SUB:
4175 case ISD::ATOMIC_LOAD_AND:
4176 case ISD::ATOMIC_LOAD_CLR:
4177 case ISD::ATOMIC_LOAD_OR:
4178 case ISD::ATOMIC_LOAD_XOR:
4179 case ISD::ATOMIC_LOAD_NAND:
4180 case ISD::ATOMIC_LOAD_MIN:
4181 case ISD::ATOMIC_LOAD_MAX:
4182 case ISD::ATOMIC_LOAD_UMIN:
4183 case ISD::ATOMIC_LOAD_UMAX:
4184 case ISD::ATOMIC_LOAD: {
4185 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4186 // If we are looking at the loaded value.
4187 if (Op.getResNo() == 0) {
4188 if (Tmp == VTBits)
4189 return 1; // early-out
4190 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4191 return VTBits - Tmp + 1;
4192 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4193 return VTBits - Tmp;
4194 }
4195 break;
4196 }
4197 }
4198
4199 // If we are looking at the loaded value of the SDNode.
4200 if (Op.getResNo() == 0) {
4201 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4202 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4203 unsigned ExtType = LD->getExtensionType();
4204 switch (ExtType) {
4205 default: break;
4206 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4207 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4208 return VTBits - Tmp + 1;
4209 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4210 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4211 return VTBits - Tmp;
4212 case ISD::NON_EXTLOAD:
4213 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4214 // We only need to handle vectors - computeKnownBits should handle
4215 // scalar cases.
4216 Type *CstTy = Cst->getType();
4217 if (CstTy->isVectorTy() &&
4218 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
4219 Tmp = VTBits;
4220 for (unsigned i = 0; i != NumElts; ++i) {
4221 if (!DemandedElts[i])
4222 continue;
4223 if (Constant *Elt = Cst->getAggregateElement(i)) {
4224 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4225 const APInt &Value = CInt->getValue();
4226 Tmp = std::min(Tmp, Value.getNumSignBits());
4227 continue;
4228 }
4229 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4230 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4231 Tmp = std::min(Tmp, Value.getNumSignBits());
4232 continue;
4233 }
4234 }
4235 // Unknown type. Conservatively assume no bits match sign bit.
4236 return 1;
4237 }
4238 return Tmp;
4239 }
4240 }
4241 break;
4242 }
4243 }
4244 }
4245
4246 // Allow the target to implement this method for its nodes.
4247 if (Opcode >= ISD::BUILTIN_OP_END ||
4248 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4249 Opcode == ISD::INTRINSIC_W_CHAIN ||
4250 Opcode == ISD::INTRINSIC_VOID) {
4251 unsigned NumBits =
4252 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4253 if (NumBits > 1)
4254 FirstAnswer = std::max(FirstAnswer, NumBits);
4255 }
4256
4257 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4258 // use this information.
4259 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4260
4261 APInt Mask;
4262 if (Known.isNonNegative()) { // sign bit is 0
4263 Mask = Known.Zero;
4264 } else if (Known.isNegative()) { // sign bit is 1;
4265 Mask = Known.One;
4266 } else {
4267 // Nothing known.
4268 return FirstAnswer;
4269 }
4270
4271 // Okay, we know that the sign bit in Mask is set. Use CLO to determine
4272 // the number of identical bits in the top of the input value.
4273 Mask <<= Mask.getBitWidth()-VTBits;
4274 return std::max(FirstAnswer, Mask.countLeadingOnes());
4275}
4276
4277bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
4278 unsigned Depth) const {
4279 // Early out for FREEZE.
4280 if (Op.getOpcode() == ISD::FREEZE)
4281 return true;
4282
4283 // TODO: Assume we don't know anything for now.
4284 EVT VT = Op.getValueType();
4285 if (VT.isScalableVector())
4286 return false;
4287
4288 APInt DemandedElts = VT.isVector()
4289 ? APInt::getAllOnes(VT.getVectorNumElements())
4290 : APInt(1, 1);
4291 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4292}
4293
4294bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
4295 const APInt &DemandedElts,
4296 bool PoisonOnly,
4297 unsigned Depth) const {
4298 unsigned Opcode = Op.getOpcode();
4299
4300 // Early out for FREEZE.
4301 if (Opcode == ISD::FREEZE)
4302 return true;
4303
4304 if (Depth >= MaxRecursionDepth)
4305 return false; // Limit search depth.
4306
4307 if (isIntOrFPConstant(Op))
4308 return true;
4309
4310 switch (Opcode) {
4311 case ISD::UNDEF:
4312 return PoisonOnly;
4313
4314 case ISD::BUILD_VECTOR:
4315 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
4316 // this shouldn't affect the result.
4317 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
4318 if (!DemandedElts[i])
4319 continue;
4320 if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly,
4321 Depth + 1))
4322 return false;
4323 }
4324 return true;
4325
4326 // TODO: Search for noundef attributes from library functions.
4327
4328 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
4329
4330 default:
4331 // Allow the target to implement this method for its nodes.
4332 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
4333 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
4334 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4335 Op, DemandedElts, *this, PoisonOnly, Depth);
4336 break;
4337 }
4338
4339 return false;
4340}
4341
4342bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
4343 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
4344 !isa<ConstantSDNode>(Op.getOperand(1)))
4345 return false;
4346
4347 if (Op.getOpcode() == ISD::OR &&
4348 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
4349 return false;
4350
4351 return true;
4352}
4353
4354bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
4355 // If we're told that NaNs won't happen, assume they won't.
4356 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
4357 return true;
4358
4359 if (Depth >= MaxRecursionDepth)
4360 return false; // Limit search depth.
4361
4362 // TODO: Handle vectors.
4363 // If the value is a constant, we can obviously see if it is a NaN or not.
4364 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
4365 return !C->getValueAPF().isNaN() ||
4366 (SNaN && !C->getValueAPF().isSignaling());
4367 }
4368
4369 unsigned Opcode = Op.getOpcode();
4370 switch (Opcode) {
4371 case ISD::FADD:
4372 case ISD::FSUB:
4373 case ISD::FMUL:
4374 case ISD::FDIV:
4375 case ISD::FREM:
4376 case ISD::FSIN:
4377 case ISD::FCOS: {
4378 if (SNaN)
4379 return true;
4380 // TODO: Need isKnownNeverInfinity
4381 return false;
4382 }
4383 case ISD::FCANONICALIZE:
4384 case ISD::FEXP:
4385 case ISD::FEXP2:
4386 case ISD::FTRUNC:
4387 case ISD::FFLOOR:
4388 case ISD::FCEIL:
4389 case ISD::FROUND:
4390 case ISD::FROUNDEVEN:
4391 case ISD::FRINT:
4392 case ISD::FNEARBYINT: {
4393 if (SNaN)
4394 return true;
4395 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4396 }
4397 case ISD::FABS:
4398 case ISD::FNEG:
4399 case ISD::FCOPYSIGN: {
4400 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4401 }
4402 case ISD::SELECT:
4403 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4404 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4405 case ISD::FP_EXTEND:
4406 case ISD::FP_ROUND: {
4407 if (SNaN)
4408 return true;
4409 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4410 }
4411 case ISD::SINT_TO_FP:
4412 case ISD::UINT_TO_FP:
4413 return true;
4414 case ISD::FMA:
4415 case ISD::FMAD: {
4416 if (SNaN)
4417 return true;
4418 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4419 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4420 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4421 }
4422 case ISD::FSQRT: // Need is known positive
4423 case ISD::FLOG:
4424 case ISD::FLOG2:
4425 case ISD::FLOG10:
4426 case ISD::FPOWI:
4427 case ISD::FPOW: {
4428 if (SNaN)
4429 return true;
4430 // TODO: Refine on operand
4431 return false;
4432 }
4433 case ISD::FMINNUM:
4434 case ISD::FMAXNUM: {
4435 // Only one needs to be known not-nan, since it will be returned if the
4436 // other ends up being one.
4437 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4438 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4439 }
4440 case ISD::FMINNUM_IEEE:
4441 case ISD::FMAXNUM_IEEE: {
4442 if (SNaN)
4443 return true;
4444 // This can return a NaN if either operand is an sNaN, or if both operands
4445 // are NaN.
4446 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4447 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4448 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4449 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4450 }
4451 case ISD::FMINIMUM:
4452 case ISD::FMAXIMUM: {
4453 // TODO: Does this quiet or return the origina NaN as-is?
4454 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4455 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4456 }
4457 case ISD::EXTRACT_VECTOR_ELT: {
4458 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4459 }
4460 default:
4461 if (Opcode >= ISD::BUILTIN_OP_END ||
4462 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4463 Opcode == ISD::INTRINSIC_W_CHAIN ||
4464 Opcode == ISD::INTRINSIC_VOID) {
4465 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4466 }
4467
4468 return false;
4469 }
4470}
4471
4472bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4473 assert(Op.getValueType().isFloatingPoint() &&(static_cast <bool> (Op.getValueType().isFloatingPoint(
) && "Floating point type expected") ? void (0) : __assert_fail
("Op.getValueType().isFloatingPoint() && \"Floating point type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4474, __extension__ __PRETTY_FUNCTION__))
4474 "Floating point type expected")(static_cast <bool> (Op.getValueType().isFloatingPoint(
) && "Floating point type expected") ? void (0) : __assert_fail
("Op.getValueType().isFloatingPoint() && \"Floating point type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4474, __extension__ __PRETTY_FUNCTION__))
;
4475
4476 // If the value is a constant, we can obviously see if it is a zero or not.
4477 // TODO: Add BuildVector support.
4478 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4479 return !C->isZero();
4480 return false;
4481}
4482
4483bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4484 assert(!Op.getValueType().isFloatingPoint() &&(static_cast <bool> (!Op.getValueType().isFloatingPoint
() && "Floating point types unsupported - use isKnownNeverZeroFloat"
) ? void (0) : __assert_fail ("!Op.getValueType().isFloatingPoint() && \"Floating point types unsupported - use isKnownNeverZeroFloat\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4485, __extension__ __PRETTY_FUNCTION__))
4485 "Floating point types unsupported - use isKnownNeverZeroFloat")(static_cast <bool> (!Op.getValueType().isFloatingPoint
() && "Floating point types unsupported - use isKnownNeverZeroFloat"
) ? void (0) : __assert_fail ("!Op.getValueType().isFloatingPoint() && \"Floating point types unsupported - use isKnownNeverZeroFloat\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4485, __extension__ __PRETTY_FUNCTION__))
;
4486
4487 // If the value is a constant, we can obviously see if it is a zero or not.
4488 if (ISD::matchUnaryPredicate(Op,
4489 [](ConstantSDNode *C) { return !C->isZero(); }))
4490 return true;
4491
4492 // TODO: Recognize more cases here.
4493 switch (Op.getOpcode()) {
4494 default: break;
4495 case ISD::OR:
4496 if (isKnownNeverZero(Op.getOperand(1)) ||
4497 isKnownNeverZero(Op.getOperand(0)))
4498 return true;
4499 break;
4500 }
4501
4502 return false;
4503}
4504
4505bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4506 // Check the obvious case.
4507 if (A == B) return true;
4508
4509 // For for negative and positive zero.
4510 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4511 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4512 if (CA->isZero() && CB->isZero()) return true;
4513
4514 // Otherwise they may not be equal.
4515 return false;
4516}
4517
4518// FIXME: unify with llvm::haveNoCommonBitsSet.
4519// FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4520bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4521 assert(A.getValueType() == B.getValueType() &&(static_cast <bool> (A.getValueType() == B.getValueType
() && "Values must have the same type") ? void (0) : __assert_fail
("A.getValueType() == B.getValueType() && \"Values must have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4522, __extension__ __PRETTY_FUNCTION__))
4522 "Values must have the same type")(static_cast <bool> (A.getValueType() == B.getValueType
() && "Values must have the same type") ? void (0) : __assert_fail
("A.getValueType() == B.getValueType() && \"Values must have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4522, __extension__ __PRETTY_FUNCTION__))
;
4523 return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),
4524 computeKnownBits(B));
4525}
4526
4527static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
4528 SelectionDAG &DAG) {
4529 if (cast<ConstantSDNode>(Step)->isZero())
4530 return DAG.getConstant(0, DL, VT);
4531
4532 return SDValue();
4533}
4534
4535static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4536 ArrayRef<SDValue> Ops,
4537 SelectionDAG &DAG) {
4538 int NumOps = Ops.size();
4539 assert(NumOps != 0 && "Can't build an empty vector!")(static_cast <bool> (NumOps != 0 && "Can't build an empty vector!"
) ? void (0) : __assert_fail ("NumOps != 0 && \"Can't build an empty vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4539, __extension__ __PRETTY_FUNCTION__))
;
4540 assert(!VT.isScalableVector() &&(static_cast <bool> (!VT.isScalableVector() && "BUILD_VECTOR cannot be used with scalable types"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"BUILD_VECTOR cannot be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4541, __extension__ __PRETTY_FUNCTION__))
4541 "BUILD_VECTOR cannot be used with scalable types")(static_cast <bool> (!VT.isScalableVector() && "BUILD_VECTOR cannot be used with scalable types"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"BUILD_VECTOR cannot be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4541, __extension__ __PRETTY_FUNCTION__))
;
4542 assert(VT.getVectorNumElements() == (unsigned)NumOps &&(static_cast <bool> (VT.getVectorNumElements() == (unsigned
)NumOps && "Incorrect element count in BUILD_VECTOR!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)NumOps && \"Incorrect element count in BUILD_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4543, __extension__ __PRETTY_FUNCTION__))
4543 "Incorrect element count in BUILD_VECTOR!")(static_cast <bool> (VT.getVectorNumElements() == (unsigned
)NumOps && "Incorrect element count in BUILD_VECTOR!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)NumOps && \"Incorrect element count in BUILD_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4543, __extension__ __PRETTY_FUNCTION__))
;
4544
4545 // BUILD_VECTOR of UNDEFs is UNDEF.
4546 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4547 return DAG.getUNDEF(VT);
4548
4549 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4550 SDValue IdentitySrc;
4551 bool IsIdentity = true;
4552 for (int i = 0; i != NumOps; ++i) {
4553 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4554 Ops[i].getOperand(0).getValueType() != VT ||
4555 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
4556 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
4557 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) {
4558 IsIdentity = false;
4559 break;
4560 }
4561 IdentitySrc = Ops[i].getOperand(0);
4562 }
4563 if (IsIdentity)
4564 return IdentitySrc;
4565
4566 return SDValue();
4567}
4568
4569/// Try to simplify vector concatenation to an input value, undef, or build
4570/// vector.
4571static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4572 ArrayRef<SDValue> Ops,
4573 SelectionDAG &DAG) {
4574 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!")(static_cast <bool> (!Ops.empty() && "Can't concatenate an empty list of vectors!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Can't concatenate an empty list of vectors!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4574, __extension__ __PRETTY_FUNCTION__))
;
4575 assert(llvm::all_of(Ops,(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4579, __extension__ __PRETTY_FUNCTION__))
4576 [Ops](SDValue Op) {(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4579, __extension__ __PRETTY_FUNCTION__))
4577 return Ops[0].getValueType() == Op.getValueType();(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4579, __extension__ __PRETTY_FUNCTION__))
4578 }) &&(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4579, __extension__ __PRETTY_FUNCTION__))
4579 "Concatenation of vectors with inconsistent value types!")(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4579, __extension__ __PRETTY_FUNCTION__))
;
4580 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==(static_cast <bool> ((Ops[0].getValueType().getVectorElementCount
() * Ops.size()) == VT.getVectorElementCount() && "Incorrect element count in vector concatenation!"
) ? void (0) : __assert_fail ("(Ops[0].getValueType().getVectorElementCount() * Ops.size()) == VT.getVectorElementCount() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4582, __extension__ __PRETTY_FUNCTION__))
4581 VT.getVectorElementCount() &&(static_cast <bool> ((Ops[0].getValueType().getVectorElementCount
() * Ops.size()) == VT.getVectorElementCount() && "Incorrect element count in vector concatenation!"
) ? void (0) : __assert_fail ("(Ops[0].getValueType().getVectorElementCount() * Ops.size()) == VT.getVectorElementCount() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4582, __extension__ __PRETTY_FUNCTION__))
4582 "Incorrect element count in vector concatenation!")(static_cast <bool> ((Ops[0].getValueType().getVectorElementCount
() * Ops.size()) == VT.getVectorElementCount() && "Incorrect element count in vector concatenation!"
) ? void (0) : __assert_fail ("(Ops[0].getValueType().getVectorElementCount() * Ops.size()) == VT.getVectorElementCount() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4582, __extension__ __PRETTY_FUNCTION__))
;
4583
4584 if (Ops.size() == 1)
4585 return Ops[0];
4586
4587 // Concat of UNDEFs is UNDEF.
4588 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4589 return DAG.getUNDEF(VT);
4590
4591 // Scan the operands and look for extract operations from a single source
4592 // that correspond to insertion at the same location via this concatenation:
4593 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4594 SDValue IdentitySrc;
4595 bool IsIdentity = true;
4596 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
4597 SDValue Op = Ops[i];
4598 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
4599 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4600 Op.getOperand(0).getValueType() != VT ||
4601 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
4602 Op.getConstantOperandVal(1) != IdentityIndex) {
4603 IsIdentity = false;
4604 break;
4605 }
4606 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&(static_cast <bool> ((!IdentitySrc || IdentitySrc == Op
.getOperand(0)) && "Unexpected identity source vector for concat of extracts"
) ? void (0) : __assert_fail ("(!IdentitySrc || IdentitySrc == Op.getOperand(0)) && \"Unexpected identity source vector for concat of extracts\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4607, __extension__ __PRETTY_FUNCTION__))
4607 "Unexpected identity source vector for concat of extracts")(static_cast <bool> ((!IdentitySrc || IdentitySrc == Op
.getOperand(0)) && "Unexpected identity source vector for concat of extracts"
) ? void (0) : __assert_fail ("(!IdentitySrc || IdentitySrc == Op.getOperand(0)) && \"Unexpected identity source vector for concat of extracts\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4607, __extension__ __PRETTY_FUNCTION__))
;
4608 IdentitySrc = Op.getOperand(0);
4609 }
4610 if (IsIdentity) {
4611 assert(IdentitySrc && "Failed to set source vector of extracts")(static_cast <bool> (IdentitySrc && "Failed to set source vector of extracts"
) ? void (0) : __assert_fail ("IdentitySrc && \"Failed to set source vector of extracts\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4611, __extension__ __PRETTY_FUNCTION__))
;
4612 return IdentitySrc;
4613 }
4614
4615 // The code below this point is only designed to work for fixed width
4616 // vectors, so we bail out for now.
4617 if (VT.isScalableVector())
4618 return SDValue();
4619
4620 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4621 // simplified to one big BUILD_VECTOR.
4622 // FIXME: Add support for SCALAR_TO_VECTOR as well.
4623 EVT SVT = VT.getScalarType();
4624 SmallVector<SDValue, 16> Elts;
4625 for (SDValue Op : Ops) {
4626 EVT OpVT = Op.getValueType();
4627 if (Op.isUndef())
4628 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4629 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4630 Elts.append(Op->op_begin(), Op->op_end());
4631 else
4632 return SDValue();
4633 }
4634
4635 // BUILD_VECTOR requires all inputs to be of the same type, find the
4636 // maximum type and extend them all.
4637 for (SDValue Op : Elts)
4638 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
4639
4640 if (SVT.bitsGT(VT.getScalarType())) {
4641 for (SDValue &Op : Elts) {
4642 if (Op.isUndef())
4643 Op = DAG.getUNDEF(SVT);
4644 else
4645 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4646 ? DAG.getZExtOrTrunc(Op, DL, SVT)
4647 : DAG.getSExtOrTrunc(Op, DL, SVT);
4648 }
4649 }
4650
4651 SDValue V = DAG.getBuildVector(VT, DL, Elts);
4652 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4653 return V;
4654}
4655
4656/// Gets or creates the specified node.
4657SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4658 FoldingSetNodeID ID;
4659 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4660 void *IP = nullptr;
4661 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4662 return SDValue(E, 0);
4663
4664 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4665 getVTList(VT));
4666 CSEMap.InsertNode(N, IP);
4667
4668 InsertNode(N);
4669 SDValue V = SDValue(N, 0);
4670 NewSDValueDbgMsg(V, "Creating new node: ", this);
4671 return V;
4672}
4673
4674SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4675 SDValue Operand) {
4676 SDNodeFlags Flags;
4677 if (Inserter)
4678 Flags = Inserter->getFlags();
4679 return getNode(Opcode, DL, VT, Operand, Flags);
4680}
4681
4682SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4683 SDValue Operand, const SDNodeFlags Flags) {
4684 assert(Operand.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (Operand.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Operand.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4685, __extension__ __PRETTY_FUNCTION__))
4685 "Operand is DELETED_NODE!")(static_cast <bool> (Operand.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Operand.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4685, __extension__ __PRETTY_FUNCTION__))
;
4686 // Constant fold unary operations with an integer constant operand. Even
4687 // opaque constant will be folded, because the folding of unary operations
4688 // doesn't create new constants with different values. Nevertheless, the
4689 // opaque flag is preserved during folding to prevent future folding with
4690 // other constants.
4691 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4692 const APInt &Val = C->getAPIntValue();
4693 switch (Opcode) {
4694 default: break;
4695 case ISD::SIGN_EXTEND:
4696 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4697 C->isTargetOpcode(), C->isOpaque());
4698 case ISD::TRUNCATE:
4699 if (C->isOpaque())
4700 break;
4701 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4702 case ISD::ZERO_EXTEND:
4703 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4704 C->isTargetOpcode(), C->isOpaque());
4705 case ISD::ANY_EXTEND:
4706 // Some targets like RISCV prefer to sign extend some types.
4707 if (TLI->isSExtCheaperThanZExt(Operand.getValueType(), VT))
4708 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4709 C->isTargetOpcode(), C->isOpaque());
4710 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4711 C->isTargetOpcode(), C->isOpaque());
4712 case ISD::UINT_TO_FP:
4713 case ISD::SINT_TO_FP: {
4714 APFloat apf(EVTToAPFloatSemantics(VT),
4715 APInt::getZero(VT.getSizeInBits()));
4716 (void)apf.convertFromAPInt(Val,
4717 Opcode==ISD::SINT_TO_FP,
4718 APFloat::rmNearestTiesToEven);
4719 return getConstantFP(apf, DL, VT);
4720 }
4721 case ISD::BITCAST:
4722 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
4723 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4724 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
4725 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4726 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
4727 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4728 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
4729 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4730 break;
4731 case ISD::ABS:
4732 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4733 C->isOpaque());
4734 case ISD::BITREVERSE:
4735 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4736 C->isOpaque());
4737 case ISD::BSWAP:
4738 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4739 C->isOpaque());
4740 case ISD::CTPOP:
4741 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4742 C->isOpaque());
4743 case ISD::CTLZ:
4744 case ISD::CTLZ_ZERO_UNDEF:
4745 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4746 C->isOpaque());
4747 case ISD::CTTZ:
4748 case ISD::CTTZ_ZERO_UNDEF:
4749 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4750 C->isOpaque());
4751 case ISD::FP16_TO_FP: {
4752 bool Ignored;
4753 APFloat FPV(APFloat::IEEEhalf(),
4754 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
4755
4756 // This can return overflow, underflow, or inexact; we don't care.
4757 // FIXME need to be more flexible about rounding mode.
4758 (void)FPV.convert(EVTToAPFloatSemantics(VT),
4759 APFloat::rmNearestTiesToEven, &Ignored);
4760 return getConstantFP(FPV, DL, VT);
4761 }
4762 case ISD::STEP_VECTOR: {
4763 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Operand, *this))
4764 return V;
4765 break;
4766 }
4767 }
4768 }
4769
4770 // Constant fold unary operations with a floating point constant operand.
4771 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4772 APFloat V = C->getValueAPF(); // make copy
4773 switch (Opcode) {
4774 case ISD::FNEG:
4775 V.changeSign();
4776 return getConstantFP(V, DL, VT);
4777 case ISD::FABS:
4778 V.clearSign();
4779 return getConstantFP(V, DL, VT);
4780 case ISD::FCEIL: {
4781 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4782 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4783 return getConstantFP(V, DL, VT);
4784 break;
4785 }
4786 case ISD::FTRUNC: {
4787 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4788 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4789 return getConstantFP(V, DL, VT);
4790 break;
4791 }
4792 case ISD::FFLOOR: {
4793 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4794 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4795 return getConstantFP(V, DL, VT);
4796 break;
4797 }
4798 case ISD::FP_EXTEND: {
4799 bool ignored;
4800 // This can return overflow, underflow, or inexact; we don't care.
4801 // FIXME need to be more flexible about rounding mode.
4802 (void)V.convert(EVTToAPFloatSemantics(VT),
4803 APFloat::rmNearestTiesToEven, &ignored);
4804 return getConstantFP(V, DL, VT);
4805 }
4806 case ISD::FP_TO_SINT:
4807 case ISD::FP_TO_UINT: {
4808 bool ignored;
4809 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4810 // FIXME need to be more flexible about rounding mode.
4811 APFloat::opStatus s =
4812 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4813 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4814 break;
4815 return getConstant(IntVal, DL, VT);
4816 }
4817 case ISD::BITCAST:
4818 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
4819 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4820 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
4821 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4822 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
4823 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4824 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
4825 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4826 break;
4827 case ISD::FP_TO_FP16: {
4828 bool Ignored;
4829 // This can return overflow, underflow, or inexact; we don't care.
4830 // FIXME need to be more flexible about rounding mode.
4831 (void)V.convert(APFloat::IEEEhalf(),
4832 APFloat::rmNearestTiesToEven, &Ignored);
4833 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4834 }
4835 }
4836 }
4837
4838 // Constant fold unary operations with a vector integer or float operand.
4839 switch (Opcode) {
4840 default:
4841 // FIXME: Entirely reasonable to perform folding of other unary
4842 // operations here as the need arises.
4843 break;
4844 case ISD::FNEG:
4845 case ISD::FABS:
4846 case ISD::FCEIL:
4847 case ISD::FTRUNC:
4848 case ISD::FFLOOR:
4849 case ISD::FP_EXTEND:
4850 case ISD::FP_TO_SINT:
4851 case ISD::FP_TO_UINT:
4852 case ISD::TRUNCATE:
4853 case ISD::ANY_EXTEND:
4854 case ISD::ZERO_EXTEND:
4855 case ISD::SIGN_EXTEND:
4856 case ISD::UINT_TO_FP:
4857 case ISD::SINT_TO_FP:
4858 case ISD::ABS:
4859 case ISD::BITREVERSE:
4860 case ISD::BSWAP:
4861 case ISD::CTLZ:
4862 case ISD::CTLZ_ZERO_UNDEF:
4863 case ISD::CTTZ:
4864 case ISD::CTTZ_ZERO_UNDEF:
4865 case ISD::CTPOP: {
4866 SDValue Ops = {Operand};
4867 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4868 return Fold;
4869 }
4870 }
4871
4872 unsigned OpOpcode = Operand.getNode()->getOpcode();
4873 switch (Opcode) {
4874 case ISD::STEP_VECTOR:
4875 assert(VT.isScalableVector() &&(static_cast <bool> (VT.isScalableVector() && "STEP_VECTOR can only be used with scalable types"
) ? void (0) : __assert_fail ("VT.isScalableVector() && \"STEP_VECTOR can only be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4876, __extension__ __PRETTY_FUNCTION__))
4876 "STEP_VECTOR can only be used with scalable types")(static_cast <bool> (VT.isScalableVector() && "STEP_VECTOR can only be used with scalable types"
) ? void (0) : __assert_fail ("VT.isScalableVector() && \"STEP_VECTOR can only be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4876, __extension__ __PRETTY_FUNCTION__))
;
4877 assert(OpOpcode == ISD::TargetConstant &&(static_cast <bool> (OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand") ? void (0) : __assert_fail ("OpOpcode == ISD::TargetConstant && VT.getVectorElementType() == Operand.getValueType() && \"Unexpected step operand\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4879, __extension__ __PRETTY_FUNCTION__))
4878 VT.getVectorElementType() == Operand.getValueType() &&(static_cast <bool> (OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand") ? void (0) : __assert_fail ("OpOpcode == ISD::TargetConstant && VT.getVectorElementType() == Operand.getValueType() && \"Unexpected step operand\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4879, __extension__ __PRETTY_FUNCTION__))
4879 "Unexpected step operand")(static_cast <bool> (OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand") ? void (0) : __assert_fail ("OpOpcode == ISD::TargetConstant && VT.getVectorElementType() == Operand.getValueType() && \"Unexpected step operand\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4879, __extension__ __PRETTY_FUNCTION__))
;
4880 break;
4881 case ISD::FREEZE:
4882 assert(VT == Operand.getValueType() && "Unexpected VT!")(static_cast <bool> (VT == Operand.getValueType() &&
"Unexpected VT!") ? void (0) : __assert_fail ("VT == Operand.getValueType() && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4882, __extension__ __PRETTY_FUNCTION__))
;
4883 break;
4884 case ISD::TokenFactor:
4885 case ISD::MERGE_VALUES:
4886 case ISD::CONCAT_VECTORS:
4887 return Operand; // Factor, merge or concat of one node? No need.
4888 case ISD::BUILD_VECTOR: {
4889 // Attempt to simplify BUILD_VECTOR.
4890 SDValue Ops[] = {Operand};
4891 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4892 return V;
4893 break;
4894 }
4895 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node")::llvm::llvm_unreachable_internal("Invalid method to make FP_ROUND node"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4895)
;
4896 case ISD::FP_EXTEND:
4897 assert(VT.isFloatingPoint() &&(static_cast <bool> (VT.isFloatingPoint() && Operand
.getValueType().isFloatingPoint() && "Invalid FP cast!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4898, __extension__ __PRETTY_FUNCTION__))
4898 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!")(static_cast <bool> (VT.isFloatingPoint() && Operand
.getValueType().isFloatingPoint() && "Invalid FP cast!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4898, __extension__ __PRETTY_FUNCTION__))
;
4899 if (Operand.getValueType() == VT) return Operand; // noop conversion.
4900 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4903, __extension__ __PRETTY_FUNCTION__))
4901 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4903, __extension__ __PRETTY_FUNCTION__))
4902 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4903, __extension__ __PRETTY_FUNCTION__))
4903 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4903, __extension__ __PRETTY_FUNCTION__))
;
4904 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid fpext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid fpext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4905, __extension__ __PRETTY_FUNCTION__))
4905 "Invalid fpext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid fpext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid fpext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4905, __extension__ __PRETTY_FUNCTION__))
;
4906 if (Operand.isUndef())
4907 return getUNDEF(VT);
4908 break;
4909 case ISD::FP_TO_SINT:
4910 case ISD::FP_TO_UINT:
4911 if (Operand.isUndef())
4912 return getUNDEF(VT);
4913 break;
4914 case ISD::SINT_TO_FP:
4915 case ISD::UINT_TO_FP:
4916 // [us]itofp(undef) = 0, because the result value is bounded.
4917 if (Operand.isUndef())
4918 return getConstantFP(0.0, DL, VT);
4919 break;
4920 case ISD::SIGN_EXTEND:
4921 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid SIGN_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid SIGN_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4922, __extension__ __PRETTY_FUNCTION__))
4922 "Invalid SIGN_EXTEND!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid SIGN_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid SIGN_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4922, __extension__ __PRETTY_FUNCTION__))
;
4923 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4925, __extension__ __PRETTY_FUNCTION__))
4924 "SIGN_EXTEND result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4925, __extension__ __PRETTY_FUNCTION__))
4925 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4925, __extension__ __PRETTY_FUNCTION__))
;
4926 if (Operand.getValueType() == VT) return Operand; // noop extension
4927 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4930, __extension__ __PRETTY_FUNCTION__))
4928 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4930, __extension__ __PRETTY_FUNCTION__))
4929 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4930, __extension__ __PRETTY_FUNCTION__))
4930 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4930, __extension__ __PRETTY_FUNCTION__))
;
4931 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid sext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid sext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4932, __extension__ __PRETTY_FUNCTION__))
4932 "Invalid sext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid sext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid sext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4932, __extension__ __PRETTY_FUNCTION__))
;
4933 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4934 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4935 if (OpOpcode == ISD::UNDEF)
4936 // sext(undef) = 0, because the top bits will all be the same.
4937 return getConstant(0, DL, VT);
4938 break;
4939 case ISD::ZERO_EXTEND:
4940 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ZERO_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ZERO_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4941, __extension__ __PRETTY_FUNCTION__))
4941 "Invalid ZERO_EXTEND!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ZERO_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ZERO_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4941, __extension__ __PRETTY_FUNCTION__))
;
4942 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4944, __extension__ __PRETTY_FUNCTION__))
4943 "ZERO_EXTEND result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4944, __extension__ __PRETTY_FUNCTION__))
4944 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4944, __extension__ __PRETTY_FUNCTION__))
;
4945 if (Operand.getValueType() == VT) return Operand; // noop extension
4946 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4949, __extension__ __PRETTY_FUNCTION__))
4947 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4949, __extension__ __PRETTY_FUNCTION__))
4948 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4949, __extension__ __PRETTY_FUNCTION__))
4949 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4949, __extension__ __PRETTY_FUNCTION__))
;
4950 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid zext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid zext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4951, __extension__ __PRETTY_FUNCTION__))
4951 "Invalid zext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid zext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid zext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4951, __extension__ __PRETTY_FUNCTION__))
;
4952 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
4953 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4954 if (OpOpcode == ISD::UNDEF)
4955 // zext(undef) = 0, because the top bits will be zero.
4956 return getConstant(0, DL, VT);
4957 break;
4958 case ISD::ANY_EXTEND:
4959 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ANY_EXTEND!") ?
void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ANY_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4960, __extension__ __PRETTY_FUNCTION__))
4960 "Invalid ANY_EXTEND!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ANY_EXTEND!") ?
void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ANY_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4960, __extension__ __PRETTY_FUNCTION__))
;
4961 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4963, __extension__ __PRETTY_FUNCTION__))
4962 "ANY_EXTEND result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4963, __extension__ __PRETTY_FUNCTION__))
4963 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4963, __extension__ __PRETTY_FUNCTION__))
;
4964 if (Operand.getValueType() == VT) return Operand; // noop extension
4965 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4968, __extension__ __PRETTY_FUNCTION__))
4966 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4968, __extension__ __PRETTY_FUNCTION__))
4967 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4968, __extension__ __PRETTY_FUNCTION__))
4968 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4968, __extension__ __PRETTY_FUNCTION__))
;
4969 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid anyext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid anyext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__))
4970 "Invalid anyext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid anyext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid anyext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__))
;
4971
4972 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4973 OpOpcode == ISD::ANY_EXTEND)
4974 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
4975 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4976 if (OpOpcode == ISD::UNDEF)
4977 return getUNDEF(VT);
4978
4979 // (ext (trunc x)) -> x
4980 if (OpOpcode == ISD::TRUNCATE) {
4981 SDValue OpOp = Operand.getOperand(0);
4982 if (OpOp.getValueType() == VT) {
4983 transferDbgValues(Operand, OpOp);
4984 return OpOp;
4985 }
4986 }
4987 break;
4988 case ISD::TRUNCATE:
4989 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid TRUNCATE!") ? void
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid TRUNCATE!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4990, __extension__ __PRETTY_FUNCTION__))
4990 "Invalid TRUNCATE!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid TRUNCATE!") ? void
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid TRUNCATE!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4990, __extension__ __PRETTY_FUNCTION__))
;
4991 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4993, __extension__ __PRETTY_FUNCTION__))
4992 "TRUNCATE result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4993, __extension__ __PRETTY_FUNCTION__))
4993 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4993, __extension__ __PRETTY_FUNCTION__))
;
4994 if (Operand.getValueType() == VT) return Operand; // noop truncate
4995 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4998, __extension__ __PRETTY_FUNCTION__))
4996 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4998, __extension__ __PRETTY_FUNCTION__))
4997 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4998, __extension__ __PRETTY_FUNCTION__))
4998 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4998, __extension__ __PRETTY_FUNCTION__))
;
4999 assert(Operand.getValueType().bitsGT(VT) &&(static_cast <bool> (Operand.getValueType().bitsGT(VT) &&
"Invalid truncate node, src < dst!") ? void (0) : __assert_fail
("Operand.getValueType().bitsGT(VT) && \"Invalid truncate node, src < dst!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5000, __extension__ __PRETTY_FUNCTION__))
5000 "Invalid truncate node, src < dst!")(static_cast <bool> (Operand.getValueType().bitsGT(VT) &&
"Invalid truncate node, src < dst!") ? void (0) : __assert_fail
("Operand.getValueType().bitsGT(VT) && \"Invalid truncate node, src < dst!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5000, __extension__ __PRETTY_FUNCTION__))
;
5001 if (OpOpcode == ISD::TRUNCATE)
5002 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
5003 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5004 OpOpcode == ISD::ANY_EXTEND) {
5005 // If the source is smaller than the dest, we still need an extend.
5006 if (Operand.getOperand(0).getValueType().getScalarType()
5007 .bitsLT(VT.getScalarType()))
5008 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
5009 if (Operand.getOperand(0).getValueType().bitsGT(VT))
5010 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
5011 return Operand.getOperand(0);
5012 }
5013 if (OpOpcode == ISD::UNDEF)
5014 return getUNDEF(VT);
5015 break;
5016 case ISD::ANY_EXTEND_VECTOR_INREG:
5017 case ISD::ZERO_EXTEND_VECTOR_INREG:
5018 case ISD::SIGN_EXTEND_VECTOR_INREG:
5019 assert(VT.isVector() && "This DAG node is restricted to vector types.")(static_cast <bool> (VT.isVector() && "This DAG node is restricted to vector types."
) ? void (0) : __assert_fail ("VT.isVector() && \"This DAG node is restricted to vector types.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5019, __extension__ __PRETTY_FUNCTION__))
;
5020 assert(Operand.getValueType().bitsLE(VT) &&(static_cast <bool> (Operand.getValueType().bitsLE(VT) &&
"The input must be the same size or smaller than the result."
) ? void (0) : __assert_fail ("Operand.getValueType().bitsLE(VT) && \"The input must be the same size or smaller than the result.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5021, __extension__ __PRETTY_FUNCTION__))
5021 "The input must be the same size or smaller than the result.")(static_cast <bool> (Operand.getValueType().bitsLE(VT) &&
"The input must be the same size or smaller than the result."
) ? void (0) : __assert_fail ("Operand.getValueType().bitsLE(VT) && \"The input must be the same size or smaller than the result.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5021, __extension__ __PRETTY_FUNCTION__))
;
5022 assert(VT.getVectorMinNumElements() <(static_cast <bool> (VT.getVectorMinNumElements() < Operand
.getValueType().getVectorMinNumElements() && "The destination vector type must have fewer lanes than the input."
) ? void (0) : __assert_fail ("VT.getVectorMinNumElements() < Operand.getValueType().getVectorMinNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5024, __extension__ __PRETTY_FUNCTION__))
5023 Operand.getValueType().getVectorMinNumElements() &&(static_cast <bool> (VT.getVectorMinNumElements() < Operand
.getValueType().getVectorMinNumElements() && "The destination vector type must have fewer lanes than the input."
) ? void (0) : __assert_fail ("VT.getVectorMinNumElements() < Operand.getValueType().getVectorMinNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5024, __extension__ __PRETTY_FUNCTION__))
5024 "The destination vector type must have fewer lanes than the input.")(static_cast <bool> (VT.getVectorMinNumElements() < Operand
.getValueType().getVectorMinNumElements() && "The destination vector type must have fewer lanes than the input."
) ? void (0) : __assert_fail ("VT.getVectorMinNumElements() < Operand.getValueType().getVectorMinNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5024, __extension__ __PRETTY_FUNCTION__))
;
5025 break;
5026 case ISD::ABS:
5027 assert(VT.isInteger() && VT == Operand.getValueType() &&(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid ABS!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid ABS!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5028, __extension__ __PRETTY_FUNCTION__))
5028 "Invalid ABS!")(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid ABS!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid ABS!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5028, __extension__ __PRETTY_FUNCTION__))
;
5029 if (OpOpcode == ISD::UNDEF)
5030 return getUNDEF(VT);
5031 break;
5032 case ISD::BSWAP:
5033 assert(VT.isInteger() && VT == Operand.getValueType() &&(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BSWAP!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BSWAP!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5034, __extension__ __PRETTY_FUNCTION__))
5034 "Invalid BSWAP!")(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BSWAP!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BSWAP!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5034, __extension__ __PRETTY_FUNCTION__))
;
5035 assert((VT.getScalarSizeInBits() % 16 == 0) &&(static_cast <bool> ((VT.getScalarSizeInBits() % 16 == 0
) && "BSWAP types must be a multiple of 16 bits!") ? void
(0) : __assert_fail ("(VT.getScalarSizeInBits() % 16 == 0) && \"BSWAP types must be a multiple of 16 bits!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5036, __extension__ __PRETTY_FUNCTION__))
5036 "BSWAP types must be a multiple of 16 bits!")(static_cast <bool> ((VT.getScalarSizeInBits() % 16 == 0
) && "BSWAP types must be a multiple of 16 bits!") ? void
(0) : __assert_fail ("(VT.getScalarSizeInBits() % 16 == 0) && \"BSWAP types must be a multiple of 16 bits!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5036, __extension__ __PRETTY_FUNCTION__))
;
5037 if (OpOpcode == ISD::UNDEF)
5038 return getUNDEF(VT);
5039 break;
5040 case ISD::BITREVERSE:
5041 assert(VT.isInteger() && VT == Operand.getValueType() &&(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BITREVERSE!") ? void (0) :
__assert_fail ("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BITREVERSE!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
5042 "Invalid BITREVERSE!")(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BITREVERSE!") ? void (0) :
__assert_fail ("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BITREVERSE!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
;
5043 if (OpOpcode == ISD::UNDEF)
5044 return getUNDEF(VT);
5045 break;
5046 case ISD::BITCAST:
5047 // Basic sanity checking.
5048 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&(static_cast <bool> (VT.getSizeInBits() == Operand.getValueSizeInBits
() && "Cannot BITCAST between types of different sizes!"
) ? void (0) : __assert_fail ("VT.getSizeInBits() == Operand.getValueSizeInBits() && \"Cannot BITCAST between types of different sizes!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5049, __extension__ __PRETTY_FUNCTION__))
5049 "Cannot BITCAST between types of different sizes!")(static_cast <bool> (VT.getSizeInBits() == Operand.getValueSizeInBits
() && "Cannot BITCAST between types of different sizes!"
) ? void (0) : __assert_fail ("VT.getSizeInBits() == Operand.getValueSizeInBits() && \"Cannot BITCAST between types of different sizes!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5049, __extension__ __PRETTY_FUNCTION__))
;
5050 if (VT == Operand.getValueType()) return Operand; // noop conversion.
5051 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5052 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
5053 if (OpOpcode == ISD::UNDEF)
5054 return getUNDEF(VT);
5055 break;
5056 case ISD::SCALAR_TO_VECTOR:
5057 assert(VT.isVector() && !Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5062, __extension__ __PRETTY_FUNCTION__))
5058 (VT.getVectorElementType() == Operand.getValueType() ||(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5062, __extension__ __PRETTY_FUNCTION__))
5059 (VT.getVectorElementType().isInteger() &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5062, __extension__ __PRETTY_FUNCTION__))
5060 Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5062, __extension__ __PRETTY_FUNCTION__))
5061 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5062, __extension__ __PRETTY_FUNCTION__))
5062 "Illegal SCALAR_TO_VECTOR node!")(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5062, __extension__ __PRETTY_FUNCTION__))
;
5063 if (OpOpcode == ISD::UNDEF)
5064 return getUNDEF(VT);
5065 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5066 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5067 isa<ConstantSDNode>(Operand.getOperand(1)) &&
5068 Operand.getConstantOperandVal(1) == 0 &&
5069 Operand.getOperand(0).getValueType() == VT)
5070 return Operand.getOperand(0);
5071 break;
5072 case ISD::FNEG:
5073 // Negation of an unknown bag of bits is still completely undefined.
5074 if (OpOpcode == ISD::UNDEF)
5075 return getUNDEF(VT);
5076
5077 if (OpOpcode == ISD::FNEG) // --X -> X
5078 return Operand.getOperand(0);
5079 break;
5080 case ISD::FABS:
5081 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5082 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
5083 break;
5084 case ISD::VSCALE:
5085 assert(VT == Operand.getValueType() && "Unexpected VT!")(static_cast <bool> (VT == Operand.getValueType() &&
"Unexpected VT!") ? void (0) : __assert_fail ("VT == Operand.getValueType() && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5085, __extension__ __PRETTY_FUNCTION__))
;
5086 break;
5087 case ISD::CTPOP:
5088 if (Operand.getValueType().getScalarType() == MVT::i1)
5089 return Operand;
5090 break;
5091 case ISD::CTLZ:
5092 case ISD::CTTZ:
5093 if (Operand.getValueType().getScalarType() == MVT::i1)
5094 return getNOT(DL, Operand, Operand.getValueType());
5095 break;
5096 case ISD::VECREDUCE_SMIN:
5097 case ISD::VECREDUCE_UMAX:
5098 if (Operand.getValueType().getScalarType() == MVT::i1)
5099 return getNode(ISD::VECREDUCE_OR, DL, VT, Operand);
5100 break;
5101 case ISD::VECREDUCE_SMAX:
5102 case ISD::VECREDUCE_UMIN:
5103 if (Operand.getValueType().getScalarType() == MVT::i1)
5104 return getNode(ISD::VECREDUCE_AND, DL, VT, Operand);
5105 break;
5106 }
5107
5108 SDNode *N;
5109 SDVTList VTs = getVTList(VT);
5110 SDValue Ops[] = {Operand};
5111 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
5112 FoldingSetNodeID ID;
5113 AddNodeIDNode(ID, Opcode, VTs, Ops);
5114 void *IP = nullptr;
5115 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5116 E->intersectFlagsWith(Flags);
5117 return SDValue(E, 0);
5118 }
5119
5120 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5121 N->setFlags(Flags);
5122 createOperands(N, Ops);
5123 CSEMap.InsertNode(N, IP);
5124 } else {
5125 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5126 createOperands(N, Ops);
5127 }
5128
5129 InsertNode(N);
5130 SDValue V = SDValue(N, 0);
5131 NewSDValueDbgMsg(V, "Creating new node: ", this);
5132 return V;
5133}
5134
5135static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5136 const APInt &C2) {
5137 switch (Opcode) {
5138 case ISD::ADD: return C1 + C2;
5139 case ISD::SUB: return C1 - C2;
5140 case ISD::MUL: return C1 * C2;
5141 case ISD::AND: return C1 & C2;
5142 case ISD::OR: return C1 | C2;
5143 case ISD::XOR: return C1 ^ C2;
5144 case ISD::SHL: return C1 << C2;
5145 case ISD::SRL: return C1.lshr(C2);
5146 case ISD::SRA: return C1.ashr(C2);
5147 case ISD::ROTL: return C1.rotl(C2);
5148 case ISD::ROTR: return C1.rotr(C2);
5149 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
5150 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
5151 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
5152 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
5153 case ISD::SADDSAT: return C1.sadd_sat(C2);
5154 case ISD::UADDSAT: return C1.uadd_sat(C2);
5155 case ISD::SSUBSAT: return C1.ssub_sat(C2);
5156 case ISD::USUBSAT: return C1.usub_sat(C2);
5157 case ISD::UDIV:
5158 if (!C2.getBoolValue())
5159 break;
5160 return C1.udiv(C2);
5161 case ISD::UREM:
5162 if (!C2.getBoolValue())
5163 break;
5164 return C1.urem(C2);
5165 case ISD::SDIV:
5166 if (!C2.getBoolValue())
5167 break;
5168 return C1.sdiv(C2);
5169 case ISD::SREM:
5170 if (!C2.getBoolValue())
5171 break;
5172 return C1.srem(C2);
5173 case ISD::MULHS: {
5174 unsigned FullWidth = C1.getBitWidth() * 2;
5175 APInt C1Ext = C1.sext(FullWidth);
5176 APInt C2Ext = C2.sext(FullWidth);
5177 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5178 }
5179 case ISD::MULHU: {
5180 unsigned FullWidth = C1.getBitWidth() * 2;
5181 APInt C1Ext = C1.zext(FullWidth);
5182 APInt C2Ext = C2.zext(FullWidth);
5183 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5184 }
5185 }
5186 return llvm::None;
5187}
5188
5189SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
5190 const GlobalAddressSDNode *GA,
5191 const SDNode *N2) {
5192 if (GA->getOpcode() != ISD::GlobalAddress)
5193 return SDValue();
5194 if (!TLI->isOffsetFoldingLegal(GA))
5195 return SDValue();
5196 auto *C2 = dyn_cast<ConstantSDNode>(N2);
5197 if (!C2)
5198 return SDValue();
5199 int64_t Offset = C2->getSExtValue();
5200 switch (Opcode) {
5201 case ISD::ADD: break;
5202 case ISD::SUB: Offset = -uint64_t(Offset); break;
5203 default: return SDValue();
5204 }
5205 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
5206 GA->getOffset() + uint64_t(Offset));
5207}
5208
5209bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
5210 switch (Opcode) {
5211 case ISD::SDIV:
5212 case ISD::UDIV:
5213 case ISD::SREM:
5214 case ISD::UREM: {
5215 // If a divisor is zero/undef or any element of a divisor vector is
5216 // zero/undef, the whole op is undef.
5217 assert(Ops.size() == 2 && "Div/rem should have 2 operands")(static_cast <bool> (Ops.size() == 2 && "Div/rem should have 2 operands"
) ? void (0) : __assert_fail ("Ops.size() == 2 && \"Div/rem should have 2 operands\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5217, __extension__ __PRETTY_FUNCTION__))
;
5218 SDValue Divisor = Ops[1];
5219 if (Divisor.isUndef() || isNullConstant(Divisor))
5220 return true;
5221
5222 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
5223 llvm::any_of(Divisor->op_values(),
5224 [](SDValue V) { return V.isUndef() ||
5225 isNullConstant(V); });
5226 // TODO: Handle signed overflow.
5227 }
5228 // TODO: Handle oversized shifts.
5229 default:
5230 return false;
5231 }
5232}
5233
5234SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
5235 EVT VT, ArrayRef<SDValue> Ops) {
5236 // If the opcode is a target-specific ISD node, there's nothing we can
5237 // do here and the operand rules may not line up with the below, so
5238 // bail early.
5239 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
5240 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
5241 // foldCONCAT_VECTORS in getNode before this is called.
5242 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
5243 return SDValue();
5244
5245 // For now, the array Ops should only contain two values.
5246 // This enforcement will be removed once this function is merged with
5247 // FoldConstantVectorArithmetic
5248 if (Ops.size() != 2)
5249 return SDValue();
5250
5251 if (isUndef(Opcode, Ops))
5252 return getUNDEF(VT);
5253
5254 SDNode *N1 = Ops[0].getNode();
5255 SDNode *N2 = Ops[1].getNode();
5256
5257 // Handle the case of two scalars.
5258 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
5259 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
5260 if (C1->isOpaque() || C2->isOpaque())
5261 return SDValue();
5262
5263 Optional<APInt> FoldAttempt =
5264 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
5265 if (!FoldAttempt)
5266 return SDValue();
5267
5268 SDValue Folded = getConstant(FoldAttempt.getValue(), DL, VT);
5269 assert((!Folded || !VT.isVector()) &&(static_cast <bool> ((!Folded || !VT.isVector()) &&
"Can't fold vectors ops with scalar operands") ? void (0) : __assert_fail
("(!Folded || !VT.isVector()) && \"Can't fold vectors ops with scalar operands\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5270, __extension__ __PRETTY_FUNCTION__))
5270 "Can't fold vectors ops with scalar operands")(static_cast <bool> ((!Folded || !VT.isVector()) &&
"Can't fold vectors ops with scalar operands") ? void (0) : __assert_fail
("(!Folded || !VT.isVector()) && \"Can't fold vectors ops with scalar operands\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5270, __extension__ __PRETTY_FUNCTION__))
;
5271 return Folded;
5272 }
5273 }
5274
5275 // fold (add Sym, c) -> Sym+c
5276 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
5277 return FoldSymbolOffset(Opcode, VT, GA, N2);
5278 if (TLI->isCommutativeBinOp(Opcode))
5279 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
5280 return FoldSymbolOffset(Opcode, VT, GA, N1);
5281
5282 // For fixed width vectors, extract each constant element and fold them
5283 // individually. Either input may be an undef value.
5284 bool IsBVOrSV1 = N1->getOpcode() == ISD::BUILD_VECTOR ||
5285 N1->getOpcode() == ISD::SPLAT_VECTOR;
5286 if (!IsBVOrSV1 && !N1->isUndef())
5287 return SDValue();
5288 bool IsBVOrSV2 = N2->getOpcode() == ISD::BUILD_VECTOR ||
5289 N2->getOpcode() == ISD::SPLAT_VECTOR;
5290 if (!IsBVOrSV2 && !N2->isUndef())
5291 return SDValue();
5292 // If both operands are undef, that's handled the same way as scalars.
5293 if (!IsBVOrSV1 && !IsBVOrSV2)
5294 return SDValue();
5295
5296 EVT SVT = VT.getScalarType();
5297 EVT LegalSVT = SVT;
5298 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5299 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5300 if (LegalSVT.bitsLT(SVT))
5301 return SDValue();
5302 }
5303
5304 SmallVector<SDValue, 4> Outputs;
5305 unsigned NumOps = 0;
5306 if (IsBVOrSV1)
5307 NumOps = std::max(NumOps, N1->getNumOperands());
5308 if (IsBVOrSV2)
5309 NumOps = std::max(NumOps, N2->getNumOperands());
5310 assert(NumOps != 0 && "Expected non-zero operands")(static_cast <bool> (NumOps != 0 && "Expected non-zero operands"
) ? void (0) : __assert_fail ("NumOps != 0 && \"Expected non-zero operands\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5310, __extension__ __PRETTY_FUNCTION__))
;
5311 // Scalable vectors should only be SPLAT_VECTOR or UNDEF here. We only need
5312 // one iteration for that.
5313 assert((!VT.isScalableVector() || NumOps == 1) &&(static_cast <bool> ((!VT.isScalableVector() || NumOps ==
1) && "Scalable vector should only have one scalar")
? void (0) : __assert_fail ("(!VT.isScalableVector() || NumOps == 1) && \"Scalable vector should only have one scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5314, __extension__ __PRETTY_FUNCTION__))
5314 "Scalable vector should only have one scalar")(static_cast <bool> ((!VT.isScalableVector() || NumOps ==
1) && "Scalable vector should only have one scalar")
? void (0) : __assert_fail ("(!VT.isScalableVector() || NumOps == 1) && \"Scalable vector should only have one scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5314, __extension__ __PRETTY_FUNCTION__))
;
5315
5316 for (unsigned I = 0; I != NumOps; ++I) {
5317 // We can have a fixed length SPLAT_VECTOR and a BUILD_VECTOR so we need
5318 // to use operand 0 of the SPLAT_VECTOR for each fixed element.
5319 SDValue V1;
5320 if (N1->getOpcode() == ISD::BUILD_VECTOR)
5321 V1 = N1->getOperand(I);
5322 else if (N1->getOpcode() == ISD::SPLAT_VECTOR)
5323 V1 = N1->getOperand(0);
5324 else
5325 V1 = getUNDEF(SVT);
5326
5327 SDValue V2;
5328 if (N2->getOpcode() == ISD::BUILD_VECTOR)
5329 V2 = N2->getOperand(I);
5330 else if (N2->getOpcode() == ISD::SPLAT_VECTOR)
5331 V2 = N2->getOperand(0);
5332 else
5333 V2 = getUNDEF(SVT);
5334
5335 if (SVT.isInteger()) {
5336 if (V1.getValueType().bitsGT(SVT))
5337 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
5338 if (V2.getValueType().bitsGT(SVT))
5339 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
5340 }
5341
5342 if (V1.getValueType() != SVT || V2.getValueType() != SVT)
5343 return SDValue();
5344
5345 // Fold one vector element.
5346 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
5347 if (LegalSVT != SVT)
5348 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5349
5350 // Scalar folding only succeeded if the result is a constant or UNDEF.
5351 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5352 ScalarResult.getOpcode() != ISD::ConstantFP)
5353 return SDValue();
5354 Outputs.push_back(ScalarResult);
5355 }
5356
5357 if (N1->getOpcode() == ISD::BUILD_VECTOR ||
5358 N2->getOpcode() == ISD::BUILD_VECTOR) {
5359 assert(VT.getVectorNumElements() == Outputs.size() &&(static_cast <bool> (VT.getVectorNumElements() == Outputs
.size() && "Vector size mismatch!") ? void (0) : __assert_fail
("VT.getVectorNumElements() == Outputs.size() && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5360, __extension__ __PRETTY_FUNCTION__))
5360 "Vector size mismatch!")(static_cast <bool> (VT.getVectorNumElements() == Outputs
.size() && "Vector size mismatch!") ? void (0) : __assert_fail
("VT.getVectorNumElements() == Outputs.size() && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5360, __extension__ __PRETTY_FUNCTION__))
;
5361
5362 // Build a big vector out of the scalar elements we generated.
5363 return getBuildVector(VT, SDLoc(), Outputs);
5364 }
5365
5366 assert((N1->getOpcode() == ISD::SPLAT_VECTOR ||(static_cast <bool> ((N1->getOpcode() == ISD::SPLAT_VECTOR
|| N2->getOpcode() == ISD::SPLAT_VECTOR) && "One operand should be a splat vector"
) ? void (0) : __assert_fail ("(N1->getOpcode() == ISD::SPLAT_VECTOR || N2->getOpcode() == ISD::SPLAT_VECTOR) && \"One operand should be a splat vector\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5368, __extension__ __PRETTY_FUNCTION__))
5367 N2->getOpcode() == ISD::SPLAT_VECTOR) &&(static_cast <bool> ((N1->getOpcode() == ISD::SPLAT_VECTOR
|| N2->getOpcode() == ISD::SPLAT_VECTOR) && "One operand should be a splat vector"
) ? void (0) : __assert_fail ("(N1->getOpcode() == ISD::SPLAT_VECTOR || N2->getOpcode() == ISD::SPLAT_VECTOR) && \"One operand should be a splat vector\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5368, __extension__ __PRETTY_FUNCTION__))
5368 "One operand should be a splat vector")(static_cast <bool> ((N1->getOpcode() == ISD::SPLAT_VECTOR
|| N2->getOpcode() == ISD::SPLAT_VECTOR) && "One operand should be a splat vector"
) ? void (0) : __assert_fail ("(N1->getOpcode() == ISD::SPLAT_VECTOR || N2->getOpcode() == ISD::SPLAT_VECTOR) && \"One operand should be a splat vector\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5368, __extension__ __PRETTY_FUNCTION__))
;
5369
5370 assert(Outputs.size() == 1 && "Vector size mismatch!")(static_cast <bool> (Outputs.size() == 1 && "Vector size mismatch!"
) ? void (0) : __assert_fail ("Outputs.size() == 1 && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5370, __extension__ __PRETTY_FUNCTION__))
;
5371 return getSplatVector(VT, SDLoc(), Outputs[0]);
5372}
5373
5374// TODO: Merge with FoldConstantArithmetic
5375SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
5376 const SDLoc &DL, EVT VT,
5377 ArrayRef<SDValue> Ops,
5378 const SDNodeFlags Flags) {
5379 // If the opcode is a target-specific ISD node, there's nothing we can
5380 // do here and the operand rules may not line up with the below, so
5381 // bail early.
5382 if (Opcode >= ISD::BUILTIN_OP_END)
5383 return SDValue();
5384
5385 if (isUndef(Opcode, Ops))
5386 return getUNDEF(VT);
5387
5388 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
5389 if (!VT.isVector())
5390 return SDValue();
5391
5392 ElementCount NumElts = VT.getVectorElementCount();
5393
5394 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
5395 return !Op.getValueType().isVector() ||
5396 Op.getValueType().getVectorElementCount() == NumElts;
5397 };
5398
5399 auto IsConstantBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
5400 APInt SplatVal;
5401 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
5402 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
5403 (BV && BV->isConstant()) ||
5404 (Op.getOpcode() == ISD::SPLAT_VECTOR &&
5405 ISD::isConstantSplatVector(Op.getNode(), SplatVal));
5406 };
5407
5408 // All operands must be vector types with the same number of elements as
5409 // the result type and must be either UNDEF or a build vector of constant
5410 // or UNDEF scalars.
5411 if (!llvm::all_of(Ops, IsConstantBuildVectorSplatVectorOrUndef) ||
5412 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
5413 return SDValue();
5414
5415 // If we are comparing vectors, then the result needs to be a i1 boolean
5416 // that is then sign-extended back to the legal result type.
5417 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
5418
5419 // Find legal integer scalar type for constant promotion and
5420 // ensure that its scalar size is at least as large as source.
5421 EVT LegalSVT = VT.getScalarType();
5422 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5423 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5424 if (LegalSVT.bitsLT(VT.getScalarType()))
5425 return SDValue();
5426 }
5427
5428 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
5429 // only have one operand to check. For fixed-length vector types we may have
5430 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
5431 unsigned NumOperands = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
5432
5433 // Constant fold each scalar lane separately.
5434 SmallVector<SDValue, 4> ScalarResults;
5435 for (unsigned I = 0; I != NumOperands; I++) {
5436 SmallVector<SDValue, 4> ScalarOps;
5437 for (SDValue Op : Ops) {
5438 EVT InSVT = Op.getValueType().getScalarType();
5439 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
5440 Op.getOpcode() != ISD::SPLAT_VECTOR) {
5441 // We've checked that this is UNDEF or a constant of some kind.
5442 if (Op.isUndef())
5443 ScalarOps.push_back(getUNDEF(InSVT));
5444 else
5445 ScalarOps.push_back(Op);
5446 continue;
5447 }
5448
5449 SDValue ScalarOp =
5450 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
5451 EVT ScalarVT = ScalarOp.getValueType();
5452
5453 // Build vector (integer) scalar operands may need implicit
5454 // truncation - do this before constant folding.
5455 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
5456 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
5457
5458 ScalarOps.push_back(ScalarOp);
5459 }
5460
5461 // Constant fold the scalar operands.
5462 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
5463
5464 // Legalize the (integer) scalar constant if necessary