Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1163, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-05-225554-32688-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/FoldingSet.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Triple.h"
26#include "llvm/ADT/Twine.h"
27#include "llvm/Analysis/ValueTracking.h"
28#include "llvm/CodeGen/ISDOpcodes.h"
29#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineConstantPool.h"
31#include "llvm/CodeGen/MachineFrameInfo.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineMemOperand.h"
34#include "llvm/CodeGen/RuntimeLibcalls.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
38#include "llvm/CodeGen/TargetLowering.h"
39#include "llvm/CodeGen/TargetRegisterInfo.h"
40#include "llvm/CodeGen/TargetSubtargetInfo.h"
41#include "llvm/CodeGen/ValueTypes.h"
42#include "llvm/IR/Constant.h"
43#include "llvm/IR/Constants.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/DebugInfoMetadata.h"
46#include "llvm/IR/DebugLoc.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/GlobalValue.h"
50#include "llvm/IR/Metadata.h"
51#include "llvm/IR/Type.h"
52#include "llvm/IR/Value.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CodeGen.h"
55#include "llvm/Support/Compiler.h"
56#include "llvm/Support/Debug.h"
57#include "llvm/Support/ErrorHandling.h"
58#include "llvm/Support/KnownBits.h"
59#include "llvm/Support/MachineValueType.h"
60#include "llvm/Support/ManagedStatic.h"
61#include "llvm/Support/MathExtras.h"
62#include "llvm/Support/Mutex.h"
63#include "llvm/Support/raw_ostream.h"
64#include "llvm/Target/TargetMachine.h"
65#include "llvm/Target/TargetOptions.h"
66#include <algorithm>
67#include <cassert>
68#include <cstdint>
69#include <cstdlib>
70#include <limits>
71#include <set>
72#include <string>
73#include <utility>
74#include <vector>
75
76using namespace llvm;
77
78/// makeVTList - Return an instance of the SDVTList struct initialized with the
79/// specified members.
80static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
81 SDVTList Res = {VTs, NumVTs};
82 return Res;
83}
84
85// Default null implementations of the callbacks.
86void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
87void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
88void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
89
90void SelectionDAG::DAGNodeDeletedListener::anchor() {}
91
92#define DEBUG_TYPE"selectiondag" "selectiondag"
93
94static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
95 cl::Hidden, cl::init(true),
96 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
97
98static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
99 cl::desc("Number limit for gluing ld/st of memcpy."),
100 cl::Hidden, cl::init(0));
101
102static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
103 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("selectiondag")) { dbgs() << Msg; V.getNode()->dump
(G);; } } while (false)
;
104}
105
106//===----------------------------------------------------------------------===//
107// ConstantFPSDNode Class
108//===----------------------------------------------------------------------===//
109
110/// isExactlyValue - We don't rely on operator== working on double values, as
111/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
112/// As such, this method can be used to do an exact bit-for-bit comparison of
113/// two floating point values.
114bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
115 return getValueAPF().bitwiseIsEqual(V);
116}
117
118bool ConstantFPSDNode::isValueValidForType(EVT VT,
119 const APFloat& Val) {
120 assert(VT.isFloatingPoint() && "Can only convert between FP types")((VT.isFloatingPoint() && "Can only convert between FP types"
) ? static_cast<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Can only convert between FP types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 120, __PRETTY_FUNCTION__))
;
121
122 // convert modifies in place, so make a copy.
123 APFloat Val2 = APFloat(Val);
124 bool losesInfo;
125 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
126 APFloat::rmNearestTiesToEven,
127 &losesInfo);
128 return !losesInfo;
129}
130
131//===----------------------------------------------------------------------===//
132// ISD Namespace
133//===----------------------------------------------------------------------===//
134
135bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
136 auto *BV = dyn_cast<BuildVectorSDNode>(N);
137 if (!BV)
138 return false;
139
140 APInt SplatUndef;
141 unsigned SplatBitSize;
142 bool HasUndefs;
143 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
144 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
145 EltSize) &&
146 EltSize == SplatBitSize;
147}
148
149// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
150// specializations of the more general isConstantSplatVector()?
151
152bool ISD::isBuildVectorAllOnes(const SDNode *N) {
153 // Look through a bit convert.
154 while (N->getOpcode() == ISD::BITCAST)
155 N = N->getOperand(0).getNode();
156
157 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
158
159 unsigned i = 0, e = N->getNumOperands();
160
161 // Skip over all of the undef values.
162 while (i != e && N->getOperand(i).isUndef())
163 ++i;
164
165 // Do not accept an all-undef vector.
166 if (i == e) return false;
167
168 // Do not accept build_vectors that aren't all constants or which have non-~0
169 // elements. We have to be a bit careful here, as the type of the constant
170 // may not be the same as the type of the vector elements due to type
171 // legalization (the elements are promoted to a legal type for the target and
172 // a vector of a type may be legal when the base element type is not).
173 // We only want to check enough bits to cover the vector elements, because
174 // we care if the resultant vector is all ones, not whether the individual
175 // constants are.
176 SDValue NotZero = N->getOperand(i);
177 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
178 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
179 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
180 return false;
181 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
182 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
183 return false;
184 } else
185 return false;
186
187 // Okay, we have at least one ~0 value, check to see if the rest match or are
188 // undefs. Even with the above element type twiddling, this should be OK, as
189 // the same type legalization should have applied to all the elements.
190 for (++i; i != e; ++i)
191 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
192 return false;
193 return true;
194}
195
196bool ISD::isBuildVectorAllZeros(const SDNode *N) {
197 // Look through a bit convert.
198 while (N->getOpcode() == ISD::BITCAST)
199 N = N->getOperand(0).getNode();
200
201 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
202
203 bool IsAllUndef = true;
204 for (const SDValue &Op : N->op_values()) {
205 if (Op.isUndef())
206 continue;
207 IsAllUndef = false;
208 // Do not accept build_vectors that aren't all constants or which have non-0
209 // elements. We have to be a bit careful here, as the type of the constant
210 // may not be the same as the type of the vector elements due to type
211 // legalization (the elements are promoted to a legal type for the target
212 // and a vector of a type may be legal when the base element type is not).
213 // We only want to check enough bits to cover the vector elements, because
214 // we care if the resultant vector is all zeros, not whether the individual
215 // constants are.
216 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
217 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
218 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
219 return false;
220 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
221 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
222 return false;
223 } else
224 return false;
225 }
226
227 // Do not accept an all-undef vector.
228 if (IsAllUndef)
229 return false;
230 return true;
231}
232
233bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
234 if (N->getOpcode() != ISD::BUILD_VECTOR)
235 return false;
236
237 for (const SDValue &Op : N->op_values()) {
238 if (Op.isUndef())
239 continue;
240 if (!isa<ConstantSDNode>(Op))
241 return false;
242 }
243 return true;
244}
245
246bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
247 if (N->getOpcode() != ISD::BUILD_VECTOR)
248 return false;
249
250 for (const SDValue &Op : N->op_values()) {
251 if (Op.isUndef())
252 continue;
253 if (!isa<ConstantFPSDNode>(Op))
254 return false;
255 }
256 return true;
257}
258
259bool ISD::allOperandsUndef(const SDNode *N) {
260 // Return false if the node has no operands.
261 // This is "logically inconsistent" with the definition of "all" but
262 // is probably the desired behavior.
263 if (N->getNumOperands() == 0)
264 return false;
265 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
266}
267
268bool ISD::matchUnaryPredicate(SDValue Op,
269 std::function<bool(ConstantSDNode *)> Match,
270 bool AllowUndefs) {
271 // FIXME: Add support for scalar UNDEF cases?
272 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
273 return Match(Cst);
274
275 // FIXME: Add support for vector UNDEF cases?
276 if (ISD::BUILD_VECTOR != Op.getOpcode())
277 return false;
278
279 EVT SVT = Op.getValueType().getScalarType();
280 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
281 if (AllowUndefs && Op.getOperand(i).isUndef()) {
282 if (!Match(nullptr))
283 return false;
284 continue;
285 }
286
287 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
288 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
289 return false;
290 }
291 return true;
292}
293
294bool ISD::matchBinaryPredicate(
295 SDValue LHS, SDValue RHS,
296 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
297 bool AllowUndefs, bool AllowTypeMismatch) {
298 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
299 return false;
300
301 // TODO: Add support for scalar UNDEF cases?
302 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
303 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
304 return Match(LHSCst, RHSCst);
305
306 // TODO: Add support for vector UNDEF cases?
307 if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
308 ISD::BUILD_VECTOR != RHS.getOpcode())
309 return false;
310
311 EVT SVT = LHS.getValueType().getScalarType();
312 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
313 SDValue LHSOp = LHS.getOperand(i);
314 SDValue RHSOp = RHS.getOperand(i);
315 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
316 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
317 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
318 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
319 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
320 return false;
321 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
322 LHSOp.getValueType() != RHSOp.getValueType()))
323 return false;
324 if (!Match(LHSCst, RHSCst))
325 return false;
326 }
327 return true;
328}
329
330ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
331 switch (ExtType) {
332 case ISD::EXTLOAD:
333 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
334 case ISD::SEXTLOAD:
335 return ISD::SIGN_EXTEND;
336 case ISD::ZEXTLOAD:
337 return ISD::ZERO_EXTEND;
338 default:
339 break;
340 }
341
342 llvm_unreachable("Invalid LoadExtType")::llvm::llvm_unreachable_internal("Invalid LoadExtType", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 342)
;
343}
344
345ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
346 // To perform this operation, we just need to swap the L and G bits of the
347 // operation.
348 unsigned OldL = (Operation >> 2) & 1;
349 unsigned OldG = (Operation >> 1) & 1;
350 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
351 (OldL << 1) | // New G bit
352 (OldG << 2)); // New L bit.
353}
354
355ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
356 unsigned Operation = Op;
357 if (isInteger)
358 Operation ^= 7; // Flip L, G, E bits, but not U.
359 else
360 Operation ^= 15; // Flip all of the condition bits.
361
362 if (Operation > ISD::SETTRUE2)
363 Operation &= ~8; // Don't let N and U bits get set.
364
365 return ISD::CondCode(Operation);
366}
367
368/// For an integer comparison, return 1 if the comparison is a signed operation
369/// and 2 if the result is an unsigned comparison. Return zero if the operation
370/// does not depend on the sign of the input (setne and seteq).
371static int isSignedOp(ISD::CondCode Opcode) {
372 switch (Opcode) {
373 default: llvm_unreachable("Illegal integer setcc operation!")::llvm::llvm_unreachable_internal("Illegal integer setcc operation!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 373)
;
374 case ISD::SETEQ:
375 case ISD::SETNE: return 0;
376 case ISD::SETLT:
377 case ISD::SETLE:
378 case ISD::SETGT:
379 case ISD::SETGE: return 1;
380 case ISD::SETULT:
381 case ISD::SETULE:
382 case ISD::SETUGT:
383 case ISD::SETUGE: return 2;
384 }
385}
386
387ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
388 bool IsInteger) {
389 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
390 // Cannot fold a signed integer setcc with an unsigned integer setcc.
391 return ISD::SETCC_INVALID;
392
393 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
394
395 // If the N and U bits get set, then the resultant comparison DOES suddenly
396 // care about orderedness, and it is true when ordered.
397 if (Op > ISD::SETTRUE2)
398 Op &= ~16; // Clear the U bit if the N bit is set.
399
400 // Canonicalize illegal integer setcc's.
401 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
402 Op = ISD::SETNE;
403
404 return ISD::CondCode(Op);
405}
406
407ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
408 bool IsInteger) {
409 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
410 // Cannot fold a signed setcc with an unsigned setcc.
411 return ISD::SETCC_INVALID;
412
413 // Combine all of the condition bits.
414 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
415
416 // Canonicalize illegal integer setcc's.
417 if (IsInteger) {
418 switch (Result) {
419 default: break;
420 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
421 case ISD::SETOEQ: // SETEQ & SETU[LG]E
422 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
423 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
424 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
425 }
426 }
427
428 return Result;
429}
430
431//===----------------------------------------------------------------------===//
432// SDNode Profile Support
433//===----------------------------------------------------------------------===//
434
435/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
436static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
437 ID.AddInteger(OpC);
438}
439
440/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
441/// solely with their pointer.
442static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
443 ID.AddPointer(VTList.VTs);
444}
445
446/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
447static void AddNodeIDOperands(FoldingSetNodeID &ID,
448 ArrayRef<SDValue> Ops) {
449 for (auto& Op : Ops) {
450 ID.AddPointer(Op.getNode());
451 ID.AddInteger(Op.getResNo());
452 }
453}
454
455/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
456static void AddNodeIDOperands(FoldingSetNodeID &ID,
457 ArrayRef<SDUse> Ops) {
458 for (auto& Op : Ops) {
459 ID.AddPointer(Op.getNode());
460 ID.AddInteger(Op.getResNo());
461 }
462}
463
464static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
465 SDVTList VTList, ArrayRef<SDValue> OpList) {
466 AddNodeIDOpcode(ID, OpC);
467 AddNodeIDValueTypes(ID, VTList);
468 AddNodeIDOperands(ID, OpList);
469}
470
471/// If this is an SDNode with special info, add this info to the NodeID data.
472static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
473 switch (N->getOpcode()) {
474 case ISD::TargetExternalSymbol:
475 case ISD::ExternalSymbol:
476 case ISD::MCSymbol:
477 llvm_unreachable("Should only be used on nodes with operands")::llvm::llvm_unreachable_internal("Should only be used on nodes with operands"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 477)
;
478 default: break; // Normal nodes don't need extra info.
479 case ISD::TargetConstant:
480 case ISD::Constant: {
481 const ConstantSDNode *C = cast<ConstantSDNode>(N);
482 ID.AddPointer(C->getConstantIntValue());
483 ID.AddBoolean(C->isOpaque());
484 break;
485 }
486 case ISD::TargetConstantFP:
487 case ISD::ConstantFP:
488 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
489 break;
490 case ISD::TargetGlobalAddress:
491 case ISD::GlobalAddress:
492 case ISD::TargetGlobalTLSAddress:
493 case ISD::GlobalTLSAddress: {
494 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
495 ID.AddPointer(GA->getGlobal());
496 ID.AddInteger(GA->getOffset());
497 ID.AddInteger(GA->getTargetFlags());
498 break;
499 }
500 case ISD::BasicBlock:
501 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
502 break;
503 case ISD::Register:
504 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
505 break;
506 case ISD::RegisterMask:
507 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
508 break;
509 case ISD::SRCVALUE:
510 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
511 break;
512 case ISD::FrameIndex:
513 case ISD::TargetFrameIndex:
514 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
515 break;
516 case ISD::LIFETIME_START:
517 case ISD::LIFETIME_END:
518 if (cast<LifetimeSDNode>(N)->hasOffset()) {
519 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
520 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
521 }
522 break;
523 case ISD::JumpTable:
524 case ISD::TargetJumpTable:
525 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
526 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
527 break;
528 case ISD::ConstantPool:
529 case ISD::TargetConstantPool: {
530 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
531 ID.AddInteger(CP->getAlignment());
532 ID.AddInteger(CP->getOffset());
533 if (CP->isMachineConstantPoolEntry())
534 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
535 else
536 ID.AddPointer(CP->getConstVal());
537 ID.AddInteger(CP->getTargetFlags());
538 break;
539 }
540 case ISD::TargetIndex: {
541 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
542 ID.AddInteger(TI->getIndex());
543 ID.AddInteger(TI->getOffset());
544 ID.AddInteger(TI->getTargetFlags());
545 break;
546 }
547 case ISD::LOAD: {
548 const LoadSDNode *LD = cast<LoadSDNode>(N);
549 ID.AddInteger(LD->getMemoryVT().getRawBits());
550 ID.AddInteger(LD->getRawSubclassData());
551 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
552 break;
553 }
554 case ISD::STORE: {
555 const StoreSDNode *ST = cast<StoreSDNode>(N);
556 ID.AddInteger(ST->getMemoryVT().getRawBits());
557 ID.AddInteger(ST->getRawSubclassData());
558 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
559 break;
560 }
561 case ISD::MLOAD: {
562 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
563 ID.AddInteger(MLD->getMemoryVT().getRawBits());
564 ID.AddInteger(MLD->getRawSubclassData());
565 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
566 break;
567 }
568 case ISD::MSTORE: {
569 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
570 ID.AddInteger(MST->getMemoryVT().getRawBits());
571 ID.AddInteger(MST->getRawSubclassData());
572 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
573 break;
574 }
575 case ISD::MGATHER: {
576 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
577 ID.AddInteger(MG->getMemoryVT().getRawBits());
578 ID.AddInteger(MG->getRawSubclassData());
579 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
580 break;
581 }
582 case ISD::MSCATTER: {
583 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
584 ID.AddInteger(MS->getMemoryVT().getRawBits());
585 ID.AddInteger(MS->getRawSubclassData());
586 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
587 break;
588 }
589 case ISD::ATOMIC_CMP_SWAP:
590 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
591 case ISD::ATOMIC_SWAP:
592 case ISD::ATOMIC_LOAD_ADD:
593 case ISD::ATOMIC_LOAD_SUB:
594 case ISD::ATOMIC_LOAD_AND:
595 case ISD::ATOMIC_LOAD_CLR:
596 case ISD::ATOMIC_LOAD_OR:
597 case ISD::ATOMIC_LOAD_XOR:
598 case ISD::ATOMIC_LOAD_NAND:
599 case ISD::ATOMIC_LOAD_MIN:
600 case ISD::ATOMIC_LOAD_MAX:
601 case ISD::ATOMIC_LOAD_UMIN:
602 case ISD::ATOMIC_LOAD_UMAX:
603 case ISD::ATOMIC_LOAD:
604 case ISD::ATOMIC_STORE: {
605 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
606 ID.AddInteger(AT->getMemoryVT().getRawBits());
607 ID.AddInteger(AT->getRawSubclassData());
608 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
609 break;
610 }
611 case ISD::PREFETCH: {
612 const MemSDNode *PF = cast<MemSDNode>(N);
613 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
614 break;
615 }
616 case ISD::VECTOR_SHUFFLE: {
617 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
618 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
619 i != e; ++i)
620 ID.AddInteger(SVN->getMaskElt(i));
621 break;
622 }
623 case ISD::TargetBlockAddress:
624 case ISD::BlockAddress: {
625 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
626 ID.AddPointer(BA->getBlockAddress());
627 ID.AddInteger(BA->getOffset());
628 ID.AddInteger(BA->getTargetFlags());
629 break;
630 }
631 } // end switch (N->getOpcode())
632
633 // Target specific memory nodes could also have address spaces to check.
634 if (N->isTargetMemoryOpcode())
635 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
636}
637
638/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
639/// data.
640static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
641 AddNodeIDOpcode(ID, N->getOpcode());
642 // Add the return value info.
643 AddNodeIDValueTypes(ID, N->getVTList());
644 // Add the operand info.
645 AddNodeIDOperands(ID, N->ops());
646
647 // Handle SDNode leafs with special info.
648 AddNodeIDCustom(ID, N);
649}
650
651//===----------------------------------------------------------------------===//
652// SelectionDAG Class
653//===----------------------------------------------------------------------===//
654
655/// doNotCSE - Return true if CSE should not be performed for this node.
656static bool doNotCSE(SDNode *N) {
657 if (N->getValueType(0) == MVT::Glue)
658 return true; // Never CSE anything that produces a flag.
659
660 switch (N->getOpcode()) {
661 default: break;
662 case ISD::HANDLENODE:
663 case ISD::EH_LABEL:
664 return true; // Never CSE these nodes.
665 }
666
667 // Check that remaining values produced are not flags.
668 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
669 if (N->getValueType(i) == MVT::Glue)
670 return true; // Never CSE anything that produces a flag.
671
672 return false;
673}
674
675/// RemoveDeadNodes - This method deletes all unreachable nodes in the
676/// SelectionDAG.
677void SelectionDAG::RemoveDeadNodes() {
678 // Create a dummy node (which is not added to allnodes), that adds a reference
679 // to the root node, preventing it from being deleted.
680 HandleSDNode Dummy(getRoot());
681
682 SmallVector<SDNode*, 128> DeadNodes;
683
684 // Add all obviously-dead nodes to the DeadNodes worklist.
685 for (SDNode &Node : allnodes())
686 if (Node.use_empty())
687 DeadNodes.push_back(&Node);
688
689 RemoveDeadNodes(DeadNodes);
690
691 // If the root changed (e.g. it was a dead load, update the root).
692 setRoot(Dummy.getValue());
693}
694
695/// RemoveDeadNodes - This method deletes the unreachable nodes in the
696/// given list, and any nodes that become unreachable as a result.
697void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
698
699 // Process the worklist, deleting the nodes and adding their uses to the
700 // worklist.
701 while (!DeadNodes.empty()) {
702 SDNode *N = DeadNodes.pop_back_val();
703 // Skip to next node if we've already managed to delete the node. This could
704 // happen if replacing a node causes a node previously added to the node to
705 // be deleted.
706 if (N->getOpcode() == ISD::DELETED_NODE)
707 continue;
708
709 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
710 DUL->NodeDeleted(N, nullptr);
711
712 // Take the node out of the appropriate CSE map.
713 RemoveNodeFromCSEMaps(N);
714
715 // Next, brutally remove the operand list. This is safe to do, as there are
716 // no cycles in the graph.
717 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
718 SDUse &Use = *I++;
719 SDNode *Operand = Use.getNode();
720 Use.set(SDValue());
721
722 // Now that we removed this operand, see if there are no uses of it left.
723 if (Operand->use_empty())
724 DeadNodes.push_back(Operand);
725 }
726
727 DeallocateNode(N);
728 }
729}
730
731void SelectionDAG::RemoveDeadNode(SDNode *N){
732 SmallVector<SDNode*, 16> DeadNodes(1, N);
733
734 // Create a dummy node that adds a reference to the root node, preventing
735 // it from being deleted. (This matters if the root is an operand of the
736 // dead node.)
737 HandleSDNode Dummy(getRoot());
738
739 RemoveDeadNodes(DeadNodes);
740}
741
742void SelectionDAG::DeleteNode(SDNode *N) {
743 // First take this out of the appropriate CSE map.
744 RemoveNodeFromCSEMaps(N);
745
746 // Finally, remove uses due to operands of this node, remove from the
747 // AllNodes list, and delete the node.
748 DeleteNodeNotInCSEMaps(N);
749}
750
751void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
752 assert(N->getIterator() != AllNodes.begin() &&((N->getIterator() != AllNodes.begin() && "Cannot delete the entry node!"
) ? static_cast<void> (0) : __assert_fail ("N->getIterator() != AllNodes.begin() && \"Cannot delete the entry node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 753, __PRETTY_FUNCTION__))
753 "Cannot delete the entry node!")((N->getIterator() != AllNodes.begin() && "Cannot delete the entry node!"
) ? static_cast<void> (0) : __assert_fail ("N->getIterator() != AllNodes.begin() && \"Cannot delete the entry node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 753, __PRETTY_FUNCTION__))
;
754 assert(N->use_empty() && "Cannot delete a node that is not dead!")((N->use_empty() && "Cannot delete a node that is not dead!"
) ? static_cast<void> (0) : __assert_fail ("N->use_empty() && \"Cannot delete a node that is not dead!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 754, __PRETTY_FUNCTION__))
;
755
756 // Drop all of the operands and decrement used node's use counts.
757 N->DropOperands();
758
759 DeallocateNode(N);
760}
761
762void SDDbgInfo::erase(const SDNode *Node) {
763 DbgValMapType::iterator I = DbgValMap.find(Node);
764 if (I == DbgValMap.end())
765 return;
766 for (auto &Val: I->second)
767 Val->setIsInvalidated();
768 DbgValMap.erase(I);
769}
770
771void SelectionDAG::DeallocateNode(SDNode *N) {
772 // If we have operands, deallocate them.
773 removeOperands(N);
774
775 NodeAllocator.Deallocate(AllNodes.remove(N));
776
777 // Set the opcode to DELETED_NODE to help catch bugs when node
778 // memory is reallocated.
779 // FIXME: There are places in SDag that have grown a dependency on the opcode
780 // value in the released node.
781 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
782 N->NodeType = ISD::DELETED_NODE;
783
784 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
785 // them and forget about that node.
786 DbgInfo->erase(N);
787}
788
789#ifndef NDEBUG
790/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
791static void VerifySDNode(SDNode *N) {
792 switch (N->getOpcode()) {
793 default:
794 break;
795 case ISD::BUILD_PAIR: {
796 EVT VT = N->getValueType(0);
797 assert(N->getNumValues() == 1 && "Too many results!")((N->getNumValues() == 1 && "Too many results!") ?
static_cast<void> (0) : __assert_fail ("N->getNumValues() == 1 && \"Too many results!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 797, __PRETTY_FUNCTION__))
;
798 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&((!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint
()) && "Wrong return type!") ? static_cast<void>
(0) : __assert_fail ("!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 799, __PRETTY_FUNCTION__))
799 "Wrong return type!")((!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint
()) && "Wrong return type!") ? static_cast<void>
(0) : __assert_fail ("!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 799, __PRETTY_FUNCTION__))
;
800 assert(N->getNumOperands() == 2 && "Wrong number of operands!")((N->getNumOperands() == 2 && "Wrong number of operands!"
) ? static_cast<void> (0) : __assert_fail ("N->getNumOperands() == 2 && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 800, __PRETTY_FUNCTION__))
;
801 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&((N->getOperand(0).getValueType() == N->getOperand(1).getValueType
() && "Mismatched operand types!") ? static_cast<void
> (0) : __assert_fail ("N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && \"Mismatched operand types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 802, __PRETTY_FUNCTION__))
802 "Mismatched operand types!")((N->getOperand(0).getValueType() == N->getOperand(1).getValueType
() && "Mismatched operand types!") ? static_cast<void
> (0) : __assert_fail ("N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && \"Mismatched operand types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 802, __PRETTY_FUNCTION__))
;
803 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&((N->getOperand(0).getValueType().isInteger() == VT.isInteger
() && "Wrong operand type!") ? static_cast<void>
(0) : __assert_fail ("N->getOperand(0).getValueType().isInteger() == VT.isInteger() && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 804, __PRETTY_FUNCTION__))
804 "Wrong operand type!")((N->getOperand(0).getValueType().isInteger() == VT.isInteger
() && "Wrong operand type!") ? static_cast<void>
(0) : __assert_fail ("N->getOperand(0).getValueType().isInteger() == VT.isInteger() && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 804, __PRETTY_FUNCTION__))
;
805 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&((VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits
() && "Wrong return type size") ? static_cast<void
> (0) : __assert_fail ("VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && \"Wrong return type size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 806, __PRETTY_FUNCTION__))
806 "Wrong return type size")((VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits
() && "Wrong return type size") ? static_cast<void
> (0) : __assert_fail ("VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && \"Wrong return type size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 806, __PRETTY_FUNCTION__))
;
807 break;
808 }
809 case ISD::BUILD_VECTOR: {
810 assert(N->getNumValues() == 1 && "Too many results!")((N->getNumValues() == 1 && "Too many results!") ?
static_cast<void> (0) : __assert_fail ("N->getNumValues() == 1 && \"Too many results!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 810, __PRETTY_FUNCTION__))
;
811 assert(N->getValueType(0).isVector() && "Wrong return type!")((N->getValueType(0).isVector() && "Wrong return type!"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0).isVector() && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 811, __PRETTY_FUNCTION__))
;
812 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&((N->getNumOperands() == N->getValueType(0).getVectorNumElements
() && "Wrong number of operands!") ? static_cast<void
> (0) : __assert_fail ("N->getNumOperands() == N->getValueType(0).getVectorNumElements() && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 813, __PRETTY_FUNCTION__))
813 "Wrong number of operands!")((N->getNumOperands() == N->getValueType(0).getVectorNumElements
() && "Wrong number of operands!") ? static_cast<void
> (0) : __assert_fail ("N->getNumOperands() == N->getValueType(0).getVectorNumElements() && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 813, __PRETTY_FUNCTION__))
;
814 EVT EltVT = N->getValueType(0).getVectorElementType();
815 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
816 assert((I->getValueType() == EltVT ||(((I->getValueType() == EltVT || (EltVT.isInteger() &&
I->getValueType().isInteger() && EltVT.bitsLE(I->
getValueType()))) && "Wrong operand type!") ? static_cast
<void> (0) : __assert_fail ("(I->getValueType() == EltVT || (EltVT.isInteger() && I->getValueType().isInteger() && EltVT.bitsLE(I->getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 819, __PRETTY_FUNCTION__))
817 (EltVT.isInteger() && I->getValueType().isInteger() &&(((I->getValueType() == EltVT || (EltVT.isInteger() &&
I->getValueType().isInteger() && EltVT.bitsLE(I->
getValueType()))) && "Wrong operand type!") ? static_cast
<void> (0) : __assert_fail ("(I->getValueType() == EltVT || (EltVT.isInteger() && I->getValueType().isInteger() && EltVT.bitsLE(I->getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 819, __PRETTY_FUNCTION__))
818 EltVT.bitsLE(I->getValueType()))) &&(((I->getValueType() == EltVT || (EltVT.isInteger() &&
I->getValueType().isInteger() && EltVT.bitsLE(I->
getValueType()))) && "Wrong operand type!") ? static_cast
<void> (0) : __assert_fail ("(I->getValueType() == EltVT || (EltVT.isInteger() && I->getValueType().isInteger() && EltVT.bitsLE(I->getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 819, __PRETTY_FUNCTION__))
819 "Wrong operand type!")(((I->getValueType() == EltVT || (EltVT.isInteger() &&
I->getValueType().isInteger() && EltVT.bitsLE(I->
getValueType()))) && "Wrong operand type!") ? static_cast
<void> (0) : __assert_fail ("(I->getValueType() == EltVT || (EltVT.isInteger() && I->getValueType().isInteger() && EltVT.bitsLE(I->getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 819, __PRETTY_FUNCTION__))
;
820 assert(I->getValueType() == N->getOperand(0).getValueType() &&((I->getValueType() == N->getOperand(0).getValueType() &&
"Operands must all have the same type") ? static_cast<void
> (0) : __assert_fail ("I->getValueType() == N->getOperand(0).getValueType() && \"Operands must all have the same type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 821, __PRETTY_FUNCTION__))
821 "Operands must all have the same type")((I->getValueType() == N->getOperand(0).getValueType() &&
"Operands must all have the same type") ? static_cast<void
> (0) : __assert_fail ("I->getValueType() == N->getOperand(0).getValueType() && \"Operands must all have the same type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 821, __PRETTY_FUNCTION__))
;
822 }
823 break;
824 }
825 }
826}
827#endif // NDEBUG
828
829/// Insert a newly allocated node into the DAG.
830///
831/// Handles insertion into the all nodes list and CSE map, as well as
832/// verification and other common operations when a new node is allocated.
833void SelectionDAG::InsertNode(SDNode *N) {
834 AllNodes.push_back(N);
835#ifndef NDEBUG
836 N->PersistentId = NextPersistentId++;
837 VerifySDNode(N);
838#endif
839 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
840 DUL->NodeInserted(N);
841}
842
843/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
844/// correspond to it. This is useful when we're about to delete or repurpose
845/// the node. We don't want future request for structurally identical nodes
846/// to return N anymore.
847bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
848 bool Erased = false;
849 switch (N->getOpcode()) {
850 case ISD::HANDLENODE: return false; // noop.
851 case ISD::CONDCODE:
852 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&((CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
"Cond code doesn't exist!") ? static_cast<void> (0) : __assert_fail
("CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && \"Cond code doesn't exist!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 853, __PRETTY_FUNCTION__))
853 "Cond code doesn't exist!")((CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
"Cond code doesn't exist!") ? static_cast<void> (0) : __assert_fail
("CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && \"Cond code doesn't exist!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 853, __PRETTY_FUNCTION__))
;
854 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
855 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
856 break;
857 case ISD::ExternalSymbol:
858 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
859 break;
860 case ISD::TargetExternalSymbol: {
861 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
862 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
863 ESN->getSymbol(), ESN->getTargetFlags()));
864 break;
865 }
866 case ISD::MCSymbol: {
867 auto *MCSN = cast<MCSymbolSDNode>(N);
868 Erased = MCSymbols.erase(MCSN->getMCSymbol());
869 break;
870 }
871 case ISD::VALUETYPE: {
872 EVT VT = cast<VTSDNode>(N)->getVT();
873 if (VT.isExtended()) {
874 Erased = ExtendedValueTypeNodes.erase(VT);
875 } else {
876 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
877 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
878 }
879 break;
880 }
881 default:
882 // Remove it from the CSE Map.
883 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!")((N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"DELETED_NODE in CSEMap!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 883, __PRETTY_FUNCTION__))
;
884 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!")((N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::EntryToken && \"EntryToken in CSEMap!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 884, __PRETTY_FUNCTION__))
;
885 Erased = CSEMap.RemoveNode(N);
886 break;
887 }
888#ifndef NDEBUG
889 // Verify that the node was actually in one of the CSE maps, unless it has a
890 // flag result (which cannot be CSE'd) or is one of the special cases that are
891 // not subject to CSE.
892 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
893 !N->isMachineOpcode() && !doNotCSE(N)) {
894 N->dump(this);
895 dbgs() << "\n";
896 llvm_unreachable("Node is not in map!")::llvm::llvm_unreachable_internal("Node is not in map!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 896)
;
897 }
898#endif
899 return Erased;
900}
901
902/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
903/// maps and modified in place. Add it back to the CSE maps, unless an identical
904/// node already exists, in which case transfer all its users to the existing
905/// node. This transfer can potentially trigger recursive merging.
906void
907SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
908 // For node types that aren't CSE'd, just act as if no identical node
909 // already exists.
910 if (!doNotCSE(N)) {
911 SDNode *Existing = CSEMap.GetOrInsertNode(N);
912 if (Existing != N) {
913 // If there was already an existing matching node, use ReplaceAllUsesWith
914 // to replace the dead one with the existing one. This can cause
915 // recursive merging of other unrelated nodes down the line.
916 ReplaceAllUsesWith(N, Existing);
917
918 // N is now dead. Inform the listeners and delete it.
919 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
920 DUL->NodeDeleted(N, Existing);
921 DeleteNodeNotInCSEMaps(N);
922 return;
923 }
924 }
925
926 // If the node doesn't already exist, we updated it. Inform listeners.
927 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
928 DUL->NodeUpdated(N);
929}
930
931/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
932/// were replaced with those specified. If this node is never memoized,
933/// return null, otherwise return a pointer to the slot it would take. If a
934/// node already exists with these operands, the slot will be non-null.
935SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
936 void *&InsertPos) {
937 if (doNotCSE(N))
938 return nullptr;
939
940 SDValue Ops[] = { Op };
941 FoldingSetNodeID ID;
942 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
943 AddNodeIDCustom(ID, N);
944 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
945 if (Node)
946 Node->intersectFlagsWith(N->getFlags());
947 return Node;
948}
949
950/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
951/// were replaced with those specified. If this node is never memoized,
952/// return null, otherwise return a pointer to the slot it would take. If a
953/// node already exists with these operands, the slot will be non-null.
954SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
955 SDValue Op1, SDValue Op2,
956 void *&InsertPos) {
957 if (doNotCSE(N))
958 return nullptr;
959
960 SDValue Ops[] = { Op1, Op2 };
961 FoldingSetNodeID ID;
962 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
963 AddNodeIDCustom(ID, N);
964 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
965 if (Node)
966 Node->intersectFlagsWith(N->getFlags());
967 return Node;
968}
969
970/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
971/// were replaced with those specified. If this node is never memoized,
972/// return null, otherwise return a pointer to the slot it would take. If a
973/// node already exists with these operands, the slot will be non-null.
974SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
975 void *&InsertPos) {
976 if (doNotCSE(N))
977 return nullptr;
978
979 FoldingSetNodeID ID;
980 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
981 AddNodeIDCustom(ID, N);
982 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
983 if (Node)
984 Node->intersectFlagsWith(N->getFlags());
985 return Node;
986}
987
988unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
989 Type *Ty = VT == MVT::iPTR ?
990 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
991 VT.getTypeForEVT(*getContext());
992
993 return getDataLayout().getABITypeAlignment(Ty);
994}
995
996// EntryNode could meaningfully have debug info if we can find it...
997SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
998 : TM(tm), OptLevel(OL),
999 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1000 Root(getEntryNode()) {
1001 InsertNode(&EntryNode);
1002 DbgInfo = new SDDbgInfo();
1003}
1004
1005void SelectionDAG::init(MachineFunction &NewMF,
1006 OptimizationRemarkEmitter &NewORE,
1007 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1008 LegacyDivergenceAnalysis * Divergence) {
1009 MF = &NewMF;
1010 SDAGISelPass = PassPtr;
1011 ORE = &NewORE;
1012 TLI = getSubtarget().getTargetLowering();
1013 TSI = getSubtarget().getSelectionDAGInfo();
1014 LibInfo = LibraryInfo;
1015 Context = &MF->getFunction().getContext();
1016 DA = Divergence;
1017}
1018
1019SelectionDAG::~SelectionDAG() {
1020 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners")((!UpdateListeners && "Dangling registered DAGUpdateListeners"
) ? static_cast<void> (0) : __assert_fail ("!UpdateListeners && \"Dangling registered DAGUpdateListeners\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1020, __PRETTY_FUNCTION__))
;
1021 allnodes_clear();
1022 OperandRecycler.clear(OperandAllocator);
1023 delete DbgInfo;
1024}
1025
1026void SelectionDAG::allnodes_clear() {
1027 assert(&*AllNodes.begin() == &EntryNode)((&*AllNodes.begin() == &EntryNode) ? static_cast<
void> (0) : __assert_fail ("&*AllNodes.begin() == &EntryNode"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1027, __PRETTY_FUNCTION__))
;
1028 AllNodes.remove(AllNodes.begin());
1029 while (!AllNodes.empty())
1030 DeallocateNode(&AllNodes.front());
1031#ifndef NDEBUG
1032 NextPersistentId = 0;
1033#endif
1034}
1035
1036SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1037 void *&InsertPos) {
1038 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1039 if (N) {
1040 switch (N->getOpcode()) {
1041 default: break;
1042 case ISD::Constant:
1043 case ISD::ConstantFP:
1044 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "::llvm::llvm_unreachable_internal("Querying for Constant and ConstantFP nodes requires "
"debug location. Use another overload.", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1045)
1045 "debug location. Use another overload.")::llvm::llvm_unreachable_internal("Querying for Constant and ConstantFP nodes requires "
"debug location. Use another overload.", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1045)
;
1046 }
1047 }
1048 return N;
1049}
1050
1051SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1052 const SDLoc &DL, void *&InsertPos) {
1053 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1054 if (N) {
1055 switch (N->getOpcode()) {
1056 case ISD::Constant:
1057 case ISD::ConstantFP:
1058 // Erase debug location from the node if the node is used at several
1059 // different places. Do not propagate one location to all uses as it
1060 // will cause a worse single stepping debugging experience.
1061 if (N->getDebugLoc() != DL.getDebugLoc())
1062 N->setDebugLoc(DebugLoc());
1063 break;
1064 default:
1065 // When the node's point of use is located earlier in the instruction
1066 // sequence than its prior point of use, update its debug info to the
1067 // earlier location.
1068 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1069 N->setDebugLoc(DL.getDebugLoc());
1070 break;
1071 }
1072 }
1073 return N;
1074}
1075
1076void SelectionDAG::clear() {
1077 allnodes_clear();
1078 OperandRecycler.clear(OperandAllocator);
1079 OperandAllocator.Reset();
1080 CSEMap.clear();
1081
1082 ExtendedValueTypeNodes.clear();
1083 ExternalSymbols.clear();
1084 TargetExternalSymbols.clear();
1085 MCSymbols.clear();
1086 SDCallSiteDbgInfo.clear();
1087 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1088 static_cast<CondCodeSDNode*>(nullptr));
1089 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1090 static_cast<SDNode*>(nullptr));
1091
1092 EntryNode.UseList = nullptr;
1093 InsertNode(&EntryNode);
1094 Root = getEntryNode();
1095 DbgInfo->clear();
1096}
1097
1098SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1099 return VT.bitsGT(Op.getValueType())
1100 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1101 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1102}
1103
1104SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1105 return VT.bitsGT(Op.getValueType()) ?
1106 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1107 getNode(ISD::TRUNCATE, DL, VT, Op);
1108}
1109
1110SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1111 return VT.bitsGT(Op.getValueType()) ?
1112 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1113 getNode(ISD::TRUNCATE, DL, VT, Op);
1114}
1115
1116SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1117 return VT.bitsGT(Op.getValueType()) ?
1118 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1119 getNode(ISD::TRUNCATE, DL, VT, Op);
1120}
1121
1122SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1123 EVT OpVT) {
1124 if (VT.bitsLE(Op.getValueType()))
1125 return getNode(ISD::TRUNCATE, SL, VT, Op);
1126
1127 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1128 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1129}
1130
1131SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1132 assert(!VT.isVector() &&((!VT.isVector() && "getZeroExtendInReg should use the vector element type instead of "
"the vector type!") ? static_cast<void> (0) : __assert_fail
("!VT.isVector() && \"getZeroExtendInReg should use the vector element type instead of \" \"the vector type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1134, __PRETTY_FUNCTION__))
1133 "getZeroExtendInReg should use the vector element type instead of "((!VT.isVector() && "getZeroExtendInReg should use the vector element type instead of "
"the vector type!") ? static_cast<void> (0) : __assert_fail
("!VT.isVector() && \"getZeroExtendInReg should use the vector element type instead of \" \"the vector type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1134, __PRETTY_FUNCTION__))
1134 "the vector type!")((!VT.isVector() && "getZeroExtendInReg should use the vector element type instead of "
"the vector type!") ? static_cast<void> (0) : __assert_fail
("!VT.isVector() && \"getZeroExtendInReg should use the vector element type instead of \" \"the vector type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1134, __PRETTY_FUNCTION__))
;
1135 if (Op.getValueType().getScalarType() == VT) return Op;
1136 unsigned BitWidth = Op.getScalarValueSizeInBits();
1137 APInt Imm = APInt::getLowBitsSet(BitWidth,
1138 VT.getSizeInBits());
1139 return getNode(ISD::AND, DL, Op.getValueType(), Op,
1140 getConstant(Imm, DL, Op.getValueType()));
1141}
1142
1143SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1144 // Only unsigned pointer semantics are supported right now. In the future this
1145 // might delegate to TLI to check pointer signedness.
1146 return getZExtOrTrunc(Op, DL, VT);
1147}
1148
1149SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1150 // Only unsigned pointer semantics are supported right now. In the future this
1151 // might delegate to TLI to check pointer signedness.
1152 return getZeroExtendInReg(Op, DL, VT);
1153}
1154
1155/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1156SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1157 EVT EltVT = VT.getScalarType();
1158 SDValue NegOne =
1159 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1160 return getNode(ISD::XOR, DL, VT, Val, NegOne);
1161}
1162
1163SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1164 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1165 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1166}
1167
1168SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1169 EVT OpVT) {
1170 if (!V)
1171 return getConstant(0, DL, VT);
1172
1173 switch (TLI->getBooleanContents(OpVT)) {
1174 case TargetLowering::ZeroOrOneBooleanContent:
1175 case TargetLowering::UndefinedBooleanContent:
1176 return getConstant(1, DL, VT);
1177 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1178 return getAllOnesConstant(DL, VT);
1179 }
1180 llvm_unreachable("Unexpected boolean content enum!")::llvm::llvm_unreachable_internal("Unexpected boolean content enum!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1180)
;
1181}
1182
1183SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1184 bool isT, bool isO) {
1185 EVT EltVT = VT.getScalarType();
1186 assert((EltVT.getSizeInBits() >= 64 ||(((EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >>
EltVT.getSizeInBits()) + 1 < 2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? static_cast<void> (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1188, __PRETTY_FUNCTION__))
1187 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&(((EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >>
EltVT.getSizeInBits()) + 1 < 2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? static_cast<void> (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1188, __PRETTY_FUNCTION__))
1188 "getConstant with a uint64_t value that doesn't fit in the type!")(((EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >>
EltVT.getSizeInBits()) + 1 < 2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? static_cast<void> (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1188, __PRETTY_FUNCTION__))
;
1189 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1190}
1191
1192SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1193 bool isT, bool isO) {
1194 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1195}
1196
1197SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1198 EVT VT, bool isT, bool isO) {
1199 assert(VT.isInteger() && "Cannot create FP integer constant!")((VT.isInteger() && "Cannot create FP integer constant!"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"Cannot create FP integer constant!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1199, __PRETTY_FUNCTION__))
;
1200
1201 EVT EltVT = VT.getScalarType();
1202 const ConstantInt *Elt = &Val;
1203
1204 // In some cases the vector type is legal but the element type is illegal and
1205 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1206 // inserted value (the type does not need to match the vector element type).
1207 // Any extra bits introduced will be truncated away.
1208 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1209 TargetLowering::TypePromoteInteger) {
1210 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1211 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1212 Elt = ConstantInt::get(*getContext(), NewVal);
1213 }
1214 // In other cases the element type is illegal and needs to be expanded, for
1215 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1216 // the value into n parts and use a vector type with n-times the elements.
1217 // Then bitcast to the type requested.
1218 // Legalizing constants too early makes the DAGCombiner's job harder so we
1219 // only legalize if the DAG tells us we must produce legal types.
1220 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1221 TLI->getTypeAction(*getContext(), EltVT) ==
1222 TargetLowering::TypeExpandInteger) {
1223 const APInt &NewVal = Elt->getValue();
1224 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1225 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1226 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1227 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1228
1229 // Check the temporary vector is the correct size. If this fails then
1230 // getTypeToTransformTo() probably returned a type whose size (in bits)
1231 // isn't a power-of-2 factor of the requested type size.
1232 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits())((ViaVecVT.getSizeInBits() == VT.getSizeInBits()) ? static_cast
<void> (0) : __assert_fail ("ViaVecVT.getSizeInBits() == VT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1232, __PRETTY_FUNCTION__))
;
1233
1234 SmallVector<SDValue, 2> EltParts;
1235 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1236 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1237 .zextOrTrunc(ViaEltSizeInBits), DL,
1238 ViaEltVT, isT, isO));
1239 }
1240
1241 // EltParts is currently in little endian order. If we actually want
1242 // big-endian order then reverse it now.
1243 if (getDataLayout().isBigEndian())
1244 std::reverse(EltParts.begin(), EltParts.end());
1245
1246 // The elements must be reversed when the element order is different
1247 // to the endianness of the elements (because the BITCAST is itself a
1248 // vector shuffle in this situation). However, we do not need any code to
1249 // perform this reversal because getConstant() is producing a vector
1250 // splat.
1251 // This situation occurs in MIPS MSA.
1252
1253 SmallVector<SDValue, 8> Ops;
1254 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1255 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1256
1257 SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1258 return V;
1259 }
1260
1261 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&((Elt->getBitWidth() == EltVT.getSizeInBits() && "APInt size does not match type size!"
) ? static_cast<void> (0) : __assert_fail ("Elt->getBitWidth() == EltVT.getSizeInBits() && \"APInt size does not match type size!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1262, __PRETTY_FUNCTION__))
1262 "APInt size does not match type size!")((Elt->getBitWidth() == EltVT.getSizeInBits() && "APInt size does not match type size!"
) ? static_cast<void> (0) : __assert_fail ("Elt->getBitWidth() == EltVT.getSizeInBits() && \"APInt size does not match type size!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1262, __PRETTY_FUNCTION__))
;
1263 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1264 FoldingSetNodeID ID;
1265 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1266 ID.AddPointer(Elt);
1267 ID.AddBoolean(isO);
1268 void *IP = nullptr;
1269 SDNode *N = nullptr;
1270 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1271 if (!VT.isVector())
1272 return SDValue(N, 0);
1273
1274 if (!N) {
1275 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1276 CSEMap.InsertNode(N, IP);
1277 InsertNode(N);
1278 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1279 }
1280
1281 SDValue Result(N, 0);
1282 if (VT.isScalableVector())
1283 Result = getSplatVector(VT, DL, Result);
1284 else if (VT.isVector())
1285 Result = getSplatBuildVector(VT, DL, Result);
1286
1287 return Result;
1288}
1289
1290SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1291 bool isTarget) {
1292 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1293}
1294
1295SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1296 const SDLoc &DL, bool LegalTypes) {
1297 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1298 return getConstant(Val, DL, ShiftVT);
1299}
1300
1301SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1302 bool isTarget) {
1303 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1304}
1305
1306SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1307 EVT VT, bool isTarget) {
1308 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!")((VT.isFloatingPoint() && "Cannot create integer FP constant!"
) ? static_cast<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Cannot create integer FP constant!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1308, __PRETTY_FUNCTION__))
;
1309
1310 EVT EltVT = VT.getScalarType();
1311
1312 // Do the map lookup using the actual bit pattern for the floating point
1313 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1314 // we don't have issues with SNANs.
1315 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1316 FoldingSetNodeID ID;
1317 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1318 ID.AddPointer(&V);
1319 void *IP = nullptr;
1320 SDNode *N = nullptr;
1321 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1322 if (!VT.isVector())
1323 return SDValue(N, 0);
1324
1325 if (!N) {
1326 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1327 CSEMap.InsertNode(N, IP);
1328 InsertNode(N);
1329 }
1330
1331 SDValue Result(N, 0);
1332 if (VT.isVector())
1333 Result = getSplatBuildVector(VT, DL, Result);
1334 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1335 return Result;
1336}
1337
1338SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1339 bool isTarget) {
1340 EVT EltVT = VT.getScalarType();
1341 if (EltVT == MVT::f32)
1342 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1343 else if (EltVT == MVT::f64)
1344 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1345 else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1346 EltVT == MVT::f16) {
1347 bool Ignored;
1348 APFloat APF = APFloat(Val);
1349 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1350 &Ignored);
1351 return getConstantFP(APF, DL, VT, isTarget);
1352 } else
1353 llvm_unreachable("Unsupported type in getConstantFP")::llvm::llvm_unreachable_internal("Unsupported type in getConstantFP"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1353)
;
1354}
1355
1356SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1357 EVT VT, int64_t Offset, bool isTargetGA,
1358 unsigned TargetFlags) {
1359 assert((TargetFlags == 0 || isTargetGA) &&(((TargetFlags == 0 || isTargetGA) && "Cannot set target flags on target-independent globals"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTargetGA) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1360, __PRETTY_FUNCTION__))
1360 "Cannot set target flags on target-independent globals")(((TargetFlags == 0 || isTargetGA) && "Cannot set target flags on target-independent globals"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTargetGA) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1360, __PRETTY_FUNCTION__))
;
1361
1362 // Truncate (with sign-extension) the offset value to the pointer size.
1363 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1364 if (BitWidth < 64)
1365 Offset = SignExtend64(Offset, BitWidth);
1366
1367 unsigned Opc;
1368 if (GV->isThreadLocal())
1369 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1370 else
1371 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1372
1373 FoldingSetNodeID ID;
1374 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1375 ID.AddPointer(GV);
1376 ID.AddInteger(Offset);
1377 ID.AddInteger(TargetFlags);
1378 void *IP = nullptr;
1379 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1380 return SDValue(E, 0);
1381
1382 auto *N = newSDNode<GlobalAddressSDNode>(
1383 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1384 CSEMap.InsertNode(N, IP);
1385 InsertNode(N);
1386 return SDValue(N, 0);
1387}
1388
1389SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1390 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1391 FoldingSetNodeID ID;
1392 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1393 ID.AddInteger(FI);
1394 void *IP = nullptr;
1395 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1396 return SDValue(E, 0);
1397
1398 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1399 CSEMap.InsertNode(N, IP);
1400 InsertNode(N);
1401 return SDValue(N, 0);
1402}
1403
1404SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1405 unsigned TargetFlags) {
1406 assert((TargetFlags == 0 || isTarget) &&(((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent jump tables"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent jump tables\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1407, __PRETTY_FUNCTION__))
1407 "Cannot set target flags on target-independent jump tables")(((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent jump tables"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent jump tables\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1407, __PRETTY_FUNCTION__))
;
1408 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1409 FoldingSetNodeID ID;
1410 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1411 ID.AddInteger(JTI);
1412 ID.AddInteger(TargetFlags);
1413 void *IP = nullptr;
1414 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1415 return SDValue(E, 0);
1416
1417 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1418 CSEMap.InsertNode(N, IP);
1419 InsertNode(N);
1420 return SDValue(N, 0);
1421}
1422
1423SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1424 unsigned Alignment, int Offset,
1425 bool isTarget,
1426 unsigned TargetFlags) {
1427 assert((TargetFlags == 0 || isTarget) &&(((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent globals"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1428, __PRETTY_FUNCTION__))
1428 "Cannot set target flags on target-independent globals")(((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent globals"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1428, __PRETTY_FUNCTION__))
;
1429 if (Alignment == 0)
1430 Alignment = MF->getFunction().hasOptSize()
1431 ? getDataLayout().getABITypeAlignment(C->getType())
1432 : getDataLayout().getPrefTypeAlignment(C->getType());
1433 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1434 FoldingSetNodeID ID;
1435 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1436 ID.AddInteger(Alignment);
1437 ID.AddInteger(Offset);
1438 ID.AddPointer(C);
1439 ID.AddInteger(TargetFlags);
1440 void *IP = nullptr;
1441 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1442 return SDValue(E, 0);
1443
1444 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1445 TargetFlags);
1446 CSEMap.InsertNode(N, IP);
1447 InsertNode(N);
1448 return SDValue(N, 0);
1449}
1450
1451SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1452 unsigned Alignment, int Offset,
1453 bool isTarget,
1454 unsigned TargetFlags) {
1455 assert((TargetFlags == 0 || isTarget) &&(((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent globals"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1456, __PRETTY_FUNCTION__))
1456 "Cannot set target flags on target-independent globals")(((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent globals"
) ? static_cast<void> (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1456, __PRETTY_FUNCTION__))
;
1457 if (Alignment == 0)
1458 Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1459 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1460 FoldingSetNodeID ID;
1461 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1462 ID.AddInteger(Alignment);
1463 ID.AddInteger(Offset);
1464 C->addSelectionDAGCSEId(ID);
1465 ID.AddInteger(TargetFlags);
1466 void *IP = nullptr;
1467 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1468 return SDValue(E, 0);
1469
1470 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1471 TargetFlags);
1472 CSEMap.InsertNode(N, IP);
1473 InsertNode(N);
1474 return SDValue(N, 0);
1475}
1476
1477SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1478 unsigned TargetFlags) {
1479 FoldingSetNodeID ID;
1480 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1481 ID.AddInteger(Index);
1482 ID.AddInteger(Offset);
1483 ID.AddInteger(TargetFlags);
1484 void *IP = nullptr;
1485 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1486 return SDValue(E, 0);
1487
1488 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1489 CSEMap.InsertNode(N, IP);
1490 InsertNode(N);
1491 return SDValue(N, 0);
1492}
1493
1494SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1495 FoldingSetNodeID ID;
1496 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1497 ID.AddPointer(MBB);
1498 void *IP = nullptr;
1499 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1500 return SDValue(E, 0);
1501
1502 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1503 CSEMap.InsertNode(N, IP);
1504 InsertNode(N);
1505 return SDValue(N, 0);
1506}
1507
1508SDValue SelectionDAG::getValueType(EVT VT) {
1509 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1510 ValueTypeNodes.size())
1511 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1512
1513 SDNode *&N = VT.isExtended() ?
1514 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1515
1516 if (N) return SDValue(N, 0);
1517 N = newSDNode<VTSDNode>(VT);
1518 InsertNode(N);
1519 return SDValue(N, 0);
1520}
1521
1522SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1523 SDNode *&N = ExternalSymbols[Sym];
1524 if (N) return SDValue(N, 0);
1525 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1526 InsertNode(N);
1527 return SDValue(N, 0);
1528}
1529
1530SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1531 SDNode *&N = MCSymbols[Sym];
1532 if (N)
1533 return SDValue(N, 0);
1534 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1535 InsertNode(N);
1536 return SDValue(N, 0);
1537}
1538
1539SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1540 unsigned TargetFlags) {
1541 SDNode *&N =
1542 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1543 if (N) return SDValue(N, 0);
1544 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1545 InsertNode(N);
1546 return SDValue(N, 0);
1547}
1548
1549SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1550 if ((unsigned)Cond >= CondCodeNodes.size())
1551 CondCodeNodes.resize(Cond+1);
1552
1553 if (!CondCodeNodes[Cond]) {
1554 auto *N = newSDNode<CondCodeSDNode>(Cond);
1555 CondCodeNodes[Cond] = N;
1556 InsertNode(N);
1557 }
1558
1559 return SDValue(CondCodeNodes[Cond], 0);
1560}
1561
1562/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1563/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1564static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1565 std::swap(N1, N2);
1566 ShuffleVectorSDNode::commuteMask(M);
1567}
1568
1569SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1570 SDValue N2, ArrayRef<int> Mask) {
1571 assert(VT.getVectorNumElements() == Mask.size() &&((VT.getVectorNumElements() == Mask.size() && "Must have the same number of vector elements as mask elements!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == Mask.size() && \"Must have the same number of vector elements as mask elements!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1572, __PRETTY_FUNCTION__))
1572 "Must have the same number of vector elements as mask elements!")((VT.getVectorNumElements() == Mask.size() && "Must have the same number of vector elements as mask elements!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == Mask.size() && \"Must have the same number of vector elements as mask elements!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1572, __PRETTY_FUNCTION__))
;
1573 assert(VT == N1.getValueType() && VT == N2.getValueType() &&((VT == N1.getValueType() && VT == N2.getValueType() &&
"Invalid VECTOR_SHUFFLE") ? static_cast<void> (0) : __assert_fail
("VT == N1.getValueType() && VT == N2.getValueType() && \"Invalid VECTOR_SHUFFLE\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1574, __PRETTY_FUNCTION__))
1574 "Invalid VECTOR_SHUFFLE")((VT == N1.getValueType() && VT == N2.getValueType() &&
"Invalid VECTOR_SHUFFLE") ? static_cast<void> (0) : __assert_fail
("VT == N1.getValueType() && VT == N2.getValueType() && \"Invalid VECTOR_SHUFFLE\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1574, __PRETTY_FUNCTION__))
;
1575
1576 // Canonicalize shuffle undef, undef -> undef
1577 if (N1.isUndef() && N2.isUndef())
1578 return getUNDEF(VT);
1579
1580 // Validate that all indices in Mask are within the range of the elements
1581 // input to the shuffle.
1582 int NElts = Mask.size();
1583 assert(llvm::all_of(Mask,((llvm::all_of(Mask, [&](int M) { return M < (NElts * 2
) && M >= -1; }) && "Index out of range") ?
static_cast<void> (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1585, __PRETTY_FUNCTION__))
1584 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&((llvm::all_of(Mask, [&](int M) { return M < (NElts * 2
) && M >= -1; }) && "Index out of range") ?
static_cast<void> (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1585, __PRETTY_FUNCTION__))
1585 "Index out of range")((llvm::all_of(Mask, [&](int M) { return M < (NElts * 2
) && M >= -1; }) && "Index out of range") ?
static_cast<void> (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1585, __PRETTY_FUNCTION__))
;
1586
1587 // Copy the mask so we can do any needed cleanup.
1588 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1589
1590 // Canonicalize shuffle v, v -> v, undef
1591 if (N1 == N2) {
1592 N2 = getUNDEF(VT);
1593 for (int i = 0; i != NElts; ++i)
1594 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1595 }
1596
1597 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1598 if (N1.isUndef())
1599 commuteShuffle(N1, N2, MaskVec);
1600
1601 if (TLI->hasVectorBlend()) {
1602 // If shuffling a splat, try to blend the splat instead. We do this here so
1603 // that even when this arises during lowering we don't have to re-handle it.
1604 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1605 BitVector UndefElements;
1606 SDValue Splat = BV->getSplatValue(&UndefElements);
1607 if (!Splat)
1608 return;
1609
1610 for (int i = 0; i < NElts; ++i) {
1611 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1612 continue;
1613
1614 // If this input comes from undef, mark it as such.
1615 if (UndefElements[MaskVec[i] - Offset]) {
1616 MaskVec[i] = -1;
1617 continue;
1618 }
1619
1620 // If we can blend a non-undef lane, use that instead.
1621 if (!UndefElements[i])
1622 MaskVec[i] = i + Offset;
1623 }
1624 };
1625 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1626 BlendSplat(N1BV, 0);
1627 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1628 BlendSplat(N2BV, NElts);
1629 }
1630
1631 // Canonicalize all index into lhs, -> shuffle lhs, undef
1632 // Canonicalize all index into rhs, -> shuffle rhs, undef
1633 bool AllLHS = true, AllRHS = true;
1634 bool N2Undef = N2.isUndef();
1635 for (int i = 0; i != NElts; ++i) {
1636 if (MaskVec[i] >= NElts) {
1637 if (N2Undef)
1638 MaskVec[i] = -1;
1639 else
1640 AllLHS = false;
1641 } else if (MaskVec[i] >= 0) {
1642 AllRHS = false;
1643 }
1644 }
1645 if (AllLHS && AllRHS)
1646 return getUNDEF(VT);
1647 if (AllLHS && !N2Undef)
1648 N2 = getUNDEF(VT);
1649 if (AllRHS) {
1650 N1 = getUNDEF(VT);
1651 commuteShuffle(N1, N2, MaskVec);
1652 }
1653 // Reset our undef status after accounting for the mask.
1654 N2Undef = N2.isUndef();
1655 // Re-check whether both sides ended up undef.
1656 if (N1.isUndef() && N2Undef)
1657 return getUNDEF(VT);
1658
1659 // If Identity shuffle return that node.
1660 bool Identity = true, AllSame = true;
1661 for (int i = 0; i != NElts; ++i) {
1662 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1663 if (MaskVec[i] != MaskVec[0]) AllSame = false;
1664 }
1665 if (Identity && NElts)
1666 return N1;
1667
1668 // Shuffling a constant splat doesn't change the result.
1669 if (N2Undef) {
1670 SDValue V = N1;
1671
1672 // Look through any bitcasts. We check that these don't change the number
1673 // (and size) of elements and just changes their types.
1674 while (V.getOpcode() == ISD::BITCAST)
1675 V = V->getOperand(0);
1676
1677 // A splat should always show up as a build vector node.
1678 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1679 BitVector UndefElements;
1680 SDValue Splat = BV->getSplatValue(&UndefElements);
1681 // If this is a splat of an undef, shuffling it is also undef.
1682 if (Splat && Splat.isUndef())
1683 return getUNDEF(VT);
1684
1685 bool SameNumElts =
1686 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1687
1688 // We only have a splat which can skip shuffles if there is a splatted
1689 // value and no undef lanes rearranged by the shuffle.
1690 if (Splat && UndefElements.none()) {
1691 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1692 // number of elements match or the value splatted is a zero constant.
1693 if (SameNumElts)
1694 return N1;
1695 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1696 if (C->isNullValue())
1697 return N1;
1698 }
1699
1700 // If the shuffle itself creates a splat, build the vector directly.
1701 if (AllSame && SameNumElts) {
1702 EVT BuildVT = BV->getValueType(0);
1703 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1704 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1705
1706 // We may have jumped through bitcasts, so the type of the
1707 // BUILD_VECTOR may not match the type of the shuffle.
1708 if (BuildVT != VT)
1709 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1710 return NewBV;
1711 }
1712 }
1713 }
1714
1715 FoldingSetNodeID ID;
1716 SDValue Ops[2] = { N1, N2 };
1717 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1718 for (int i = 0; i != NElts; ++i)
1719 ID.AddInteger(MaskVec[i]);
1720
1721 void* IP = nullptr;
1722 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1723 return SDValue(E, 0);
1724
1725 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1726 // SDNode doesn't have access to it. This memory will be "leaked" when
1727 // the node is deallocated, but recovered when the NodeAllocator is released.
1728 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1729 llvm::copy(MaskVec, MaskAlloc);
1730
1731 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1732 dl.getDebugLoc(), MaskAlloc);
1733 createOperands(N, Ops);
1734
1735 CSEMap.InsertNode(N, IP);
1736 InsertNode(N);
1737 SDValue V = SDValue(N, 0);
1738 NewSDValueDbgMsg(V, "Creating new node: ", this);
1739 return V;
1740}
1741
1742SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1743 EVT VT = SV.getValueType(0);
1744 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1745 ShuffleVectorSDNode::commuteMask(MaskVec);
1746
1747 SDValue Op0 = SV.getOperand(0);
1748 SDValue Op1 = SV.getOperand(1);
1749 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1750}
1751
1752SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1753 FoldingSetNodeID ID;
1754 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1755 ID.AddInteger(RegNo);
1756 void *IP = nullptr;
1757 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1758 return SDValue(E, 0);
1759
1760 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1761 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1762 CSEMap.InsertNode(N, IP);
1763 InsertNode(N);
1764 return SDValue(N, 0);
1765}
1766
1767SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1768 FoldingSetNodeID ID;
1769 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1770 ID.AddPointer(RegMask);
1771 void *IP = nullptr;
1772 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1773 return SDValue(E, 0);
1774
1775 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1776 CSEMap.InsertNode(N, IP);
1777 InsertNode(N);
1778 return SDValue(N, 0);
1779}
1780
1781SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1782 MCSymbol *Label) {
1783 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1784}
1785
1786SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1787 SDValue Root, MCSymbol *Label) {
1788 FoldingSetNodeID ID;
1789 SDValue Ops[] = { Root };
1790 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1791 ID.AddPointer(Label);
1792 void *IP = nullptr;
1793 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1794 return SDValue(E, 0);
1795
1796 auto *N =
1797 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
1798 createOperands(N, Ops);
1799
1800 CSEMap.InsertNode(N, IP);
1801 InsertNode(N);
1802 return SDValue(N, 0);
1803}
1804
1805SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1806 int64_t Offset, bool isTarget,
1807 unsigned TargetFlags) {
1808 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1809
1810 FoldingSetNodeID ID;
1811 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1812 ID.AddPointer(BA);
1813 ID.AddInteger(Offset);
1814 ID.AddInteger(TargetFlags);
1815 void *IP = nullptr;
1816 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1817 return SDValue(E, 0);
1818
1819 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
1820 CSEMap.InsertNode(N, IP);
1821 InsertNode(N);
1822 return SDValue(N, 0);
1823}
1824
1825SDValue SelectionDAG::getSrcValue(const Value *V) {
1826 assert((!V || V->getType()->isPointerTy()) &&(((!V || V->getType()->isPointerTy()) && "SrcValue is not a pointer?"
) ? static_cast<void> (0) : __assert_fail ("(!V || V->getType()->isPointerTy()) && \"SrcValue is not a pointer?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1827, __PRETTY_FUNCTION__))
1827 "SrcValue is not a pointer?")(((!V || V->getType()->isPointerTy()) && "SrcValue is not a pointer?"
) ? static_cast<void> (0) : __assert_fail ("(!V || V->getType()->isPointerTy()) && \"SrcValue is not a pointer?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1827, __PRETTY_FUNCTION__))
;
1828
1829 FoldingSetNodeID ID;
1830 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1831 ID.AddPointer(V);
1832
1833 void *IP = nullptr;
1834 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1835 return SDValue(E, 0);
1836
1837 auto *N = newSDNode<SrcValueSDNode>(V);
1838 CSEMap.InsertNode(N, IP);
1839 InsertNode(N);
1840 return SDValue(N, 0);
1841}
1842
1843SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1844 FoldingSetNodeID ID;
1845 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1846 ID.AddPointer(MD);
1847
1848 void *IP = nullptr;
1849 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1850 return SDValue(E, 0);
1851
1852 auto *N = newSDNode<MDNodeSDNode>(MD);
1853 CSEMap.InsertNode(N, IP);
1854 InsertNode(N);
1855 return SDValue(N, 0);
1856}
1857
1858SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1859 if (VT == V.getValueType())
1860 return V;
1861
1862 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1863}
1864
1865SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1866 unsigned SrcAS, unsigned DestAS) {
1867 SDValue Ops[] = {Ptr};
1868 FoldingSetNodeID ID;
1869 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1870 ID.AddInteger(SrcAS);
1871 ID.AddInteger(DestAS);
1872
1873 void *IP = nullptr;
1874 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1875 return SDValue(E, 0);
1876
1877 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
1878 VT, SrcAS, DestAS);
1879 createOperands(N, Ops);
1880
1881 CSEMap.InsertNode(N, IP);
1882 InsertNode(N);
1883 return SDValue(N, 0);
1884}
1885
1886/// getShiftAmountOperand - Return the specified value casted to
1887/// the target's desired shift amount type.
1888SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1889 EVT OpTy = Op.getValueType();
1890 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1891 if (OpTy == ShTy || OpTy.isVector()) return Op;
1892
1893 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1894}
1895
1896SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1897 SDLoc dl(Node);
1898 const TargetLowering &TLI = getTargetLoweringInfo();
1899 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1900 EVT VT = Node->getValueType(0);
1901 SDValue Tmp1 = Node->getOperand(0);
1902 SDValue Tmp2 = Node->getOperand(1);
1903 const MaybeAlign MA(Node->getConstantOperandVal(3));
1904
1905 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1906 Tmp2, MachinePointerInfo(V));
1907 SDValue VAList = VAListLoad;
1908
1909 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
1910 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1911 getConstant(MA->value() - 1, dl, VAList.getValueType()));
1912
1913 VAList =
1914 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
1915 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
1916 }
1917
1918 // Increment the pointer, VAList, to the next vaarg
1919 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1920 getConstant(getDataLayout().getTypeAllocSize(
1921 VT.getTypeForEVT(*getContext())),
1922 dl, VAList.getValueType()));
1923 // Store the incremented VAList to the legalized pointer
1924 Tmp1 =
1925 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
1926 // Load the actual argument out of the pointer VAList
1927 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
1928}
1929
1930SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1931 SDLoc dl(Node);
1932 const TargetLowering &TLI = getTargetLoweringInfo();
1933 // This defaults to loading a pointer from the input and storing it to the
1934 // output, returning the chain.
1935 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
1936 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
1937 SDValue Tmp1 =
1938 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
1939 Node->getOperand(2), MachinePointerInfo(VS));
1940 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
1941 MachinePointerInfo(VD));
1942}
1943
1944SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1945 MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1946 unsigned ByteSize = VT.getStoreSize();
1947 Type *Ty = VT.getTypeForEVT(*getContext());
1948 unsigned StackAlign =
1949 std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1950
1951 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
1952 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1953}
1954
1955SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1956 unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
1957 Type *Ty1 = VT1.getTypeForEVT(*getContext());
1958 Type *Ty2 = VT2.getTypeForEVT(*getContext());
1959 const DataLayout &DL = getDataLayout();
1960 unsigned Align =
1961 std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
1962
1963 MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1964 int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
1965 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1966}
1967
1968SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
1969 ISD::CondCode Cond, const SDLoc &dl) {
1970 EVT OpVT = N1.getValueType();
1971
1972 // These setcc operations always fold.
1973 switch (Cond) {
1974 default: break;
1975 case ISD::SETFALSE:
1976 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
1977 case ISD::SETTRUE:
1978 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
1979
1980 case ISD::SETOEQ:
1981 case ISD::SETOGT:
1982 case ISD::SETOGE:
1983 case ISD::SETOLT:
1984 case ISD::SETOLE:
1985 case ISD::SETONE:
1986 case ISD::SETO:
1987 case ISD::SETUO:
1988 case ISD::SETUEQ:
1989 case ISD::SETUNE:
1990 assert(!OpVT.isInteger() && "Illegal setcc for integer!")((!OpVT.isInteger() && "Illegal setcc for integer!") ?
static_cast<void> (0) : __assert_fail ("!OpVT.isInteger() && \"Illegal setcc for integer!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1990, __PRETTY_FUNCTION__))
;
1991 break;
1992 }
1993
1994 if (OpVT.isInteger()) {
1995 // For EQ and NE, we can always pick a value for the undef to make the
1996 // predicate pass or fail, so we can return undef.
1997 // Matches behavior in llvm::ConstantFoldCompareInstruction.
1998 // icmp eq/ne X, undef -> undef.
1999 if ((N1.isUndef() || N2.isUndef()) &&
2000 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2001 return getUNDEF(VT);
2002
2003 // If both operands are undef, we can return undef for int comparison.
2004 // icmp undef, undef -> undef.
2005 if (N1.isUndef() && N2.isUndef())
2006 return getUNDEF(VT);
2007
2008 // icmp X, X -> true/false
2009 // icmp X, undef -> true/false because undef could be X.
2010 if (N1 == N2)
2011 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2012 }
2013
2014 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2015 const APInt &C2 = N2C->getAPIntValue();
2016 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2017 const APInt &C1 = N1C->getAPIntValue();
2018
2019 switch (Cond) {
2020 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2020)
;
2021 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT);
2022 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT);
2023 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
2024 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
2025 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
2026 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2027 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2028 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2029 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2030 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2031 }
2032 }
2033 }
2034
2035 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2036 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2037
2038 if (N1CFP && N2CFP) {
2039 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2040 switch (Cond) {
2041 default: break;
2042 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2043 return getUNDEF(VT);
2044 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2045 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2046 OpVT);
2047 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2048 return getUNDEF(VT);
2049 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2050 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2051 R==APFloat::cmpLessThan, dl, VT,
2052 OpVT);
2053 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2054 return getUNDEF(VT);
2055 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2056 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2057 OpVT);
2058 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2059 return getUNDEF(VT);
2060 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2061 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2062 VT, OpVT);
2063 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2064 return getUNDEF(VT);
2065 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2066 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2067 R==APFloat::cmpEqual, dl, VT,
2068 OpVT);
2069 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2070 return getUNDEF(VT);
2071 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2072 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2073 R==APFloat::cmpEqual, dl, VT, OpVT);
2074 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2075 OpVT);
2076 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2077 OpVT);
2078 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2079 R==APFloat::cmpEqual, dl, VT,
2080 OpVT);
2081 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2082 OpVT);
2083 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2084 R==APFloat::cmpLessThan, dl, VT,
2085 OpVT);
2086 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2087 R==APFloat::cmpUnordered, dl, VT,
2088 OpVT);
2089 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2090 VT, OpVT);
2091 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2092 OpVT);
2093 }
2094 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2095 // Ensure that the constant occurs on the RHS.
2096 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2097 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2098 return SDValue();
2099 return getSetCC(dl, VT, N2, N1, SwappedCond);
2100 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2101 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2102 // If an operand is known to be a nan (or undef that could be a nan), we can
2103 // fold it.
2104 // Choosing NaN for the undef will always make unordered comparison succeed
2105 // and ordered comparison fails.
2106 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2107 switch (ISD::getUnorderedFlavor(Cond)) {
2108 default:
2109 llvm_unreachable("Unknown flavor!")::llvm::llvm_unreachable_internal("Unknown flavor!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2109)
;
2110 case 0: // Known false.
2111 return getBoolConstant(false, dl, VT, OpVT);
2112 case 1: // Known true.
2113 return getBoolConstant(true, dl, VT, OpVT);
2114 case 2: // Undefined.
2115 return getUNDEF(VT);
2116 }
2117 }
2118
2119 // Could not fold it.
2120 return SDValue();
2121}
2122
2123/// See if the specified operand can be simplified with the knowledge that only
2124/// the bits specified by DemandedBits are used.
2125/// TODO: really we should be making this into the DAG equivalent of
2126/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2127SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2128 EVT VT = V.getValueType();
2129 APInt DemandedElts = VT.isVector()
2130 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2131 : APInt(1, 1);
2132 return GetDemandedBits(V, DemandedBits, DemandedElts);
2133}
2134
2135/// See if the specified operand can be simplified with the knowledge that only
2136/// the bits specified by DemandedBits are used in the elements specified by
2137/// DemandedElts.
2138/// TODO: really we should be making this into the DAG equivalent of
2139/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2140SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2141 const APInt &DemandedElts) {
2142 switch (V.getOpcode()) {
2143 default:
2144 break;
2145 case ISD::Constant: {
2146 auto *CV = cast<ConstantSDNode>(V.getNode());
2147 assert(CV && "Const value should be ConstSDNode.")((CV && "Const value should be ConstSDNode.") ? static_cast
<void> (0) : __assert_fail ("CV && \"Const value should be ConstSDNode.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2147, __PRETTY_FUNCTION__))
;
2148 const APInt &CVal = CV->getAPIntValue();
2149 APInt NewVal = CVal & DemandedBits;
2150 if (NewVal != CVal)
2151 return getConstant(NewVal, SDLoc(V), V.getValueType());
2152 break;
2153 }
2154 case ISD::OR:
2155 case ISD::XOR:
2156 case ISD::SIGN_EXTEND_INREG:
2157 return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts,
2158 *this, 0);
2159 case ISD::SRL:
2160 // Only look at single-use SRLs.
2161 if (!V.getNode()->hasOneUse())
2162 break;
2163 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2164 // See if we can recursively simplify the LHS.
2165 unsigned Amt = RHSC->getZExtValue();
2166
2167 // Watch out for shift count overflow though.
2168 if (Amt >= DemandedBits.getBitWidth())
2169 break;
2170 APInt SrcDemandedBits = DemandedBits << Amt;
2171 if (SDValue SimplifyLHS =
2172 GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2173 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2174 V.getOperand(1));
2175 }
2176 break;
2177 case ISD::AND: {
2178 // X & -1 -> X (ignoring bits which aren't demanded).
2179 // Also handle the case where masked out bits in X are known to be zero.
2180 if (ConstantSDNode *RHSC = isConstOrConstSplat(V.getOperand(1))) {
2181 const APInt &AndVal = RHSC->getAPIntValue();
2182 if (DemandedBits.isSubsetOf(AndVal) ||
2183 DemandedBits.isSubsetOf(computeKnownBits(V.getOperand(0)).Zero |
2184 AndVal))
2185 return V.getOperand(0);
2186 }
2187 break;
2188 }
2189 case ISD::ANY_EXTEND: {
2190 SDValue Src = V.getOperand(0);
2191 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
2192 // Being conservative here - only peek through if we only demand bits in the
2193 // non-extended source (even though the extended bits are technically
2194 // undef).
2195 if (DemandedBits.getActiveBits() > SrcBitWidth)
2196 break;
2197 APInt SrcDemandedBits = DemandedBits.trunc(SrcBitWidth);
2198 if (SDValue DemandedSrc = GetDemandedBits(Src, SrcDemandedBits))
2199 return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
2200 break;
2201 }
2202 }
2203 return SDValue();
2204}
2205
2206/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2207/// use this predicate to simplify operations downstream.
2208bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2209 unsigned BitWidth = Op.getScalarValueSizeInBits();
2210 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2211}
2212
2213/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2214/// this predicate to simplify operations downstream. Mask is known to be zero
2215/// for bits that V cannot have.
2216bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2217 unsigned Depth) const {
2218 EVT VT = V.getValueType();
2219 APInt DemandedElts = VT.isVector()
2220 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2221 : APInt(1, 1);
2222 return MaskedValueIsZero(V, Mask, DemandedElts, Depth);
2223}
2224
2225/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2226/// DemandedElts. We use this predicate to simplify operations downstream.
2227/// Mask is known to be zero for bits that V cannot have.
2228bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2229 const APInt &DemandedElts,
2230 unsigned Depth) const {
2231 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2232}
2233
2234/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2235bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2236 unsigned Depth) const {
2237 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2238}
2239
2240/// isSplatValue - Return true if the vector V has the same value
2241/// across all DemandedElts.
2242bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2243 APInt &UndefElts) {
2244 if (!DemandedElts)
2245 return false; // No demanded elts, better to assume we don't know anything.
2246
2247 EVT VT = V.getValueType();
2248 assert(VT.isVector() && "Vector type expected")((VT.isVector() && "Vector type expected") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Vector type expected\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2248, __PRETTY_FUNCTION__))
;
2249
2250 unsigned NumElts = VT.getVectorNumElements();
2251 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch")((NumElts == DemandedElts.getBitWidth() && "Vector size mismatch"
) ? static_cast<void> (0) : __assert_fail ("NumElts == DemandedElts.getBitWidth() && \"Vector size mismatch\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2251, __PRETTY_FUNCTION__))
;
2252 UndefElts = APInt::getNullValue(NumElts);
2253
2254 switch (V.getOpcode()) {
2255 case ISD::BUILD_VECTOR: {
2256 SDValue Scl;
2257 for (unsigned i = 0; i != NumElts; ++i) {
2258 SDValue Op = V.getOperand(i);
2259 if (Op.isUndef()) {
2260 UndefElts.setBit(i);
2261 continue;
2262 }
2263 if (!DemandedElts[i])
2264 continue;
2265 if (Scl && Scl != Op)
2266 return false;
2267 Scl = Op;
2268 }
2269 return true;
2270 }
2271 case ISD::VECTOR_SHUFFLE: {
2272 // Check if this is a shuffle node doing a splat.
2273 // TODO: Do we need to handle shuffle(splat, undef, mask)?
2274 int SplatIndex = -1;
2275 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2276 for (int i = 0; i != (int)NumElts; ++i) {
2277 int M = Mask[i];
2278 if (M < 0) {
2279 UndefElts.setBit(i);
2280 continue;
2281 }
2282 if (!DemandedElts[i])
2283 continue;
2284 if (0 <= SplatIndex && SplatIndex != M)
2285 return false;
2286 SplatIndex = M;
2287 }
2288 return true;
2289 }
2290 case ISD::EXTRACT_SUBVECTOR: {
2291 SDValue Src = V.getOperand(0);
2292 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(V.getOperand(1));
2293 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2294 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2295 // Offset the demanded elts by the subvector index.
2296 uint64_t Idx = SubIdx->getZExtValue();
2297 APInt UndefSrcElts;
2298 APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2299 if (isSplatValue(Src, DemandedSrc, UndefSrcElts)) {
2300 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2301 return true;
2302 }
2303 }
2304 break;
2305 }
2306 case ISD::ADD:
2307 case ISD::SUB:
2308 case ISD::AND: {
2309 APInt UndefLHS, UndefRHS;
2310 SDValue LHS = V.getOperand(0);
2311 SDValue RHS = V.getOperand(1);
2312 if (isSplatValue(LHS, DemandedElts, UndefLHS) &&
2313 isSplatValue(RHS, DemandedElts, UndefRHS)) {
2314 UndefElts = UndefLHS | UndefRHS;
2315 return true;
2316 }
2317 break;
2318 }
2319 }
2320
2321 return false;
2322}
2323
2324/// Helper wrapper to main isSplatValue function.
2325bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2326 EVT VT = V.getValueType();
2327 assert(VT.isVector() && "Vector type expected")((VT.isVector() && "Vector type expected") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Vector type expected\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2327, __PRETTY_FUNCTION__))
;
2328 unsigned NumElts = VT.getVectorNumElements();
2329
2330 APInt UndefElts;
2331 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
2332 return isSplatValue(V, DemandedElts, UndefElts) &&
2333 (AllowUndefs || !UndefElts);
2334}
2335
2336SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2337 V = peekThroughExtractSubvectors(V);
2338
2339 EVT VT = V.getValueType();
2340 unsigned Opcode = V.getOpcode();
2341 switch (Opcode) {
2342 default: {
2343 APInt UndefElts;
2344 APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2345 if (isSplatValue(V, DemandedElts, UndefElts)) {
2346 // Handle case where all demanded elements are UNDEF.
2347 if (DemandedElts.isSubsetOf(UndefElts)) {
2348 SplatIdx = 0;
2349 return getUNDEF(VT);
2350 }
2351 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2352 return V;
2353 }
2354 break;
2355 }
2356 case ISD::VECTOR_SHUFFLE: {
2357 // Check if this is a shuffle node doing a splat.
2358 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2359 // getTargetVShiftNode currently struggles without the splat source.
2360 auto *SVN = cast<ShuffleVectorSDNode>(V);
2361 if (!SVN->isSplat())
2362 break;
2363 int Idx = SVN->getSplatIndex();
2364 int NumElts = V.getValueType().getVectorNumElements();
2365 SplatIdx = Idx % NumElts;
2366 return V.getOperand(Idx / NumElts);
2367 }
2368 }
2369
2370 return SDValue();
2371}
2372
2373SDValue SelectionDAG::getSplatValue(SDValue V) {
2374 int SplatIdx;
2375 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx))
2376 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V),
2377 SrcVector.getValueType().getScalarType(), SrcVector,
2378 getIntPtrConstant(SplatIdx, SDLoc(V)));
2379 return SDValue();
2380}
2381
2382/// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
2383/// is less than the element bit-width of the shift node, return it.
2384static const APInt *getValidShiftAmountConstant(SDValue V) {
2385 unsigned BitWidth = V.getScalarValueSizeInBits();
2386 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
2387 // Shifting more than the bitwidth is not valid.
2388 const APInt &ShAmt = SA->getAPIntValue();
2389 if (ShAmt.ult(BitWidth))
2390 return &ShAmt;
2391 }
2392 return nullptr;
2393}
2394
2395/// If a SHL/SRA/SRL node has constant vector shift amounts that are all less
2396/// than the element bit-width of the shift node, return the minimum value.
2397static const APInt *getValidMinimumShiftAmountConstant(SDValue V) {
2398 unsigned BitWidth = V.getScalarValueSizeInBits();
2399 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2400 if (!BV)
2401 return nullptr;
2402 const APInt *MinShAmt = nullptr;
2403 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2404 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2405 if (!SA)
2406 return nullptr;
2407 // Shifting more than the bitwidth is not valid.
2408 const APInt &ShAmt = SA->getAPIntValue();
2409 if (ShAmt.uge(BitWidth))
2410 return nullptr;
2411 if (MinShAmt && MinShAmt->ule(ShAmt))
2412 continue;
2413 MinShAmt = &ShAmt;
2414 }
2415 return MinShAmt;
2416}
2417
2418/// Determine which bits of Op are known to be either zero or one and return
2419/// them in Known. For vectors, the known bits are those that are shared by
2420/// every vector element.
2421KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2422 EVT VT = Op.getValueType();
2423 APInt DemandedElts = VT.isVector()
2424 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2425 : APInt(1, 1);
2426 return computeKnownBits(Op, DemandedElts, Depth);
2427}
2428
2429/// Determine which bits of Op are known to be either zero or one and return
2430/// them in Known. The DemandedElts argument allows us to only collect the known
2431/// bits that are shared by the requested vector elements.
2432KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2433 unsigned Depth) const {
2434 unsigned BitWidth = Op.getScalarValueSizeInBits();
2435
2436 KnownBits Known(BitWidth); // Don't know anything.
2437
2438 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2439 // We know all of the bits for a constant!
2440 Known.One = C->getAPIntValue();
2441 Known.Zero = ~Known.One;
2442 return Known;
2443 }
2444 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2445 // We know all of the bits for a constant fp!
2446 Known.One = C->getValueAPF().bitcastToAPInt();
2447 Known.Zero = ~Known.One;
2448 return Known;
2449 }
2450
2451 if (Depth >= MaxRecursionDepth)
2452 return Known; // Limit search depth.
2453
2454 KnownBits Known2;
2455 unsigned NumElts = DemandedElts.getBitWidth();
2456 assert((!Op.getValueType().isVector() ||(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2458, __PRETTY_FUNCTION__))
2457 NumElts == Op.getValueType().getVectorNumElements()) &&(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2458, __PRETTY_FUNCTION__))
2458 "Unexpected vector size")(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2458, __PRETTY_FUNCTION__))
;
2459
2460 if (!DemandedElts)
2461 return Known; // No demanded elts, better to assume we don't know anything.
2462
2463 unsigned Opcode = Op.getOpcode();
2464 switch (Opcode) {
2465 case ISD::BUILD_VECTOR:
2466 // Collect the known bits that are shared by every demanded vector element.
2467 Known.Zero.setAllBits(); Known.One.setAllBits();
2468 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2469 if (!DemandedElts[i])
2470 continue;
2471
2472 SDValue SrcOp = Op.getOperand(i);
2473 Known2 = computeKnownBits(SrcOp, Depth + 1);
2474
2475 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2476 if (SrcOp.getValueSizeInBits() != BitWidth) {
2477 assert(SrcOp.getValueSizeInBits() > BitWidth &&((SrcOp.getValueSizeInBits() > BitWidth && "Expected BUILD_VECTOR implicit truncation"
) ? static_cast<void> (0) : __assert_fail ("SrcOp.getValueSizeInBits() > BitWidth && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2478, __PRETTY_FUNCTION__))
2478 "Expected BUILD_VECTOR implicit truncation")((SrcOp.getValueSizeInBits() > BitWidth && "Expected BUILD_VECTOR implicit truncation"
) ? static_cast<void> (0) : __assert_fail ("SrcOp.getValueSizeInBits() > BitWidth && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2478, __PRETTY_FUNCTION__))
;
2479 Known2 = Known2.trunc(BitWidth);
2480 }
2481
2482 // Known bits are the values that are shared by every demanded element.
2483 Known.One &= Known2.One;
2484 Known.Zero &= Known2.Zero;
2485
2486 // If we don't know any bits, early out.
2487 if (Known.isUnknown())
2488 break;
2489 }
2490 break;
2491 case ISD::VECTOR_SHUFFLE: {
2492 // Collect the known bits that are shared by every vector element referenced
2493 // by the shuffle.
2494 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2495 Known.Zero.setAllBits(); Known.One.setAllBits();
2496 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2497 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")((NumElts == SVN->getMask().size() && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("NumElts == SVN->getMask().size() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2497, __PRETTY_FUNCTION__))
;
2498 for (unsigned i = 0; i != NumElts; ++i) {
2499 if (!DemandedElts[i])
2500 continue;
2501
2502 int M = SVN->getMaskElt(i);
2503 if (M < 0) {
2504 // For UNDEF elements, we don't know anything about the common state of
2505 // the shuffle result.
2506 Known.resetAll();
2507 DemandedLHS.clearAllBits();
2508 DemandedRHS.clearAllBits();
2509 break;
2510 }
2511
2512 if ((unsigned)M < NumElts)
2513 DemandedLHS.setBit((unsigned)M % NumElts);
2514 else
2515 DemandedRHS.setBit((unsigned)M % NumElts);
2516 }
2517 // Known bits are the values that are shared by every demanded element.
2518 if (!!DemandedLHS) {
2519 SDValue LHS = Op.getOperand(0);
2520 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2521 Known.One &= Known2.One;
2522 Known.Zero &= Known2.Zero;
2523 }
2524 // If we don't know any bits, early out.
2525 if (Known.isUnknown())
2526 break;
2527 if (!!DemandedRHS) {
2528 SDValue RHS = Op.getOperand(1);
2529 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2530 Known.One &= Known2.One;
2531 Known.Zero &= Known2.Zero;
2532 }
2533 break;
2534 }
2535 case ISD::CONCAT_VECTORS: {
2536 // Split DemandedElts and test each of the demanded subvectors.
2537 Known.Zero.setAllBits(); Known.One.setAllBits();
2538 EVT SubVectorVT = Op.getOperand(0).getValueType();
2539 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2540 unsigned NumSubVectors = Op.getNumOperands();
2541 for (unsigned i = 0; i != NumSubVectors; ++i) {
2542 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
2543 DemandedSub = DemandedSub.trunc(NumSubVectorElts);
2544 if (!!DemandedSub) {
2545 SDValue Sub = Op.getOperand(i);
2546 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2547 Known.One &= Known2.One;
2548 Known.Zero &= Known2.Zero;
2549 }
2550 // If we don't know any bits, early out.
2551 if (Known.isUnknown())
2552 break;
2553 }
2554 break;
2555 }
2556 case ISD::INSERT_SUBVECTOR: {
2557 // If we know the element index, demand any elements from the subvector and
2558 // the remainder from the src its inserted into, otherwise demand them all.
2559 SDValue Src = Op.getOperand(0);
2560 SDValue Sub = Op.getOperand(1);
2561 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2562 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2563 if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
2564 Known.One.setAllBits();
2565 Known.Zero.setAllBits();
2566 uint64_t Idx = SubIdx->getZExtValue();
2567 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2568 if (!!DemandedSubElts) {
2569 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2570 if (Known.isUnknown())
2571 break; // early-out.
2572 }
2573 APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
2574 APInt DemandedSrcElts = DemandedElts & ~SubMask;
2575 if (!!DemandedSrcElts) {
2576 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2577 Known.One &= Known2.One;
2578 Known.Zero &= Known2.Zero;
2579 }
2580 } else {
2581 Known = computeKnownBits(Sub, Depth + 1);
2582 if (Known.isUnknown())
2583 break; // early-out.
2584 Known2 = computeKnownBits(Src, Depth + 1);
2585 Known.One &= Known2.One;
2586 Known.Zero &= Known2.Zero;
2587 }
2588 break;
2589 }
2590 case ISD::EXTRACT_SUBVECTOR: {
2591 // If we know the element index, just demand that subvector elements,
2592 // otherwise demand them all.
2593 SDValue Src = Op.getOperand(0);
2594 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2595 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2596 APInt DemandedSrc = APInt::getAllOnesValue(NumSrcElts);
2597 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2598 // Offset the demanded elts by the subvector index.
2599 uint64_t Idx = SubIdx->getZExtValue();
2600 DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2601 }
2602 Known = computeKnownBits(Src, DemandedSrc, Depth + 1);
2603 break;
2604 }
2605 case ISD::SCALAR_TO_VECTOR: {
2606 // We know about scalar_to_vector as much as we know about it source,
2607 // which becomes the first element of otherwise unknown vector.
2608 if (DemandedElts != 1)
2609 break;
2610
2611 SDValue N0 = Op.getOperand(0);
2612 Known = computeKnownBits(N0, Depth + 1);
2613 if (N0.getValueSizeInBits() != BitWidth)
2614 Known = Known.trunc(BitWidth);
2615
2616 break;
2617 }
2618 case ISD::BITCAST: {
2619 SDValue N0 = Op.getOperand(0);
2620 EVT SubVT = N0.getValueType();
2621 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2622
2623 // Ignore bitcasts from unsupported types.
2624 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2625 break;
2626
2627 // Fast handling of 'identity' bitcasts.
2628 if (BitWidth == SubBitWidth) {
2629 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2630 break;
2631 }
2632
2633 bool IsLE = getDataLayout().isLittleEndian();
2634
2635 // Bitcast 'small element' vector to 'large element' scalar/vector.
2636 if ((BitWidth % SubBitWidth) == 0) {
2637 assert(N0.getValueType().isVector() && "Expected bitcast from vector")((N0.getValueType().isVector() && "Expected bitcast from vector"
) ? static_cast<void> (0) : __assert_fail ("N0.getValueType().isVector() && \"Expected bitcast from vector\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2637, __PRETTY_FUNCTION__))
;
2638
2639 // Collect known bits for the (larger) output by collecting the known
2640 // bits from each set of sub elements and shift these into place.
2641 // We need to separately call computeKnownBits for each set of
2642 // sub elements as the knownbits for each is likely to be different.
2643 unsigned SubScale = BitWidth / SubBitWidth;
2644 APInt SubDemandedElts(NumElts * SubScale, 0);
2645 for (unsigned i = 0; i != NumElts; ++i)
2646 if (DemandedElts[i])
2647 SubDemandedElts.setBit(i * SubScale);
2648
2649 for (unsigned i = 0; i != SubScale; ++i) {
2650 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2651 Depth + 1);
2652 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2653 Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts);
2654 Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts);
2655 }
2656 }
2657
2658 // Bitcast 'large element' scalar/vector to 'small element' vector.
2659 if ((SubBitWidth % BitWidth) == 0) {
2660 assert(Op.getValueType().isVector() && "Expected bitcast to vector")((Op.getValueType().isVector() && "Expected bitcast to vector"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Expected bitcast to vector\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2660, __PRETTY_FUNCTION__))
;
2661
2662 // Collect known bits for the (smaller) output by collecting the known
2663 // bits from the overlapping larger input elements and extracting the
2664 // sub sections we actually care about.
2665 unsigned SubScale = SubBitWidth / BitWidth;
2666 APInt SubDemandedElts(NumElts / SubScale, 0);
2667 for (unsigned i = 0; i != NumElts; ++i)
2668 if (DemandedElts[i])
2669 SubDemandedElts.setBit(i / SubScale);
2670
2671 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2672
2673 Known.Zero.setAllBits(); Known.One.setAllBits();
2674 for (unsigned i = 0; i != NumElts; ++i)
2675 if (DemandedElts[i]) {
2676 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
2677 unsigned Offset = (Shifts % SubScale) * BitWidth;
2678 Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
2679 Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
2680 // If we don't know any bits, early out.
2681 if (Known.isUnknown())
2682 break;
2683 }
2684 }
2685 break;
2686 }
2687 case ISD::AND:
2688 // If either the LHS or the RHS are Zero, the result is zero.
2689 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2690 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2691
2692 // Output known-1 bits are only known if set in both the LHS & RHS.
2693 Known.One &= Known2.One;
2694 // Output known-0 are known to be clear if zero in either the LHS | RHS.
2695 Known.Zero |= Known2.Zero;
2696 break;
2697 case ISD::OR:
2698 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2699 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2700
2701 // Output known-0 bits are only known if clear in both the LHS & RHS.
2702 Known.Zero &= Known2.Zero;
2703 // Output known-1 are known to be set if set in either the LHS | RHS.
2704 Known.One |= Known2.One;
2705 break;
2706 case ISD::XOR: {
2707 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2708 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2709
2710 // Output known-0 bits are known if clear or set in both the LHS & RHS.
2711 APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
2712 // Output known-1 are known to be set if set in only one of the LHS, RHS.
2713 Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
2714 Known.Zero = KnownZeroOut;
2715 break;
2716 }
2717 case ISD::MUL: {
2718 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2719 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2720
2721 // If low bits are zero in either operand, output low known-0 bits.
2722 // Also compute a conservative estimate for high known-0 bits.
2723 // More trickiness is possible, but this is sufficient for the
2724 // interesting case of alignment computation.
2725 unsigned TrailZ = Known.countMinTrailingZeros() +
2726 Known2.countMinTrailingZeros();
2727 unsigned LeadZ = std::max(Known.countMinLeadingZeros() +
2728 Known2.countMinLeadingZeros(),
2729 BitWidth) - BitWidth;
2730
2731 Known.resetAll();
2732 Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
2733 Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
2734 break;
2735 }
2736 case ISD::UDIV: {
2737 // For the purposes of computing leading zeros we can conservatively
2738 // treat a udiv as a logical right shift by the power of 2 known to
2739 // be less than the denominator.
2740 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2741 unsigned LeadZ = Known2.countMinLeadingZeros();
2742
2743 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2744 unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
2745 if (RHSMaxLeadingZeros != BitWidth)
2746 LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
2747
2748 Known.Zero.setHighBits(LeadZ);
2749 break;
2750 }
2751 case ISD::SELECT:
2752 case ISD::VSELECT:
2753 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2754 // If we don't know any bits, early out.
2755 if (Known.isUnknown())
2756 break;
2757 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
2758
2759 // Only known if known in both the LHS and RHS.
2760 Known.One &= Known2.One;
2761 Known.Zero &= Known2.Zero;
2762 break;
2763 case ISD::SELECT_CC:
2764 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
2765 // If we don't know any bits, early out.
2766 if (Known.isUnknown())
2767 break;
2768 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2769
2770 // Only known if known in both the LHS and RHS.
2771 Known.One &= Known2.One;
2772 Known.Zero &= Known2.Zero;
2773 break;
2774 case ISD::SMULO:
2775 case ISD::UMULO:
2776 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
2777 if (Op.getResNo() != 1)
2778 break;
2779 // The boolean result conforms to getBooleanContents.
2780 // If we know the result of a setcc has the top bits zero, use this info.
2781 // We know that we have an integer-based boolean since these operations
2782 // are only available for integer.
2783 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2784 TargetLowering::ZeroOrOneBooleanContent &&
2785 BitWidth > 1)
2786 Known.Zero.setBitsFrom(1);
2787 break;
2788 case ISD::SETCC:
2789 // If we know the result of a setcc has the top bits zero, use this info.
2790 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2791 TargetLowering::ZeroOrOneBooleanContent &&
2792 BitWidth > 1)
2793 Known.Zero.setBitsFrom(1);
2794 break;
2795 case ISD::SHL:
2796 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2797 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2798 unsigned Shift = ShAmt->getZExtValue();
2799 Known.Zero <<= Shift;
2800 Known.One <<= Shift;
2801 // Low bits are known zero.
2802 Known.Zero.setLowBits(Shift);
2803 }
2804 break;
2805 case ISD::SRL:
2806 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2807 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2808 unsigned Shift = ShAmt->getZExtValue();
2809 Known.Zero.lshrInPlace(Shift);
2810 Known.One.lshrInPlace(Shift);
2811 // High bits are known zero.
2812 Known.Zero.setHighBits(Shift);
2813 } else if (const APInt *ShMinAmt = getValidMinimumShiftAmountConstant(Op)) {
2814 // Minimum shift high bits are known zero.
2815 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
2816 }
2817 break;
2818 case ISD::SRA:
2819 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2820 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2821 unsigned Shift = ShAmt->getZExtValue();
2822 // Sign extend known zero/one bit (else is unknown).
2823 Known.Zero.ashrInPlace(Shift);
2824 Known.One.ashrInPlace(Shift);
2825 }
2826 break;
2827 case ISD::FSHL:
2828 case ISD::FSHR:
2829 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
2830 unsigned Amt = C->getAPIntValue().urem(BitWidth);
2831
2832 // For fshl, 0-shift returns the 1st arg.
2833 // For fshr, 0-shift returns the 2nd arg.
2834 if (Amt == 0) {
2835 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
2836 DemandedElts, Depth + 1);
2837 break;
2838 }
2839
2840 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
2841 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
2842 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2843 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2844 if (Opcode == ISD::FSHL) {
2845 Known.One <<= Amt;
2846 Known.Zero <<= Amt;
2847 Known2.One.lshrInPlace(BitWidth - Amt);
2848 Known2.Zero.lshrInPlace(BitWidth - Amt);
2849 } else {
2850 Known.One <<= BitWidth - Amt;
2851 Known.Zero <<= BitWidth - Amt;
2852 Known2.One.lshrInPlace(Amt);
2853 Known2.Zero.lshrInPlace(Amt);
2854 }
2855 Known.One |= Known2.One;
2856 Known.Zero |= Known2.Zero;
2857 }
2858 break;
2859 case ISD::SIGN_EXTEND_INREG: {
2860 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2861 unsigned EBits = EVT.getScalarSizeInBits();
2862
2863 // Sign extension. Compute the demanded bits in the result that are not
2864 // present in the input.
2865 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2866
2867 APInt InSignMask = APInt::getSignMask(EBits);
2868 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2869
2870 // If the sign extended bits are demanded, we know that the sign
2871 // bit is demanded.
2872 InSignMask = InSignMask.zext(BitWidth);
2873 if (NewBits.getBoolValue())
2874 InputDemandedBits |= InSignMask;
2875
2876 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2877 Known.One &= InputDemandedBits;
2878 Known.Zero &= InputDemandedBits;
2879
2880 // If the sign bit of the input is known set or clear, then we know the
2881 // top bits of the result.
2882 if (Known.Zero.intersects(InSignMask)) { // Input sign bit known clear
2883 Known.Zero |= NewBits;
2884 Known.One &= ~NewBits;
2885 } else if (Known.One.intersects(InSignMask)) { // Input sign bit known set
2886 Known.One |= NewBits;
2887 Known.Zero &= ~NewBits;
2888 } else { // Input sign bit unknown
2889 Known.Zero &= ~NewBits;
2890 Known.One &= ~NewBits;
2891 }
2892 break;
2893 }
2894 case ISD::CTTZ:
2895 case ISD::CTTZ_ZERO_UNDEF: {
2896 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2897 // If we have a known 1, its position is our upper bound.
2898 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2899 unsigned LowBits = Log2_32(PossibleTZ) + 1;
2900 Known.Zero.setBitsFrom(LowBits);
2901 break;
2902 }
2903 case ISD::CTLZ:
2904 case ISD::CTLZ_ZERO_UNDEF: {
2905 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2906 // If we have a known 1, its position is our upper bound.
2907 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2908 unsigned LowBits = Log2_32(PossibleLZ) + 1;
2909 Known.Zero.setBitsFrom(LowBits);
2910 break;
2911 }
2912 case ISD::CTPOP: {
2913 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2914 // If we know some of the bits are zero, they can't be one.
2915 unsigned PossibleOnes = Known2.countMaxPopulation();
2916 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
2917 break;
2918 }
2919 case ISD::LOAD: {
2920 LoadSDNode *LD = cast<LoadSDNode>(Op);
2921 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
2922 if (ISD::isNON_EXTLoad(LD) && Cst) {
2923 // Determine any common known bits from the loaded constant pool value.
2924 Type *CstTy = Cst->getType();
2925 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
2926 // If its a vector splat, then we can (quickly) reuse the scalar path.
2927 // NOTE: We assume all elements match and none are UNDEF.
2928 if (CstTy->isVectorTy()) {
2929 if (const Constant *Splat = Cst->getSplatValue()) {
2930 Cst = Splat;
2931 CstTy = Cst->getType();
2932 }
2933 }
2934 // TODO - do we need to handle different bitwidths?
2935 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
2936 // Iterate across all vector elements finding common known bits.
2937 Known.One.setAllBits();
2938 Known.Zero.setAllBits();
2939 for (unsigned i = 0; i != NumElts; ++i) {
2940 if (!DemandedElts[i])
2941 continue;
2942 if (Constant *Elt = Cst->getAggregateElement(i)) {
2943 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
2944 const APInt &Value = CInt->getValue();
2945 Known.One &= Value;
2946 Known.Zero &= ~Value;
2947 continue;
2948 }
2949 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
2950 APInt Value = CFP->getValueAPF().bitcastToAPInt();
2951 Known.One &= Value;
2952 Known.Zero &= ~Value;
2953 continue;
2954 }
2955 }
2956 Known.One.clearAllBits();
2957 Known.Zero.clearAllBits();
2958 break;
2959 }
2960 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
2961 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
2962 const APInt &Value = CInt->getValue();
2963 Known.One = Value;
2964 Known.Zero = ~Value;
2965 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
2966 APInt Value = CFP->getValueAPF().bitcastToAPInt();
2967 Known.One = Value;
2968 Known.Zero = ~Value;
2969 }
2970 }
2971 }
2972 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
2973 // If this is a ZEXTLoad and we are looking at the loaded value.
2974 EVT VT = LD->getMemoryVT();
2975 unsigned MemBits = VT.getScalarSizeInBits();
2976 Known.Zero.setBitsFrom(MemBits);
2977 } else if (const MDNode *Ranges = LD->getRanges()) {
2978 if (LD->getExtensionType() == ISD::NON_EXTLOAD)
2979 computeKnownBitsFromRangeMetadata(*Ranges, Known);
2980 }
2981 break;
2982 }
2983 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2984 EVT InVT = Op.getOperand(0).getValueType();
2985 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
2986 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
2987 Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
2988 break;
2989 }
2990 case ISD::ZERO_EXTEND: {
2991 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2992 Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
2993 break;
2994 }
2995 case ISD::SIGN_EXTEND_VECTOR_INREG: {
2996 EVT InVT = Op.getOperand(0).getValueType();
2997 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
2998 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
2999 // If the sign bit is known to be zero or one, then sext will extend
3000 // it to the top bits, else it will just zext.
3001 Known = Known.sext(BitWidth);
3002 break;
3003 }
3004 case ISD::SIGN_EXTEND: {
3005 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3006 // If the sign bit is known to be zero or one, then sext will extend
3007 // it to the top bits, else it will just zext.
3008 Known = Known.sext(BitWidth);
3009 break;
3010 }
3011 case ISD::ANY_EXTEND: {
3012 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3013 Known = Known.zext(BitWidth, false /* ExtendedBitsAreKnownZero */);
3014 break;
3015 }
3016 case ISD::TRUNCATE: {
3017 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3018 Known = Known.trunc(BitWidth);
3019 break;
3020 }
3021 case ISD::AssertZext: {
3022 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3023 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3024 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3025 Known.Zero |= (~InMask);
3026 Known.One &= (~Known.Zero);
3027 break;
3028 }
3029 case ISD::FGETSIGN:
3030 // All bits are zero except the low bit.
3031 Known.Zero.setBitsFrom(1);
3032 break;
3033 case ISD::USUBO:
3034 case ISD::SSUBO:
3035 if (Op.getResNo() == 1) {
3036 // If we know the result of a setcc has the top bits zero, use this info.
3037 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3038 TargetLowering::ZeroOrOneBooleanContent &&
3039 BitWidth > 1)
3040 Known.Zero.setBitsFrom(1);
3041 break;
3042 }
3043 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3044 case ISD::SUB:
3045 case ISD::SUBC: {
3046 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3047 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3048 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3049 Known, Known2);
3050 break;
3051 }
3052 case ISD::UADDO:
3053 case ISD::SADDO:
3054 case ISD::ADDCARRY:
3055 if (Op.getResNo() == 1) {
3056 // If we know the result of a setcc has the top bits zero, use this info.
3057 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3058 TargetLowering::ZeroOrOneBooleanContent &&
3059 BitWidth > 1)
3060 Known.Zero.setBitsFrom(1);
3061 break;
3062 }
3063 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3064 case ISD::ADD:
3065 case ISD::ADDC:
3066 case ISD::ADDE: {
3067 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.")((Op.getResNo() == 0 && "We only compute knownbits for the sum here."
) ? static_cast<void> (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the sum here.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3067, __PRETTY_FUNCTION__))
;
3068
3069 // With ADDE and ADDCARRY, a carry bit may be added in.
3070 KnownBits Carry(1);
3071 if (Opcode == ISD::ADDE)
3072 // Can't track carry from glue, set carry to unknown.
3073 Carry.resetAll();
3074 else if (Opcode == ISD::ADDCARRY)
3075 // TODO: Compute known bits for the carry operand. Not sure if it is worth
3076 // the trouble (how often will we find a known carry bit). And I haven't
3077 // tested this very much yet, but something like this might work:
3078 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3079 // Carry = Carry.zextOrTrunc(1, false);
3080 Carry.resetAll();
3081 else
3082 Carry.setAllZero();
3083
3084 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3085 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3086 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3087 break;
3088 }
3089 case ISD::SREM:
3090 if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
3091 const APInt &RA = Rem->getAPIntValue().abs();
3092 if (RA.isPowerOf2()) {
3093 APInt LowBits = RA - 1;
3094 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3095
3096 // The low bits of the first operand are unchanged by the srem.
3097 Known.Zero = Known2.Zero & LowBits;
3098 Known.One = Known2.One & LowBits;
3099
3100 // If the first operand is non-negative or has all low bits zero, then
3101 // the upper bits are all zero.
3102 if (Known2.isNonNegative() || LowBits.isSubsetOf(Known2.Zero))
3103 Known.Zero |= ~LowBits;
3104
3105 // If the first operand is negative and not all low bits are zero, then
3106 // the upper bits are all one.
3107 if (Known2.isNegative() && LowBits.intersects(Known2.One))
3108 Known.One |= ~LowBits;
3109 assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?")(((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("(Known.Zero & Known.One) == 0&&\"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3109, __PRETTY_FUNCTION__))
;
3110 }
3111 }
3112 break;
3113 case ISD::UREM: {
3114 if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
3115 const APInt &RA = Rem->getAPIntValue();
3116 if (RA.isPowerOf2()) {
3117 APInt LowBits = (RA - 1);
3118 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3119
3120 // The upper bits are all zero, the lower ones are unchanged.
3121 Known.Zero = Known2.Zero | ~LowBits;
3122 Known.One = Known2.One & LowBits;
3123 break;
3124 }
3125 }
3126
3127 // Since the result is less than or equal to either operand, any leading
3128 // zero bits in either operand must also exist in the result.
3129 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3130 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3131
3132 uint32_t Leaders =
3133 std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
3134 Known.resetAll();
3135 Known.Zero.setHighBits(Leaders);
3136 break;
3137 }
3138 case ISD::EXTRACT_ELEMENT: {
3139 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3140 const unsigned Index = Op.getConstantOperandVal(1);
3141 const unsigned EltBitWidth = Op.getValueSizeInBits();
3142
3143 // Remove low part of known bits mask
3144 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3145 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3146
3147 // Remove high part of known bit mask
3148 Known = Known.trunc(EltBitWidth);
3149 break;
3150 }
3151 case ISD::EXTRACT_VECTOR_ELT: {
3152 SDValue InVec = Op.getOperand(0);
3153 SDValue EltNo = Op.getOperand(1);
3154 EVT VecVT = InVec.getValueType();
3155 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3156 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3157 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3158 // anything about the extended bits.
3159 if (BitWidth > EltBitWidth)
3160 Known = Known.trunc(EltBitWidth);
3161 ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3162 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) {
3163 // If we know the element index, just demand that vector element.
3164 unsigned Idx = ConstEltNo->getZExtValue();
3165 APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
3166 Known = computeKnownBits(InVec, DemandedElt, Depth + 1);
3167 } else {
3168 // Unknown element index, so ignore DemandedElts and demand them all.
3169 Known = computeKnownBits(InVec, Depth + 1);
3170 }
3171 if (BitWidth > EltBitWidth)
3172 Known = Known.zext(BitWidth, false /* => any extend */);
3173 break;
3174 }
3175 case ISD::INSERT_VECTOR_ELT: {
3176 SDValue InVec = Op.getOperand(0);
3177 SDValue InVal = Op.getOperand(1);
3178 SDValue EltNo = Op.getOperand(2);
3179
3180 ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3181 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3182 // If we know the element index, split the demand between the
3183 // source vector and the inserted element.
3184 Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
3185 unsigned EltIdx = CEltNo->getZExtValue();
3186
3187 // If we demand the inserted element then add its common known bits.
3188 if (DemandedElts[EltIdx]) {
3189 Known2 = computeKnownBits(InVal, Depth + 1);
3190 Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
3191 Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
3192 }
3193
3194 // If we demand the source vector then add its common known bits, ensuring
3195 // that we don't demand the inserted element.
3196 APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
3197 if (!!VectorElts) {
3198 Known2 = computeKnownBits(InVec, VectorElts, Depth + 1);
3199 Known.One &= Known2.One;
3200 Known.Zero &= Known2.Zero;
3201 }
3202 } else {
3203 // Unknown element index, so ignore DemandedElts and demand them all.
3204 Known = computeKnownBits(InVec, Depth + 1);
3205 Known2 = computeKnownBits(InVal, Depth + 1);
3206 Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
3207 Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
3208 }
3209 break;
3210 }
3211 case ISD::BITREVERSE: {
3212 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3213 Known.Zero = Known2.Zero.reverseBits();
3214 Known.One = Known2.One.reverseBits();
3215 break;
3216 }
3217 case ISD::BSWAP: {
3218 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3219 Known.Zero = Known2.Zero.byteSwap();
3220 Known.One = Known2.One.byteSwap();
3221 break;
3222 }
3223 case ISD::ABS: {
3224 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3225
3226 // If the source's MSB is zero then we know the rest of the bits already.
3227 if (Known2.isNonNegative()) {
3228 Known.Zero = Known2.Zero;
3229 Known.One = Known2.One;
3230 break;
3231 }
3232
3233 // We only know that the absolute values's MSB will be zero iff there is
3234 // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
3235 Known2.One.clearSignBit();
3236 if (Known2.One.getBoolValue()) {
3237 Known.Zero = APInt::getSignMask(BitWidth);
3238 break;
3239 }
3240 break;
3241 }
3242 case ISD::UMIN: {
3243 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3244 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3245
3246 // UMIN - we know that the result will have the maximum of the
3247 // known zero leading bits of the inputs.
3248 unsigned LeadZero = Known.countMinLeadingZeros();
3249 LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
3250
3251 Known.Zero &= Known2.Zero;
3252 Known.One &= Known2.One;
3253 Known.Zero.setHighBits(LeadZero);
3254 break;
3255 }
3256 case ISD::UMAX: {
3257 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3258 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3259
3260 // UMAX - we know that the result will have the maximum of the
3261 // known one leading bits of the inputs.
3262 unsigned LeadOne = Known.countMinLeadingOnes();
3263 LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
3264
3265 Known.Zero &= Known2.Zero;
3266 Known.One &= Known2.One;
3267 Known.One.setHighBits(LeadOne);
3268 break;
3269 }
3270 case ISD::SMIN:
3271 case ISD::SMAX: {
3272 // If we have a clamp pattern, we know that the number of sign bits will be
3273 // the minimum of the clamp min/max range.
3274 bool IsMax = (Opcode == ISD::SMAX);
3275 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3276 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3277 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3278 CstHigh =
3279 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3280 if (CstLow && CstHigh) {
3281 if (!IsMax)
3282 std::swap(CstLow, CstHigh);
3283
3284 const APInt &ValueLow = CstLow->getAPIntValue();
3285 const APInt &ValueHigh = CstHigh->getAPIntValue();
3286 if (ValueLow.sle(ValueHigh)) {
3287 unsigned LowSignBits = ValueLow.getNumSignBits();
3288 unsigned HighSignBits = ValueHigh.getNumSignBits();
3289 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3290 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3291 Known.One.setHighBits(MinSignBits);
3292 break;
3293 }
3294 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3295 Known.Zero.setHighBits(MinSignBits);
3296 break;
3297 }
3298 }
3299 }
3300
3301 // Fallback - just get the shared known bits of the operands.
3302 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3303 if (Known.isUnknown()) break; // Early-out
3304 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3305 Known.Zero &= Known2.Zero;
3306 Known.One &= Known2.One;
3307 break;
3308 }
3309 case ISD::FrameIndex:
3310 case ISD::TargetFrameIndex:
3311 TLI->computeKnownBitsForFrameIndex(Op, Known, DemandedElts, *this, Depth);
3312 break;
3313
3314 default:
3315 if (Opcode < ISD::BUILTIN_OP_END)
3316 break;
3317 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3318 case ISD::INTRINSIC_WO_CHAIN:
3319 case ISD::INTRINSIC_W_CHAIN:
3320 case ISD::INTRINSIC_VOID:
3321 // Allow the target to implement this method for its nodes.
3322 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3323 break;
3324 }
3325
3326 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3326, __PRETTY_FUNCTION__))
;
3327 return Known;
3328}
3329
3330SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3331 SDValue N1) const {
3332 // X + 0 never overflow
3333 if (isNullConstant(N1))
3334 return OFK_Never;
3335
3336 KnownBits N1Known = computeKnownBits(N1);
3337 if (N1Known.Zero.getBoolValue()) {
3338 KnownBits N0Known = computeKnownBits(N0);
3339
3340 bool overflow;
3341 (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
3342 if (!overflow)
3343 return OFK_Never;
3344 }
3345
3346 // mulhi + 1 never overflow
3347 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3348 (~N1Known.Zero & 0x01) == ~N1Known.Zero)
3349 return OFK_Never;
3350
3351 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3352 KnownBits N0Known = computeKnownBits(N0);
3353
3354 if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
3355 return OFK_Never;
3356 }
3357
3358 return OFK_Sometime;
3359}
3360
3361bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3362 EVT OpVT = Val.getValueType();
3363 unsigned BitWidth = OpVT.getScalarSizeInBits();
3364
3365 // Is the constant a known power of 2?
3366 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3367 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3368
3369 // A left-shift of a constant one will have exactly one bit set because
3370 // shifting the bit off the end is undefined.
3371 if (Val.getOpcode() == ISD::SHL) {
3372 auto *C = isConstOrConstSplat(Val.getOperand(0));
3373 if (C && C->getAPIntValue() == 1)
3374 return true;
3375 }
3376
3377 // Similarly, a logical right-shift of a constant sign-bit will have exactly
3378 // one bit set.
3379 if (Val.getOpcode() == ISD::SRL) {
3380 auto *C = isConstOrConstSplat(Val.getOperand(0));
3381 if (C && C->getAPIntValue().isSignMask())
3382 return true;
3383 }
3384
3385 // Are all operands of a build vector constant powers of two?
3386 if (Val.getOpcode() == ISD::BUILD_VECTOR)
3387 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3388 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3389 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3390 return false;
3391 }))
3392 return true;
3393
3394 // More could be done here, though the above checks are enough
3395 // to handle some common cases.
3396
3397 // Fall back to computeKnownBits to catch other known cases.
3398 KnownBits Known = computeKnownBits(Val);
3399 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3400}
3401
3402unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3403 EVT VT = Op.getValueType();
14
Calling 'SDValue::getValueType'
3404 APInt DemandedElts = VT.isVector()
3405 ? APInt::getAllOnesValue(VT.getVectorNumElements())
3406 : APInt(1, 1);
3407 return ComputeNumSignBits(Op, DemandedElts, Depth);
3408}
3409
3410unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3411 unsigned Depth) const {
3412 EVT VT = Op.getValueType();
3413 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!")(((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!"
) ? static_cast<void> (0) : __assert_fail ("(VT.isInteger() || VT.isFloatingPoint()) && \"Invalid VT!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3413, __PRETTY_FUNCTION__))
;
1
Assuming the condition is true
2
'?' condition is true
3414 unsigned VTBits = VT.getScalarSizeInBits();
3415 unsigned NumElts = DemandedElts.getBitWidth();
3416 unsigned Tmp, Tmp2;
3417 unsigned FirstAnswer = 1;
3418
3419 if (auto *C
2.1
'C' is null
2.1
'C' is null
= dyn_cast<ConstantSDNode>(Op)) {
3
Taking false branch
3420 const APInt &Val = C->getAPIntValue();
3421 return Val.getNumSignBits();
3422 }
3423
3424 if (Depth >= MaxRecursionDepth)
4
Assuming 'Depth' is < 'MaxRecursionDepth'
5
Taking false branch
3425 return 1; // Limit search depth.
3426
3427 if (!DemandedElts)
6
Taking false branch
3428 return 1; // No demanded elts, better to assume we don't know anything.
3429
3430 unsigned Opcode = Op.getOpcode();
3431 switch (Opcode) {
7
Control jumps to 'case ADD:' at line 3673
3432 default: break;
3433 case ISD::AssertSext:
3434 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3435 return VTBits-Tmp+1;
3436 case ISD::AssertZext:
3437 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3438 return VTBits-Tmp;
3439
3440 case ISD::BUILD_VECTOR:
3441 Tmp = VTBits;
3442 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3443 if (!DemandedElts[i])
3444 continue;
3445
3446 SDValue SrcOp = Op.getOperand(i);
3447 Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
3448
3449 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3450 if (SrcOp.getValueSizeInBits() != VTBits) {
3451 assert(SrcOp.getValueSizeInBits() > VTBits &&((SrcOp.getValueSizeInBits() > VTBits && "Expected BUILD_VECTOR implicit truncation"
) ? static_cast<void> (0) : __assert_fail ("SrcOp.getValueSizeInBits() > VTBits && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3452, __PRETTY_FUNCTION__))
3452 "Expected BUILD_VECTOR implicit truncation")((SrcOp.getValueSizeInBits() > VTBits && "Expected BUILD_VECTOR implicit truncation"
) ? static_cast<void> (0) : __assert_fail ("SrcOp.getValueSizeInBits() > VTBits && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3452, __PRETTY_FUNCTION__))
;
3453 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3454 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3455 }
3456 Tmp = std::min(Tmp, Tmp2);
3457 }
3458 return Tmp;
3459
3460 case ISD::VECTOR_SHUFFLE: {
3461 // Collect the minimum number of sign bits that are shared by every vector
3462 // element referenced by the shuffle.
3463 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3464 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3465 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")((NumElts == SVN->getMask().size() && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("NumElts == SVN->getMask().size() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3465, __PRETTY_FUNCTION__))
;
3466 for (unsigned i = 0; i != NumElts; ++i) {
3467 int M = SVN->getMaskElt(i);
3468 if (!DemandedElts[i])
3469 continue;
3470 // For UNDEF elements, we don't know anything about the common state of
3471 // the shuffle result.
3472 if (M < 0)
3473 return 1;
3474 if ((unsigned)M < NumElts)
3475 DemandedLHS.setBit((unsigned)M % NumElts);
3476 else
3477 DemandedRHS.setBit((unsigned)M % NumElts);
3478 }
3479 Tmp = std::numeric_limits<unsigned>::max();
3480 if (!!DemandedLHS)
3481 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3482 if (!!DemandedRHS) {
3483 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3484 Tmp = std::min(Tmp, Tmp2);
3485 }
3486 // If we don't know anything, early out and try computeKnownBits fall-back.
3487 if (Tmp == 1)
3488 break;
3489 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? static_cast<void> (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3489, __PRETTY_FUNCTION__))
;
3490 return Tmp;
3491 }
3492
3493 case ISD::BITCAST: {
3494 SDValue N0 = Op.getOperand(0);
3495 EVT SrcVT = N0.getValueType();
3496 unsigned SrcBits = SrcVT.getScalarSizeInBits();
3497
3498 // Ignore bitcasts from unsupported types..
3499 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3500 break;
3501
3502 // Fast handling of 'identity' bitcasts.
3503 if (VTBits == SrcBits)
3504 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3505
3506 bool IsLE = getDataLayout().isLittleEndian();
3507
3508 // Bitcast 'large element' scalar/vector to 'small element' vector.
3509 if ((SrcBits % VTBits) == 0) {
3510 assert(VT.isVector() && "Expected bitcast to vector")((VT.isVector() && "Expected bitcast to vector") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Expected bitcast to vector\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3510, __PRETTY_FUNCTION__))
;
3511
3512 unsigned Scale = SrcBits / VTBits;
3513 APInt SrcDemandedElts(NumElts / Scale, 0);
3514 for (unsigned i = 0; i != NumElts; ++i)
3515 if (DemandedElts[i])
3516 SrcDemandedElts.setBit(i / Scale);
3517
3518 // Fast case - sign splat can be simply split across the small elements.
3519 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3520 if (Tmp == SrcBits)
3521 return VTBits;
3522
3523 // Slow case - determine how far the sign extends into each sub-element.
3524 Tmp2 = VTBits;
3525 for (unsigned i = 0; i != NumElts; ++i)
3526 if (DemandedElts[i]) {
3527 unsigned SubOffset = i % Scale;
3528 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3529 SubOffset = SubOffset * VTBits;
3530 if (Tmp <= SubOffset)
3531 return 1;
3532 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3533 }
3534 return Tmp2;
3535 }
3536 break;
3537 }
3538
3539 case ISD::SIGN_EXTEND:
3540 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3541 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3542 case ISD::SIGN_EXTEND_INREG:
3543 // Max of the input and what this extends.
3544 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3545 Tmp = VTBits-Tmp+1;
3546 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3547 return std::max(Tmp, Tmp2);
3548 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3549 SDValue Src = Op.getOperand(0);
3550 EVT SrcVT = Src.getValueType();
3551 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3552 Tmp = VTBits - SrcVT.getScalarSizeInBits();
3553 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3554 }
3555
3556 case ISD::SRA:
3557 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3558 // SRA X, C -> adds C sign bits.
3559 if (ConstantSDNode *C =
3560 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3561 APInt ShiftVal = C->getAPIntValue();
3562 ShiftVal += Tmp;
3563 Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
3564 }
3565 return Tmp;
3566 case ISD::SHL:
3567 if (ConstantSDNode *C =
3568 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3569 // shl destroys sign bits.
3570 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3571 if (C->getAPIntValue().uge(VTBits) || // Bad shift.
3572 C->getAPIntValue().uge(Tmp)) break; // Shifted all sign bits out.
3573 return Tmp - C->getZExtValue();
3574 }
3575 break;
3576 case ISD::AND:
3577 case ISD::OR:
3578 case ISD::XOR: // NOT is handled here.
3579 // Logical binary ops preserve the number of sign bits at the worst.
3580 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3581 if (Tmp != 1) {
3582 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3583 FirstAnswer = std::min(Tmp, Tmp2);
3584 // We computed what we know about the sign bits as our first
3585 // answer. Now proceed to the generic code that uses
3586 // computeKnownBits, and pick whichever answer is better.
3587 }
3588 break;
3589
3590 case ISD::SELECT:
3591 case ISD::VSELECT:
3592 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3593 if (Tmp == 1) return 1; // Early out.
3594 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3595 return std::min(Tmp, Tmp2);
3596 case ISD::SELECT_CC:
3597 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3598 if (Tmp == 1) return 1; // Early out.
3599 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3600 return std::min(Tmp, Tmp2);
3601
3602 case ISD::SMIN:
3603 case ISD::SMAX: {
3604 // If we have a clamp pattern, we know that the number of sign bits will be
3605 // the minimum of the clamp min/max range.
3606 bool IsMax = (Opcode == ISD::SMAX);
3607 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3608 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3609 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3610 CstHigh =
3611 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3612 if (CstLow && CstHigh) {
3613 if (!IsMax)
3614 std::swap(CstLow, CstHigh);
3615 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3616 Tmp = CstLow->getAPIntValue().getNumSignBits();
3617 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3618 return std::min(Tmp, Tmp2);
3619 }
3620 }
3621
3622 // Fallback - just get the minimum number of sign bits of the operands.
3623 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3624 if (Tmp == 1)
3625 return 1; // Early out.
3626 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3627 return std::min(Tmp, Tmp2);
3628 }
3629 case ISD::UMIN:
3630 case ISD::UMAX:
3631 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3632 if (Tmp == 1)
3633 return 1; // Early out.
3634 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3635 return std::min(Tmp, Tmp2);
3636 case ISD::SADDO:
3637 case ISD::UADDO:
3638 case ISD::SSUBO:
3639 case ISD::USUBO:
3640 case ISD::SMULO:
3641 case ISD::UMULO:
3642 if (Op.getResNo() != 1)
3643 break;
3644 // The boolean result conforms to getBooleanContents. Fall through.
3645 // If setcc returns 0/-1, all bits are sign bits.
3646 // We know that we have an integer-based boolean since these operations
3647 // are only available for integer.
3648 if (TLI->getBooleanContents(VT.isVector(), false) ==
3649 TargetLowering::ZeroOrNegativeOneBooleanContent)
3650 return VTBits;
3651 break;
3652 case ISD::SETCC:
3653 // If setcc returns 0/-1, all bits are sign bits.
3654 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3655 TargetLowering::ZeroOrNegativeOneBooleanContent)
3656 return VTBits;
3657 break;
3658 case ISD::ROTL:
3659 case ISD::ROTR:
3660 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
3661 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3662
3663 // Handle rotate right by N like a rotate left by 32-N.
3664 if (Opcode == ISD::ROTR)
3665 RotAmt = (VTBits - RotAmt) % VTBits;
3666
3667 // If we aren't rotating out all of the known-in sign bits, return the
3668 // number that are left. This handles rotl(sext(x), 1) for example.
3669 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3670 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3671 }
3672 break;
3673 case ISD::ADD:
3674 case ISD::ADDC:
3675 // Add can have at most one carry bit. Thus we know that the output
3676 // is, at worst, one more bit than the inputs.
3677 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3678 if (Tmp == 1) return 1; // Early out.
8
Assuming 'Tmp' is not equal to 1
9
Taking false branch
3679
3680 // Special case decrementing a value (ADD X, -1):
3681 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
10
Assuming 'CRHS' is null
11
Taking false branch
3682 if (CRHS->isAllOnesValue()) {
3683 KnownBits Known = computeKnownBits(Op.getOperand(0), Depth+1);
3684
3685 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3686 // sign bits set.
3687 if ((Known.Zero | 1).isAllOnesValue())
3688 return VTBits;
3689
3690 // If we are subtracting one from a positive number, there is no carry
3691 // out of the result.
3692 if (Known.isNonNegative())
3693 return Tmp;
3694 }
3695
3696 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
12
Value assigned to 'Op.Node'
13
Calling 'SelectionDAG::ComputeNumSignBits'
3697 if (Tmp2 == 1) return 1;
3698 return std::min(Tmp, Tmp2)-1;
3699
3700 case ISD::SUB:
3701 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3702 if (Tmp2 == 1) return 1;
3703
3704 // Handle NEG.
3705 if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
3706 if (CLHS->isNullValue()) {
3707 KnownBits Known = computeKnownBits(Op.getOperand(1), Depth+1);
3708 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3709 // sign bits set.
3710 if ((Known.Zero | 1).isAllOnesValue())
3711 return VTBits;
3712
3713 // If the input is known to be positive (the sign bit is known clear),
3714 // the output of the NEG has the same number of sign bits as the input.
3715 if (Known.isNonNegative())
3716 return Tmp2;
3717
3718 // Otherwise, we treat this like a SUB.
3719 }
3720
3721 // Sub can have at most one carry bit. Thus we know that the output
3722 // is, at worst, one more bit than the inputs.
3723 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3724 if (Tmp == 1) return 1; // Early out.
3725 return std::min(Tmp, Tmp2)-1;
3726 case ISD::MUL: {
3727 // The output of the Mul can be at most twice the valid bits in the inputs.
3728 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3729 if (SignBitsOp0 == 1)
3730 break;
3731 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3732 if (SignBitsOp1 == 1)
3733 break;
3734 unsigned OutValidBits =
3735 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
3736 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
3737 }
3738 case ISD::TRUNCATE: {
3739 // Check if the sign bits of source go down as far as the truncated value.
3740 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
3741 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3742 if (NumSrcSignBits > (NumSrcBits - VTBits))
3743 return NumSrcSignBits - (NumSrcBits - VTBits);
3744 break;
3745 }
3746 case ISD::EXTRACT_ELEMENT: {
3747 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3748 const int BitWidth = Op.getValueSizeInBits();
3749 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
3750
3751 // Get reverse index (starting from 1), Op1 value indexes elements from
3752 // little end. Sign starts at big end.
3753 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
3754
3755 // If the sign portion ends in our element the subtraction gives correct
3756 // result. Otherwise it gives either negative or > bitwidth result
3757 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
3758 }
3759 case ISD::INSERT_VECTOR_ELT: {
3760 SDValue InVec = Op.getOperand(0);
3761 SDValue InVal = Op.getOperand(1);
3762 SDValue EltNo = Op.getOperand(2);
3763
3764 ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3765 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3766 // If we know the element index, split the demand between the
3767 // source vector and the inserted element.
3768 unsigned EltIdx = CEltNo->getZExtValue();
3769
3770 // If we demand the inserted element then get its sign bits.
3771 Tmp = std::numeric_limits<unsigned>::max();
3772 if (DemandedElts[EltIdx]) {
3773 // TODO - handle implicit truncation of inserted elements.
3774 if (InVal.getScalarValueSizeInBits() != VTBits)
3775 break;
3776 Tmp = ComputeNumSignBits(InVal, Depth + 1);
3777 }
3778
3779 // If we demand the source vector then get its sign bits, and determine
3780 // the minimum.
3781 APInt VectorElts = DemandedElts;
3782 VectorElts.clearBit(EltIdx);
3783 if (!!VectorElts) {
3784 Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
3785 Tmp = std::min(Tmp, Tmp2);
3786 }
3787 } else {
3788 // Unknown element index, so ignore DemandedElts and demand them all.
3789 Tmp = ComputeNumSignBits(InVec, Depth + 1);
3790 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
3791 Tmp = std::min(Tmp, Tmp2);
3792 }
3793 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? static_cast<void> (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3793, __PRETTY_FUNCTION__))
;
3794 return Tmp;
3795 }
3796 case ISD::EXTRACT_VECTOR_ELT: {
3797 SDValue InVec = Op.getOperand(0);
3798 SDValue EltNo = Op.getOperand(1);
3799 EVT VecVT = InVec.getValueType();
3800 const unsigned BitWidth = Op.getValueSizeInBits();
3801 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
3802 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3803
3804 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
3805 // anything about sign bits. But if the sizes match we can derive knowledge
3806 // about sign bits from the vector operand.
3807 if (BitWidth != EltBitWidth)
3808 break;
3809
3810 // If we know the element index, just demand that vector element, else for
3811 // an unknown element index, ignore DemandedElts and demand them all.
3812 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3813 ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3814 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3815 DemandedSrcElts =
3816 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3817
3818 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
3819 }
3820 case ISD::EXTRACT_SUBVECTOR: {
3821 // If we know the element index, just demand that subvector elements,
3822 // otherwise demand them all.
3823 SDValue Src = Op.getOperand(0);
3824 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3825 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3826 APInt DemandedSrc = APInt::getAllOnesValue(NumSrcElts);
3827 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
3828 // Offset the demanded elts by the subvector index.
3829 uint64_t Idx = SubIdx->getZExtValue();
3830 DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
3831 }
3832 return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
3833 }
3834 case ISD::CONCAT_VECTORS: {
3835 // Determine the minimum number of sign bits across all demanded
3836 // elts of the input vectors. Early out if the result is already 1.
3837 Tmp = std::numeric_limits<unsigned>::max();
3838 EVT SubVectorVT = Op.getOperand(0).getValueType();
3839 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3840 unsigned NumSubVectors = Op.getNumOperands();
3841 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
3842 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
3843 DemandedSub = DemandedSub.trunc(NumSubVectorElts);
3844 if (!DemandedSub)
3845 continue;
3846 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
3847 Tmp = std::min(Tmp, Tmp2);
3848 }
3849 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? static_cast<void> (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3849, __PRETTY_FUNCTION__))
;
3850 return Tmp;
3851 }
3852 case ISD::INSERT_SUBVECTOR: {
3853 // If we know the element index, demand any elements from the subvector and
3854 // the remainder from the src its inserted into, otherwise demand them all.
3855 SDValue Src = Op.getOperand(0);
3856 SDValue Sub = Op.getOperand(1);
3857 auto *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3858 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3859 if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
3860 Tmp = std::numeric_limits<unsigned>::max();
3861 uint64_t Idx = SubIdx->getZExtValue();
3862 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3863 if (!!DemandedSubElts) {
3864 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
3865 if (Tmp == 1) return 1; // early-out
3866 }
3867 APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
3868 APInt DemandedSrcElts = DemandedElts & ~SubMask;
3869 if (!!DemandedSrcElts) {
3870 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
3871 Tmp = std::min(Tmp, Tmp2);
3872 }
3873 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? static_cast<void> (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3873, __PRETTY_FUNCTION__))
;
3874 return Tmp;
3875 }
3876
3877 // Not able to determine the index so just assume worst case.
3878 Tmp = ComputeNumSignBits(Sub, Depth + 1);
3879 if (Tmp == 1) return 1; // early-out
3880 Tmp2 = ComputeNumSignBits(Src, Depth + 1);
3881 Tmp = std::min(Tmp, Tmp2);
3882 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? static_cast<void> (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3882, __PRETTY_FUNCTION__))
;
3883 return Tmp;
3884 }
3885 }
3886
3887 // If we are looking at the loaded value of the SDNode.
3888 if (Op.getResNo() == 0) {
3889 // Handle LOADX separately here. EXTLOAD case will fallthrough.
3890 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
3891 unsigned ExtType = LD->getExtensionType();
3892 switch (ExtType) {
3893 default: break;
3894 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
3895 Tmp = LD->getMemoryVT().getScalarSizeInBits();
3896 return VTBits - Tmp + 1;
3897 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
3898 Tmp = LD->getMemoryVT().getScalarSizeInBits();
3899 return VTBits - Tmp;
3900 case ISD::NON_EXTLOAD:
3901 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
3902 // We only need to handle vectors - computeKnownBits should handle
3903 // scalar cases.
3904 Type *CstTy = Cst->getType();
3905 if (CstTy->isVectorTy() &&
3906 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
3907 Tmp = VTBits;
3908 for (unsigned i = 0; i != NumElts; ++i) {
3909 if (!DemandedElts[i])
3910 continue;
3911 if (Constant *Elt = Cst->getAggregateElement(i)) {
3912 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3913 const APInt &Value = CInt->getValue();
3914 Tmp = std::min(Tmp, Value.getNumSignBits());
3915 continue;
3916 }
3917 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3918 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3919 Tmp = std::min(Tmp, Value.getNumSignBits());
3920 continue;
3921 }
3922 }
3923 // Unknown type. Conservatively assume no bits match sign bit.
3924 return 1;
3925 }
3926 return Tmp;
3927 }
3928 }
3929 break;
3930 }
3931 }
3932 }
3933
3934 // Allow the target to implement this method for its nodes.
3935 if (Opcode >= ISD::BUILTIN_OP_END ||
3936 Opcode == ISD::INTRINSIC_WO_CHAIN ||
3937 Opcode == ISD::INTRINSIC_W_CHAIN ||
3938 Opcode == ISD::INTRINSIC_VOID) {
3939 unsigned NumBits =
3940 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
3941 if (NumBits > 1)
3942 FirstAnswer = std::max(FirstAnswer, NumBits);
3943 }
3944
3945 // Finally, if we can prove that the top bits of the result are 0's or 1's,
3946 // use this information.
3947 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
3948
3949 APInt Mask;
3950 if (Known.isNonNegative()) { // sign bit is 0
3951 Mask = Known.Zero;
3952 } else if (Known.isNegative()) { // sign bit is 1;
3953 Mask = Known.One;
3954 } else {
3955 // Nothing known.
3956 return FirstAnswer;
3957 }
3958
3959 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
3960 // the number of identical bits in the top of the input value.
3961 Mask = ~Mask;
3962 Mask <<= Mask.getBitWidth()-VTBits;
3963 // Return # leading zeros. We use 'min' here in case Val was zero before
3964 // shifting. We don't want to return '64' as for an i32 "0".
3965 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
3966}
3967
3968bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
3969 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
3970 !isa<ConstantSDNode>(Op.getOperand(1)))
3971 return false;
3972
3973 if (Op.getOpcode() == ISD::OR &&
3974 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
3975 return false;
3976
3977 return true;
3978}
3979
3980bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
3981 // If we're told that NaNs won't happen, assume they won't.
3982 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
3983 return true;
3984
3985 if (Depth >= MaxRecursionDepth)
3986 return false; // Limit search depth.
3987
3988 // TODO: Handle vectors.
3989 // If the value is a constant, we can obviously see if it is a NaN or not.
3990 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
3991 return !C->getValueAPF().isNaN() ||
3992 (SNaN && !C->getValueAPF().isSignaling());
3993 }
3994
3995 unsigned Opcode = Op.getOpcode();
3996 switch (Opcode) {
3997 case ISD::FADD:
3998 case ISD::FSUB:
3999 case ISD::FMUL:
4000 case ISD::FDIV:
4001 case ISD::FREM:
4002 case ISD::FSIN:
4003 case ISD::FCOS: {
4004 if (SNaN)
4005 return true;
4006 // TODO: Need isKnownNeverInfinity
4007 return false;
4008 }
4009 case ISD::FCANONICALIZE:
4010 case ISD::FEXP:
4011 case ISD::FEXP2:
4012 case ISD::FTRUNC:
4013 case ISD::FFLOOR:
4014 case ISD::FCEIL:
4015 case ISD::FROUND:
4016 case ISD::FRINT:
4017 case ISD::FNEARBYINT: {
4018 if (SNaN)
4019 return true;
4020 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4021 }
4022 case ISD::FABS:
4023 case ISD::FNEG:
4024 case ISD::FCOPYSIGN: {
4025 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4026 }
4027 case ISD::SELECT:
4028 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4029 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4030 case ISD::FP_EXTEND:
4031 case ISD::FP_ROUND: {
4032 if (SNaN)
4033 return true;
4034 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4035 }
4036 case ISD::SINT_TO_FP:
4037 case ISD::UINT_TO_FP:
4038 return true;
4039 case ISD::FMA:
4040 case ISD::FMAD: {
4041 if (SNaN)
4042 return true;
4043 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4044 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4045 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4046 }
4047 case ISD::FSQRT: // Need is known positive
4048 case ISD::FLOG:
4049 case ISD::FLOG2:
4050 case ISD::FLOG10:
4051 case ISD::FPOWI:
4052 case ISD::FPOW: {
4053 if (SNaN)
4054 return true;
4055 // TODO: Refine on operand
4056 return false;
4057 }
4058 case ISD::FMINNUM:
4059 case ISD::FMAXNUM: {
4060 // Only one needs to be known not-nan, since it will be returned if the
4061 // other ends up being one.
4062 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4063 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4064 }
4065 case ISD::FMINNUM_IEEE:
4066 case ISD::FMAXNUM_IEEE: {
4067 if (SNaN)
4068 return true;
4069 // This can return a NaN if either operand is an sNaN, or if both operands
4070 // are NaN.
4071 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4072 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4073 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4074 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4075 }
4076 case ISD::FMINIMUM:
4077 case ISD::FMAXIMUM: {
4078 // TODO: Does this quiet or return the origina NaN as-is?
4079 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4080 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4081 }
4082 case ISD::EXTRACT_VECTOR_ELT: {
4083 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4084 }
4085 default:
4086 if (Opcode >= ISD::BUILTIN_OP_END ||
4087 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4088 Opcode == ISD::INTRINSIC_W_CHAIN ||
4089 Opcode == ISD::INTRINSIC_VOID) {
4090 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4091 }
4092
4093 return false;
4094 }
4095}
4096
4097bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4098 assert(Op.getValueType().isFloatingPoint() &&((Op.getValueType().isFloatingPoint() && "Floating point type expected"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isFloatingPoint() && \"Floating point type expected\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4099, __PRETTY_FUNCTION__))
4099 "Floating point type expected")((Op.getValueType().isFloatingPoint() && "Floating point type expected"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isFloatingPoint() && \"Floating point type expected\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4099, __PRETTY_FUNCTION__))
;
4100
4101 // If the value is a constant, we can obviously see if it is a zero or not.
4102 // TODO: Add BuildVector support.
4103 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4104 return !C->isZero();
4105 return false;
4106}
4107
4108bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4109 assert(!Op.getValueType().isFloatingPoint() &&((!Op.getValueType().isFloatingPoint() && "Floating point types unsupported - use isKnownNeverZeroFloat"
) ? static_cast<void> (0) : __assert_fail ("!Op.getValueType().isFloatingPoint() && \"Floating point types unsupported - use isKnownNeverZeroFloat\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4110, __PRETTY_FUNCTION__))
4110 "Floating point types unsupported - use isKnownNeverZeroFloat")((!Op.getValueType().isFloatingPoint() && "Floating point types unsupported - use isKnownNeverZeroFloat"
) ? static_cast<void> (0) : __assert_fail ("!Op.getValueType().isFloatingPoint() && \"Floating point types unsupported - use isKnownNeverZeroFloat\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4110, __PRETTY_FUNCTION__))
;
4111
4112 // If the value is a constant, we can obviously see if it is a zero or not.
4113 if (ISD::matchUnaryPredicate(
4114 Op, [](ConstantSDNode *C) { return !C->isNullValue(); }))
4115 return true;
4116
4117 // TODO: Recognize more cases here.
4118 switch (Op.getOpcode()) {
4119 default: break;
4120 case ISD::OR:
4121 if (isKnownNeverZero(Op.getOperand(1)) ||
4122 isKnownNeverZero(Op.getOperand(0)))
4123 return true;
4124 break;
4125 }
4126
4127 return false;
4128}
4129
4130bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4131 // Check the obvious case.
4132 if (A == B) return true;
4133
4134 // For for negative and positive zero.
4135 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4136 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4137 if (CA->isZero() && CB->isZero()) return true;
4138
4139 // Otherwise they may not be equal.
4140 return false;
4141}
4142
4143// FIXME: unify with llvm::haveNoCommonBitsSet.
4144// FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4145bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4146 assert(A.getValueType() == B.getValueType() &&((A.getValueType() == B.getValueType() && "Values must have the same type"
) ? static_cast<void> (0) : __assert_fail ("A.getValueType() == B.getValueType() && \"Values must have the same type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4147, __PRETTY_FUNCTION__))
4147 "Values must have the same type")((A.getValueType() == B.getValueType() && "Values must have the same type"
) ? static_cast<void> (0) : __assert_fail ("A.getValueType() == B.getValueType() && \"Values must have the same type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4147, __PRETTY_FUNCTION__))
;
4148 return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
4149}
4150
4151static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4152 ArrayRef<SDValue> Ops,
4153 SelectionDAG &DAG) {
4154 int NumOps = Ops.size();
4155 assert(NumOps != 0 && "Can't build an empty vector!")((NumOps != 0 && "Can't build an empty vector!") ? static_cast
<void> (0) : __assert_fail ("NumOps != 0 && \"Can't build an empty vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4155, __PRETTY_FUNCTION__))
;
4156 assert(VT.getVectorNumElements() == (unsigned)NumOps &&((VT.getVectorNumElements() == (unsigned)NumOps && "Incorrect element count in BUILD_VECTOR!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)NumOps && \"Incorrect element count in BUILD_VECTOR!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4157, __PRETTY_FUNCTION__))
4157 "Incorrect element count in BUILD_VECTOR!")((VT.getVectorNumElements() == (unsigned)NumOps && "Incorrect element count in BUILD_VECTOR!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)NumOps && \"Incorrect element count in BUILD_VECTOR!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4157, __PRETTY_FUNCTION__))
;
4158
4159 // BUILD_VECTOR of UNDEFs is UNDEF.
4160 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4161 return DAG.getUNDEF(VT);
4162
4163 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4164 SDValue IdentitySrc;
4165 bool IsIdentity = true;
4166 for (int i = 0; i != NumOps; ++i) {
4167 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4168 Ops[i].getOperand(0).getValueType() != VT ||
4169 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
4170 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
4171 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) {
4172 IsIdentity = false;
4173 break;
4174 }
4175 IdentitySrc = Ops[i].getOperand(0);
4176 }
4177 if (IsIdentity)
4178 return IdentitySrc;
4179
4180 return SDValue();
4181}
4182
4183/// Try to simplify vector concatenation to an input value, undef, or build
4184/// vector.
4185static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4186 ArrayRef<SDValue> Ops,
4187 SelectionDAG &DAG) {
4188 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!")((!Ops.empty() && "Can't concatenate an empty list of vectors!"
) ? static_cast<void> (0) : __assert_fail ("!Ops.empty() && \"Can't concatenate an empty list of vectors!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4188, __PRETTY_FUNCTION__))
;
4189 assert(llvm::all_of(Ops,((llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType
() == Op.getValueType(); }) && "Concatenation of vectors with inconsistent value types!"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
4190 [Ops](SDValue Op) {((llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType
() == Op.getValueType(); }) && "Concatenation of vectors with inconsistent value types!"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
4191 return Ops[0].getValueType() == Op.getValueType();((llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType
() == Op.getValueType(); }) && "Concatenation of vectors with inconsistent value types!"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
4192 }) &&((llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType
() == Op.getValueType(); }) && "Concatenation of vectors with inconsistent value types!"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
4193 "Concatenation of vectors with inconsistent value types!")((llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType
() == Op.getValueType(); }) && "Concatenation of vectors with inconsistent value types!"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
;
4194 assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==(((Ops.size() * Ops[0].getValueType().getVectorNumElements())
== VT.getVectorNumElements() && "Incorrect element count in vector concatenation!"
) ? static_cast<void> (0) : __assert_fail ("(Ops.size() * Ops[0].getValueType().getVectorNumElements()) == VT.getVectorNumElements() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4196, __PRETTY_FUNCTION__))
4195 VT.getVectorNumElements() &&(((Ops.size() * Ops[0].getValueType().getVectorNumElements())
== VT.getVectorNumElements() && "Incorrect element count in vector concatenation!"
) ? static_cast<void> (0) : __assert_fail ("(Ops.size() * Ops[0].getValueType().getVectorNumElements()) == VT.getVectorNumElements() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4196, __PRETTY_FUNCTION__))
4196 "Incorrect element count in vector concatenation!")(((Ops.size() * Ops[0].getValueType().getVectorNumElements())
== VT.getVectorNumElements() && "Incorrect element count in vector concatenation!"
) ? static_cast<void> (0) : __assert_fail ("(Ops.size() * Ops[0].getValueType().getVectorNumElements()) == VT.getVectorNumElements() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4196, __PRETTY_FUNCTION__))
;
4197
4198 if (Ops.size() == 1)
4199 return Ops[0];
4200
4201 // Concat of UNDEFs is UNDEF.
4202 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4203 return DAG.getUNDEF(VT);
4204
4205 // Scan the operands and look for extract operations from a single source
4206 // that correspond to insertion at the same location via this concatenation:
4207 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4208 SDValue IdentitySrc;
4209 bool IsIdentity = true;
4210 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
4211 SDValue Op = Ops[i];
4212 unsigned IdentityIndex = i * Op.getValueType().getVectorNumElements();
4213 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4214 Op.getOperand(0).getValueType() != VT ||
4215 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
4216 !isa<ConstantSDNode>(Op.getOperand(1)) ||
4217 Op.getConstantOperandVal(1) != IdentityIndex) {
4218 IsIdentity = false;
4219 break;
4220 }
4221 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&(((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
"Unexpected identity source vector for concat of extracts") ?
static_cast<void> (0) : __assert_fail ("(!IdentitySrc || IdentitySrc == Op.getOperand(0)) && \"Unexpected identity source vector for concat of extracts\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4222, __PRETTY_FUNCTION__))
4222 "Unexpected identity source vector for concat of extracts")(((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
"Unexpected identity source vector for concat of extracts") ?
static_cast<void> (0) : __assert_fail ("(!IdentitySrc || IdentitySrc == Op.getOperand(0)) && \"Unexpected identity source vector for concat of extracts\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4222, __PRETTY_FUNCTION__))
;
4223 IdentitySrc = Op.getOperand(0);
4224 }
4225 if (IsIdentity) {
4226 assert(IdentitySrc && "Failed to set source vector of extracts")((IdentitySrc && "Failed to set source vector of extracts"
) ? static_cast<void> (0) : __assert_fail ("IdentitySrc && \"Failed to set source vector of extracts\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4226, __PRETTY_FUNCTION__))
;
4227 return IdentitySrc;
4228 }
4229
4230 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4231 // simplified to one big BUILD_VECTOR.
4232 // FIXME: Add support for SCALAR_TO_VECTOR as well.
4233 EVT SVT = VT.getScalarType();
4234 SmallVector<SDValue, 16> Elts;
4235 for (SDValue Op : Ops) {
4236 EVT OpVT = Op.getValueType();
4237 if (Op.isUndef())
4238 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4239 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4240 Elts.append(Op->op_begin(), Op->op_end());
4241 else
4242 return SDValue();
4243 }
4244
4245 // BUILD_VECTOR requires all inputs to be of the same type, find the
4246 // maximum type and extend them all.
4247 for (SDValue Op : Elts)
4248 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
4249
4250 if (SVT.bitsGT(VT.getScalarType()))
4251 for (SDValue &Op : Elts)
4252 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4253 ? DAG.getZExtOrTrunc(Op, DL, SVT)
4254 : DAG.getSExtOrTrunc(Op, DL, SVT);
4255
4256 SDValue V = DAG.getBuildVector(VT, DL, Elts);
4257 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4258 return V;
4259}
4260
4261/// Gets or creates the specified node.
4262SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4263 FoldingSetNodeID ID;
4264 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4265 void *IP = nullptr;
4266 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4267 return SDValue(E, 0);
4268
4269 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4270 getVTList(VT));
4271 CSEMap.InsertNode(N, IP);
4272
4273 InsertNode(N);
4274 SDValue V = SDValue(N, 0);
4275 NewSDValueDbgMsg(V, "Creating new node: ", this);
4276 return V;
4277}
4278
4279SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4280 SDValue Operand, const SDNodeFlags Flags) {
4281 // Constant fold unary operations with an integer constant operand. Even
4282 // opaque constant will be folded, because the folding of unary operations
4283 // doesn't create new constants with different values. Nevertheless, the
4284 // opaque flag is preserved during folding to prevent future folding with
4285 // other constants.
4286 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4287 const APInt &Val = C->getAPIntValue();
4288 switch (Opcode) {
4289 default: break;
4290 case ISD::SIGN_EXTEND:
4291 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4292 C->isTargetOpcode(), C->isOpaque());
4293 case ISD::TRUNCATE:
4294 if (C->isOpaque())
4295 break;
4296 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4297 case ISD::ANY_EXTEND:
4298 case ISD::ZERO_EXTEND:
4299 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4300 C->isTargetOpcode(), C->isOpaque());
4301 case ISD::UINT_TO_FP:
4302 case ISD::SINT_TO_FP: {
4303 APFloat apf(EVTToAPFloatSemantics(VT),
4304 APInt::getNullValue(VT.getSizeInBits()));
4305 (void)apf.convertFromAPInt(Val,
4306 Opcode==ISD::SINT_TO_FP,
4307 APFloat::rmNearestTiesToEven);
4308 return getConstantFP(apf, DL, VT);
4309 }
4310 case ISD::BITCAST:
4311 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
4312 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4313 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
4314 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4315 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
4316 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4317 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
4318 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4319 break;
4320 case ISD::ABS:
4321 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4322 C->isOpaque());
4323 case ISD::BITREVERSE:
4324 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4325 C->isOpaque());
4326 case ISD::BSWAP:
4327 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4328 C->isOpaque());
4329 case ISD::CTPOP:
4330 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4331 C->isOpaque());
4332 case ISD::CTLZ:
4333 case ISD::CTLZ_ZERO_UNDEF:
4334 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4335 C->isOpaque());
4336 case ISD::CTTZ:
4337 case ISD::CTTZ_ZERO_UNDEF:
4338 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4339 C->isOpaque());
4340 case ISD::FP16_TO_FP: {
4341 bool Ignored;
4342 APFloat FPV(APFloat::IEEEhalf(),
4343 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
4344
4345 // This can return overflow, underflow, or inexact; we don't care.
4346 // FIXME need to be more flexible about rounding mode.
4347 (void)FPV.convert(EVTToAPFloatSemantics(VT),
4348 APFloat::rmNearestTiesToEven, &Ignored);
4349 return getConstantFP(FPV, DL, VT);
4350 }
4351 }
4352 }
4353
4354 // Constant fold unary operations with a floating point constant operand.
4355 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4356 APFloat V = C->getValueAPF(); // make copy
4357 switch (Opcode) {
4358 case ISD::FNEG:
4359 V.changeSign();
4360 return getConstantFP(V, DL, VT);
4361 case ISD::FABS:
4362 V.clearSign();
4363 return getConstantFP(V, DL, VT);
4364 case ISD::FCEIL: {
4365 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4366 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4367 return getConstantFP(V, DL, VT);
4368 break;
4369 }
4370 case ISD::FTRUNC: {
4371 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4372 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4373 return getConstantFP(V, DL, VT);
4374 break;
4375 }
4376 case ISD::FFLOOR: {
4377 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4378 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4379 return getConstantFP(V, DL, VT);
4380 break;
4381 }
4382 case ISD::FP_EXTEND: {
4383 bool ignored;
4384 // This can return overflow, underflow, or inexact; we don't care.
4385 // FIXME need to be more flexible about rounding mode.
4386 (void)V.convert(EVTToAPFloatSemantics(VT),
4387 APFloat::rmNearestTiesToEven, &ignored);
4388 return getConstantFP(V, DL, VT);
4389 }
4390 case ISD::FP_TO_SINT:
4391 case ISD::FP_TO_UINT: {
4392 bool ignored;
4393 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4394 // FIXME need to be more flexible about rounding mode.
4395 APFloat::opStatus s =
4396 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4397 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4398 break;
4399 return getConstant(IntVal, DL, VT);
4400 }
4401 case ISD::BITCAST:
4402 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
4403 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4404 else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
4405 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4406 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
4407 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4408 break;
4409 case ISD::FP_TO_FP16: {
4410 bool Ignored;
4411 // This can return overflow, underflow, or inexact; we don't care.
4412 // FIXME need to be more flexible about rounding mode.
4413 (void)V.convert(APFloat::IEEEhalf(),
4414 APFloat::rmNearestTiesToEven, &Ignored);
4415 return getConstant(V.bitcastToAPInt(), DL, VT);
4416 }
4417 }
4418 }
4419
4420 // Constant fold unary operations with a vector integer or float operand.
4421 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
4422 if (BV->isConstant()) {
4423 switch (Opcode) {
4424 default:
4425 // FIXME: Entirely reasonable to perform folding of other unary
4426 // operations here as the need arises.
4427 break;
4428 case ISD::FNEG:
4429 case ISD::FABS:
4430 case ISD::FCEIL:
4431 case ISD::FTRUNC:
4432 case ISD::FFLOOR:
4433 case ISD::FP_EXTEND:
4434 case ISD::FP_TO_SINT:
4435 case ISD::FP_TO_UINT:
4436 case ISD::TRUNCATE:
4437 case ISD::ANY_EXTEND:
4438 case ISD::ZERO_EXTEND:
4439 case ISD::SIGN_EXTEND:
4440 case ISD::UINT_TO_FP:
4441 case ISD::SINT_TO_FP:
4442 case ISD::ABS:
4443 case ISD::BITREVERSE:
4444 case ISD::BSWAP:
4445 case ISD::CTLZ:
4446 case ISD::CTLZ_ZERO_UNDEF:
4447 case ISD::CTTZ:
4448 case ISD::CTTZ_ZERO_UNDEF:
4449 case ISD::CTPOP: {
4450 SDValue Ops = { Operand };
4451 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4452 return Fold;
4453 }
4454 }
4455 }
4456 }
4457
4458 unsigned OpOpcode = Operand.getNode()->getOpcode();
4459 switch (Opcode) {
4460 case ISD::TokenFactor:
4461 case ISD::MERGE_VALUES:
4462 case ISD::CONCAT_VECTORS:
4463 return Operand; // Factor, merge or concat of one node? No need.
4464 case ISD::BUILD_VECTOR: {
4465 // Attempt to simplify BUILD_VECTOR.
4466 SDValue Ops[] = {Operand};
4467 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4468 return V;
4469 break;
4470 }
4471 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node")::llvm::llvm_unreachable_internal("Invalid method to make FP_ROUND node"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4471)
;
4472 case ISD::FP_EXTEND:
4473 assert(VT.isFloatingPoint() &&((VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint
() && "Invalid FP cast!") ? static_cast<void> (
0) : __assert_fail ("VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4474, __PRETTY_FUNCTION__))
4474 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!")((VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint
() && "Invalid FP cast!") ? static_cast<void> (
0) : __assert_fail ("VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4474, __PRETTY_FUNCTION__))
;
4475 if (Operand.getValueType() == VT) return Operand; // noop conversion.
4476 assert((!VT.isVector() ||(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4479, __PRETTY_FUNCTION__))
4477 VT.getVectorNumElements() ==(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4479, __PRETTY_FUNCTION__))
4478 Operand.getValueType().getVectorNumElements()) &&(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4479, __PRETTY_FUNCTION__))
4479 "Vector element count mismatch!")(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4479, __PRETTY_FUNCTION__))
;
4480 assert(Operand.getValueType().bitsLT(VT) &&((Operand.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid fpext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4481, __PRETTY_FUNCTION__))
4481 "Invalid fpext node, dst < src!")((Operand.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid fpext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4481, __PRETTY_FUNCTION__))
;
4482 if (Operand.isUndef())
4483 return getUNDEF(VT);
4484 break;
4485 case ISD::FP_TO_SINT:
4486 case ISD::FP_TO_UINT:
4487 if (Operand.isUndef())
4488 return getUNDEF(VT);
4489 break;
4490 case ISD::SINT_TO_FP:
4491 case ISD::UINT_TO_FP:
4492 // [us]itofp(undef) = 0, because the result value is bounded.
4493 if (Operand.isUndef())
4494 return getConstantFP(0.0, DL, VT);
4495 break;
4496 case ISD::SIGN_EXTEND:
4497 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid SIGN_EXTEND!") ? static_cast<void>
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid SIGN_EXTEND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4498, __PRETTY_FUNCTION__))
4498 "Invalid SIGN_EXTEND!")((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid SIGN_EXTEND!") ? static_cast<void>
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid SIGN_EXTEND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4498, __PRETTY_FUNCTION__))
;
4499 assert(VT.isVector() == Operand.getValueType().isVector() &&((VT.isVector() == Operand.getValueType().isVector() &&
"SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4501, __PRETTY_FUNCTION__))
4500 "SIGN_EXTEND result type type should be vector iff the operand "((VT.isVector() == Operand.getValueType().isVector() &&
"SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4501, __PRETTY_FUNCTION__))
4501 "type is vector!")((VT.isVector() == Operand.getValueType().isVector() &&
"SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4501, __PRETTY_FUNCTION__))
;
4502 if (Operand.getValueType() == VT) return Operand; // noop extension
4503 assert((!VT.isVector() ||(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4506, __PRETTY_FUNCTION__))
4504 VT.getVectorNumElements() ==(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4506, __PRETTY_FUNCTION__))
4505 Operand.getValueType().getVectorNumElements()) &&(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4506, __PRETTY_FUNCTION__))
4506 "Vector element count mismatch!")(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4506, __PRETTY_FUNCTION__))
;
4507 assert(Operand.getValueType().bitsLT(VT) &&((Operand.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid sext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4508, __PRETTY_FUNCTION__))
4508 "Invalid sext node, dst < src!")((Operand.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid sext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4508, __PRETTY_FUNCTION__))
;
4509 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4510 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4511 else if (OpOpcode == ISD::UNDEF)
4512 // sext(undef) = 0, because the top bits will all be the same.
4513 return getConstant(0, DL, VT);
4514 break;
4515 case ISD::ZERO_EXTEND:
4516 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid ZERO_EXTEND!") ? static_cast<void>
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ZERO_EXTEND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4517, __PRETTY_FUNCTION__))
4517 "Invalid ZERO_EXTEND!")((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid ZERO_EXTEND!") ? static_cast<void>
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ZERO_EXTEND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4517, __PRETTY_FUNCTION__))
;
4518 assert(VT.isVector() == Operand.getValueType().isVector() &&((VT.isVector() == Operand.getValueType().isVector() &&
"ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4520, __PRETTY_FUNCTION__))
4519 "ZERO_EXTEND result type type should be vector iff the operand "((VT.isVector() == Operand.getValueType().isVector() &&
"ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4520, __PRETTY_FUNCTION__))
4520 "type is vector!")((VT.isVector() == Operand.getValueType().isVector() &&
"ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4520, __PRETTY_FUNCTION__))
;
4521 if (Operand.getValueType() == VT) return Operand; // noop extension
4522 assert((!VT.isVector() ||(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4525, __PRETTY_FUNCTION__))
4523 VT.getVectorNumElements() ==(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4525, __PRETTY_FUNCTION__))
4524 Operand.getValueType().getVectorNumElements()) &&(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4525, __PRETTY_FUNCTION__))
4525 "Vector element count mismatch!")(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4525, __PRETTY_FUNCTION__))
;
4526 assert(Operand.getValueType().bitsLT(VT) &&((Operand.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid zext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4527, __PRETTY_FUNCTION__))
4527 "Invalid zext node, dst < src!")((Operand.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid zext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4527, __PRETTY_FUNCTION__))
;
4528 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
4529 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4530 else if (OpOpcode == ISD::UNDEF)
4531 // zext(undef) = 0, because the top bits will be zero.
4532 return getConstant(0, DL, VT);
4533 break;
4534 case ISD::ANY_EXTEND:
4535 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid ANY_EXTEND!") ? static_cast<void>
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ANY_EXTEND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4536, __PRETTY_FUNCTION__))
4536 "Invalid ANY_EXTEND!")((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid ANY_EXTEND!") ? static_cast<void>
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ANY_EXTEND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4536, __PRETTY_FUNCTION__))
;
4537 assert(VT.isVector() == Operand.getValueType().isVector() &&((VT.isVector() == Operand.getValueType().isVector() &&
"ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4539, __PRETTY_FUNCTION__))
4538 "ANY_EXTEND result type type should be vector iff the operand "((VT.isVector() == Operand.getValueType().isVector() &&
"ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4539, __PRETTY_FUNCTION__))
4539 "type is vector!")((VT.isVector() == Operand.getValueType().isVector() &&
"ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4539, __PRETTY_FUNCTION__))
;
4540 if (Operand.getValueType() == VT) return Operand; // noop extension
4541 assert((!VT.isVector() ||(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4544, __PRETTY_FUNCTION__))
4542 VT.getVectorNumElements() ==(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4544, __PRETTY_FUNCTION__))
4543 Operand.getValueType().getVectorNumElements()) &&(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4544, __PRETTY_FUNCTION__))
4544 "Vector element count mismatch!")(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4544, __PRETTY_FUNCTION__))
;
4545 assert(Operand.getValueType().bitsLT(VT) &&((Operand.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid anyext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4546, __PRETTY_FUNCTION__))
4546 "Invalid anyext node, dst < src!")((Operand.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLT(VT) && \"Invalid anyext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4546, __PRETTY_FUNCTION__))
;
4547
4548 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4549 OpOpcode == ISD::ANY_EXTEND)
4550 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
4551 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4552 else if (OpOpcode == ISD::UNDEF)
4553 return getUNDEF(VT);
4554
4555 // (ext (trunc x)) -> x
4556 if (OpOpcode == ISD::TRUNCATE) {
4557 SDValue OpOp = Operand.getOperand(0);
4558 if (OpOp.getValueType() == VT) {
4559 transferDbgValues(Operand, OpOp);
4560 return OpOp;
4561 }
4562 }
4563 break;
4564 case ISD::TRUNCATE:
4565 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid TRUNCATE!") ? static_cast<void> (
0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid TRUNCATE!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4566, __PRETTY_FUNCTION__))
4566 "Invalid TRUNCATE!")((VT.isInteger() && Operand.getValueType().isInteger(
) && "Invalid TRUNCATE!") ? static_cast<void> (
0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid TRUNCATE!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4566, __PRETTY_FUNCTION__))
;
4567 assert(VT.isVector() == Operand.getValueType().isVector() &&((VT.isVector() == Operand.getValueType().isVector() &&
"TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4569, __PRETTY_FUNCTION__))
4568 "TRUNCATE result type type should be vector iff the operand "((VT.isVector() == Operand.getValueType().isVector() &&
"TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4569, __PRETTY_FUNCTION__))
4569 "type is vector!")((VT.isVector() == Operand.getValueType().isVector() &&
"TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4569, __PRETTY_FUNCTION__))
;
4570 if (Operand.getValueType() == VT) return Operand; // noop truncate
4571 assert((!VT.isVector() ||(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4574, __PRETTY_FUNCTION__))
4572 VT.getVectorNumElements() ==(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4574, __PRETTY_FUNCTION__))
4573 Operand.getValueType().getVectorNumElements()) &&(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4574, __PRETTY_FUNCTION__))
4574 "Vector element count mismatch!")(((!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType
().getVectorNumElements()) && "Vector element count mismatch!"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4574, __PRETTY_FUNCTION__))
;
4575 assert(Operand.getValueType().bitsGT(VT) &&((Operand.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsGT(VT) && \"Invalid truncate node, src < dst!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4576, __PRETTY_FUNCTION__))
4576 "Invalid truncate node, src < dst!")((Operand.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!"
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsGT(VT) && \"Invalid truncate node, src < dst!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4576, __PRETTY_FUNCTION__))
;
4577 if (OpOpcode == ISD::TRUNCATE)
4578 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4579 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4580 OpOpcode == ISD::ANY_EXTEND) {
4581 // If the source is smaller than the dest, we still need an extend.
4582 if (Operand.getOperand(0).getValueType().getScalarType()
4583 .bitsLT(VT.getScalarType()))
4584 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4585 if (Operand.getOperand(0).getValueType().bitsGT(VT))
4586 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4587 return Operand.getOperand(0);
4588 }
4589 if (OpOpcode == ISD::UNDEF)
4590 return getUNDEF(VT);
4591 break;
4592 case ISD::ANY_EXTEND_VECTOR_INREG:
4593 case ISD::ZERO_EXTEND_VECTOR_INREG:
4594 case ISD::SIGN_EXTEND_VECTOR_INREG:
4595 assert(VT.isVector() && "This DAG node is restricted to vector types.")((VT.isVector() && "This DAG node is restricted to vector types."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"This DAG node is restricted to vector types.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4595, __PRETTY_FUNCTION__))
;
4596 assert(Operand.getValueType().bitsLE(VT) &&((Operand.getValueType().bitsLE(VT) && "The input must be the same size or smaller than the result."
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLE(VT) && \"The input must be the same size or smaller than the result.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4597, __PRETTY_FUNCTION__))
4597 "The input must be the same size or smaller than the result.")((Operand.getValueType().bitsLE(VT) && "The input must be the same size or smaller than the result."
) ? static_cast<void> (0) : __assert_fail ("Operand.getValueType().bitsLE(VT) && \"The input must be the same size or smaller than the result.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4597, __PRETTY_FUNCTION__))
;
4598 assert(VT.getVectorNumElements() <((VT.getVectorNumElements() < Operand.getValueType().getVectorNumElements
() && "The destination vector type must have fewer lanes than the input."
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() < Operand.getValueType().getVectorNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4600, __PRETTY_FUNCTION__))
4599 Operand.getValueType().getVectorNumElements() &&((VT.getVectorNumElements() < Operand.getValueType().getVectorNumElements
() && "The destination vector type must have fewer lanes than the input."
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() < Operand.getValueType().getVectorNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4600, __PRETTY_FUNCTION__))
4600 "The destination vector type must have fewer lanes than the input.")((VT.getVectorNumElements() < Operand.getValueType().getVectorNumElements
() && "The destination vector type must have fewer lanes than the input."
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() < Operand.getValueType().getVectorNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4600, __PRETTY_FUNCTION__))
;
4601 break;
4602 case ISD::ABS:
4603 assert(VT.isInteger() && VT == Operand.getValueType() &&((VT.isInteger() && VT == Operand.getValueType() &&
"Invalid ABS!") ? static_cast<void> (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid ABS!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4604, __PRETTY_FUNCTION__))
4604 "Invalid ABS!")((VT.isInteger() && VT == Operand.getValueType() &&
"Invalid ABS!") ? static_cast<void> (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid ABS!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4604, __PRETTY_FUNCTION__))
;
4605 if (OpOpcode == ISD::UNDEF)
4606 return getUNDEF(VT);
4607 break;
4608 case ISD::BSWAP:
4609 assert(VT.isInteger() && VT == Operand.getValueType() &&((VT.isInteger() && VT == Operand.getValueType() &&
"Invalid BSWAP!") ? static_cast<void> (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BSWAP!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4610, __PRETTY_FUNCTION__))
4610 "Invalid BSWAP!")((VT.isInteger() && VT == Operand.getValueType() &&
"Invalid BSWAP!") ? static_cast<void> (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BSWAP!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4610, __PRETTY_FUNCTION__))
;
4611 assert((VT.getScalarSizeInBits() % 16 == 0) &&(((VT.getScalarSizeInBits() % 16 == 0) && "BSWAP types must be a multiple of 16 bits!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getScalarSizeInBits() % 16 == 0) && \"BSWAP types must be a multiple of 16 bits!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4612, __PRETTY_FUNCTION__))
4612 "BSWAP types must be a multiple of 16 bits!")(((VT.getScalarSizeInBits() % 16 == 0) && "BSWAP types must be a multiple of 16 bits!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getScalarSizeInBits() % 16 == 0) && \"BSWAP types must be a multiple of 16 bits!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4612, __PRETTY_FUNCTION__))
;
4613 if (OpOpcode == ISD::UNDEF)
4614 return getUNDEF(VT);
4615 break;
4616 case ISD::BITREVERSE:
4617 assert(VT.isInteger() && VT == Operand.getValueType() &&((VT.isInteger() && VT == Operand.getValueType() &&
"Invalid BITREVERSE!") ? static_cast<void> (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BITREVERSE!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4618, __PRETTY_FUNCTION__))
4618 "Invalid BITREVERSE!")((VT.isInteger() && VT == Operand.getValueType() &&
"Invalid BITREVERSE!") ? static_cast<void> (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BITREVERSE!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4618, __PRETTY_FUNCTION__))
;
4619 if (OpOpcode == ISD::UNDEF)
4620 return getUNDEF(VT);
4621 break;
4622 case ISD::BITCAST:
4623 // Basic sanity checking.
4624 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&((VT.getSizeInBits() == Operand.getValueSizeInBits() &&
"Cannot BITCAST between types of different sizes!") ? static_cast
<void> (0) : __assert_fail ("VT.getSizeInBits() == Operand.getValueSizeInBits() && \"Cannot BITCAST between types of different sizes!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4625, __PRETTY_FUNCTION__))
4625 "Cannot BITCAST between types of different sizes!")((VT.getSizeInBits() == Operand.getValueSizeInBits() &&
"Cannot BITCAST between types of different sizes!") ? static_cast
<void> (0) : __assert_fail ("VT.getSizeInBits() == Operand.getValueSizeInBits() && \"Cannot BITCAST between types of different sizes!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4625, __PRETTY_FUNCTION__))
;
4626 if (VT == Operand.getValueType()) return Operand; // noop conversion.
4627 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
4628 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
4629 if (OpOpcode == ISD::UNDEF)
4630 return getUNDEF(VT);
4631 break;
4632 case ISD::SCALAR_TO_VECTOR:
4633 assert(VT.isVector() && !Operand.getValueType().isVector() &&((VT.isVector() && !Operand.getValueType().isVector()
&& (VT.getVectorElementType() == Operand.getValueType
() || (VT.getVectorElementType().isInteger() && Operand
.getValueType().isInteger() && VT.getVectorElementType
().bitsLE(Operand.getValueType()))) && "Illegal SCALAR_TO_VECTOR node!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4638, __PRETTY_FUNCTION__))
4634 (VT.getVectorElementType() == Operand.getValueType() ||((VT.isVector() && !Operand.getValueType().isVector()
&& (VT.getVectorElementType() == Operand.getValueType
() || (VT.getVectorElementType().isInteger() && Operand
.getValueType().isInteger() && VT.getVectorElementType
().bitsLE(Operand.getValueType()))) && "Illegal SCALAR_TO_VECTOR node!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4638, __PRETTY_FUNCTION__))
4635 (VT.getVectorElementType().isInteger() &&((VT.isVector() && !Operand.getValueType().isVector()
&& (VT.getVectorElementType() == Operand.getValueType
() || (VT.getVectorElementType().isInteger() && Operand
.getValueType().isInteger() && VT.getVectorElementType
().bitsLE(Operand.getValueType()))) && "Illegal SCALAR_TO_VECTOR node!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4638, __PRETTY_FUNCTION__))
4636 Operand.getValueType().isInteger() &&((VT.isVector() && !Operand.getValueType().isVector()
&& (VT.getVectorElementType() == Operand.getValueType
() || (VT.getVectorElementType().isInteger() && Operand
.getValueType().isInteger() && VT.getVectorElementType
().bitsLE(Operand.getValueType()))) && "Illegal SCALAR_TO_VECTOR node!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4638, __PRETTY_FUNCTION__))
4637 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&((VT.isVector() && !Operand.getValueType().isVector()
&& (VT.getVectorElementType() == Operand.getValueType
() || (VT.getVectorElementType().isInteger() && Operand
.getValueType().isInteger() && VT.getVectorElementType
().bitsLE(Operand.getValueType()))) && "Illegal SCALAR_TO_VECTOR node!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4638, __PRETTY_FUNCTION__))
4638 "Illegal SCALAR_TO_VECTOR node!")((VT.isVector() && !Operand.getValueType().isVector()
&& (VT.getVectorElementType() == Operand.getValueType
() || (VT.getVectorElementType().isInteger() && Operand
.getValueType().isInteger() && VT.getVectorElementType
().bitsLE(Operand.getValueType()))) && "Illegal SCALAR_TO_VECTOR node!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4638, __PRETTY_FUNCTION__))
;
4639 if (OpOpcode == ISD::UNDEF)
4640 return getUNDEF(VT);
4641 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
4642 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
4643 isa<ConstantSDNode>(Operand.getOperand(1)) &&
4644 Operand.getConstantOperandVal(1) == 0 &&
4645 Operand.getOperand(0).getValueType() == VT)
4646 return Operand.getOperand(0);
4647 break;
4648 case ISD::FNEG:
4649 // Negation of an unknown bag of bits is still completely undefined.
4650 if (OpOpcode == ISD::UNDEF)
4651 return getUNDEF(VT);
4652
4653 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
4654 if ((getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) &&
4655 OpOpcode == ISD::FSUB)
4656 return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
4657 Operand.getOperand(0), Flags);
4658 if (OpOpcode == ISD::FNEG) // --X -> X
4659 return Operand.getOperand(0);
4660 break;
4661 case ISD::FABS:
4662 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
4663 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
4664 break;
4665 }
4666
4667 SDNode *N;
4668 SDVTList VTs = getVTList(VT);
4669 SDValue Ops[] = {Operand};
4670 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
4671 FoldingSetNodeID ID;
4672 AddNodeIDNode(ID, Opcode, VTs, Ops);
4673 void *IP = nullptr;
4674 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
4675 E->intersectFlagsWith(Flags);
4676 return SDValue(E, 0);
4677 }
4678
4679 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4680 N->setFlags(Flags);
4681 createOperands(N, Ops);
4682 CSEMap.InsertNode(N, IP);
4683 } else {
4684 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4685 createOperands(N, Ops);
4686 }
4687
4688 InsertNode(N);
4689 SDValue V = SDValue(N, 0);
4690 NewSDValueDbgMsg(V, "Creating new node: ", this);
4691 return V;
4692}
4693
4694static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
4695 const APInt &C2) {
4696 switch (Opcode) {
4697 case ISD::ADD: return std::make_pair(C1 + C2, true);
4698 case ISD::SUB: return std::make_pair(C1 - C2, true);
4699 case ISD::MUL: return std::make_pair(C1 * C2, true);
4700 case ISD::AND: return std::make_pair(C1 & C2, true);
4701 case ISD::OR: return std::make_pair(C1 | C2, true);
4702 case ISD::XOR: return std::make_pair(C1 ^ C2, true);
4703 case ISD::SHL: return std::make_pair(C1 << C2, true);
4704 case ISD::SRL: return std::make_pair(C1.lshr(C2), true);
4705 case ISD::SRA: return std::make_pair(C1.ashr(C2), true);
4706 case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
4707 case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
4708 case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
4709 case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
4710 case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
4711 case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
4712 case ISD::SADDSAT: return std::make_pair(C1.sadd_sat(C2), true);
4713 case ISD::UADDSAT: return std::make_pair(C1.uadd_sat(C2), true);
4714 case ISD::SSUBSAT: return std::make_pair(C1.ssub_sat(C2), true);
4715 case ISD::USUBSAT: return std::make_pair(C1.usub_sat(C2), true);
4716 case ISD::UDIV:
4717 if (!C2.getBoolValue())
4718 break;
4719 return std::make_pair(C1.udiv(C2), true);
4720 case ISD::UREM:
4721 if (!C2.getBoolValue())
4722 break;
4723 return std::make_pair(C1.urem(C2), true);
4724 case ISD::SDIV:
4725 if (!C2.getBoolValue())
4726 break;
4727 return std::make_pair(C1.sdiv(C2), true);
4728 case ISD::SREM:
4729 if (!C2.getBoolValue())
4730 break;
4731 return std::make_pair(C1.srem(C2), true);
4732 }
4733 return std::make_pair(APInt(1, 0), false);
4734}
4735
4736SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4737 EVT VT, const ConstantSDNode *C1,
4738 const ConstantSDNode *C2) {
4739 if (C1->isOpaque() || C2->isOpaque())
4740 return SDValue();
4741
4742 std::pair<APInt, bool> Folded = FoldValue(Opcode, C1->getAPIntValue(),
4743 C2->getAPIntValue());
4744 if (!Folded.second)
4745 return SDValue();
4746 return getConstant(Folded.first, DL, VT);
4747}
4748
4749SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
4750 const GlobalAddressSDNode *GA,
4751 const SDNode *N2) {
4752 if (GA->getOpcode() != ISD::GlobalAddress)
4753 return SDValue();
4754 if (!TLI->isOffsetFoldingLegal(GA))
4755 return SDValue();
4756 auto *C2 = dyn_cast<ConstantSDNode>(N2);
4757 if (!C2)
4758 return SDValue();
4759 int64_t Offset = C2->getSExtValue();
4760 switch (Opcode) {
4761 case ISD::ADD: break;
4762 case ISD::SUB: Offset = -uint64_t(Offset); break;
4763 default: return SDValue();
4764 }
4765 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
4766 GA->getOffset() + uint64_t(Offset));
4767}
4768
4769bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
4770 switch (Opcode) {
4771 case ISD::SDIV:
4772 case ISD::UDIV:
4773 case ISD::SREM:
4774 case ISD::UREM: {
4775 // If a divisor is zero/undef or any element of a divisor vector is
4776 // zero/undef, the whole op is undef.
4777 assert(Ops.size() == 2 && "Div/rem should have 2 operands")((Ops.size() == 2 && "Div/rem should have 2 operands"
) ? static_cast<void> (0) : __assert_fail ("Ops.size() == 2 && \"Div/rem should have 2 operands\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4777, __PRETTY_FUNCTION__))
;
4778 SDValue Divisor = Ops[1];
4779 if (Divisor.isUndef() || isNullConstant(Divisor))
4780 return true;
4781
4782 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
4783 llvm::any_of(Divisor->op_values(),
4784 [](SDValue V) { return V.isUndef() ||
4785 isNullConstant(V); });
4786 // TODO: Handle signed overflow.
4787 }
4788 // TODO: Handle oversized shifts.
4789 default:
4790 return false;
4791 }
4792}
4793
4794SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4795 EVT VT, SDNode *N1, SDNode *N2) {
4796 // If the opcode is a target-specific ISD node, there's nothing we can
4797 // do here and the operand rules may not line up with the below, so
4798 // bail early.
4799 if (Opcode >= ISD::BUILTIN_OP_END)
4800 return SDValue();
4801
4802 if (isUndef(Opcode, {SDValue(N1, 0), SDValue(N2, 0)}))
4803 return getUNDEF(VT);
4804
4805 // Handle the case of two scalars.
4806 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
4807 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
4808 SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, C1, C2);
4809 assert((!Folded || !VT.isVector()) &&(((!Folded || !VT.isVector()) && "Can't fold vectors ops with scalar operands"
) ? static_cast<void> (0) : __assert_fail ("(!Folded || !VT.isVector()) && \"Can't fold vectors ops with scalar operands\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4810, __PRETTY_FUNCTION__))
4810 "Can't fold vectors ops with scalar operands")(((!Folded || !VT.isVector()) && "Can't fold vectors ops with scalar operands"
) ? static_cast<void> (0) : __assert_fail ("(!Folded || !VT.isVector()) && \"Can't fold vectors ops with scalar operands\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4810, __PRETTY_FUNCTION__))
;
4811 return Folded;
4812 }
4813 }
4814
4815 // fold (add Sym, c) -> Sym+c
4816 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
4817 return FoldSymbolOffset(Opcode, VT, GA, N2);
4818 if (TLI->isCommutativeBinOp(Opcode))
4819 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
4820 return FoldSymbolOffset(Opcode, VT, GA, N1);
4821
4822 // For vectors, extract each constant element and fold them individually.
4823 // Either input may be an undef value.
4824 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
4825 if (!BV1 && !N1->isUndef())
4826 return SDValue();
4827 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
4828 if (!BV2 && !N2->isUndef())
4829 return SDValue();
4830 // If both operands are undef, that's handled the same way as scalars.
4831 if (!BV1 && !BV2)
4832 return SDValue();
4833
4834 assert((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) &&(((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands
()) && "Vector binop with different number of elements in operands?"
) ? static_cast<void> (0) : __assert_fail ("(!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) && \"Vector binop with different number of elements in operands?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4835, __PRETTY_FUNCTION__))
4835 "Vector binop with different number of elements in operands?")(((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands
()) && "Vector binop with different number of elements in operands?"
) ? static_cast<void> (0) : __assert_fail ("(!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) && \"Vector binop with different number of elements in operands?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4835, __PRETTY_FUNCTION__))
;
4836
4837 EVT SVT = VT.getScalarType();
4838 EVT LegalSVT = SVT;
4839 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4840 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4841 if (LegalSVT.bitsLT(SVT))
4842 return SDValue();
4843 }
4844 SmallVector<SDValue, 4> Outputs;
4845 unsigned NumOps = BV1 ? BV1->getNumOperands() : BV2->getNumOperands();
4846 for (unsigned I = 0; I != NumOps; ++I) {
4847 SDValue V1 = BV1 ? BV1->getOperand(I) : getUNDEF(SVT);
4848 SDValue V2 = BV2 ? BV2->getOperand(I) : getUNDEF(SVT);
4849 if (SVT.isInteger()) {
4850 if (V1->getValueType(0).bitsGT(SVT))
4851 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
4852 if (V2->getValueType(0).bitsGT(SVT))
4853 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
4854 }
4855
4856 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
4857 return SDValue();
4858
4859 // Fold one vector element.
4860 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4861 if (LegalSVT != SVT)
4862 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4863
4864 // Scalar folding only succeeded if the result is a constant or UNDEF.
4865 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
4866 ScalarResult.getOpcode() != ISD::ConstantFP)
4867 return SDValue();
4868 Outputs.push_back(ScalarResult);
4869 }
4870
4871 assert(VT.getVectorNumElements() == Outputs.size() &&((VT.getVectorNumElements() == Outputs.size() && "Vector size mismatch!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == Outputs.size() && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4872, __PRETTY_FUNCTION__))
4872 "Vector size mismatch!")((VT.getVectorNumElements() == Outputs.size() && "Vector size mismatch!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == Outputs.size() && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4872, __PRETTY_FUNCTION__))
;
4873
4874 // We may have a vector type but a scalar result. Create a splat.
4875 Outputs.resize(VT.getVectorNumElements(), Outputs.back());
4876
4877 // Build a big vector out of the scalar elements we generated.
4878 return getBuildVector(VT, SDLoc(), Outputs);
4879}
4880
4881// TODO: Merge with FoldConstantArithmetic
4882SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4883 const SDLoc &DL, EVT VT,
4884 ArrayRef<SDValue> Ops,
4885 const SDNodeFlags Flags) {
4886 // If the opcode is a target-specific ISD node, there's nothing we can
4887 // do here and the operand rules may not line up with the below, so
4888 // bail early.
4889 if (Opcode >= ISD::BUILTIN_OP_END)
4890 return SDValue();
4891
4892 if (isUndef(Opcode, Ops))
4893 return getUNDEF(VT);
4894
4895 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
4896 if (!VT.isVector())
4897 return SDValue();
4898
4899 unsigned NumElts = VT.getVectorNumElements();
4900
4901 auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
4902 return !Op.getValueType().isVector() ||
4903 Op.getValueType().getVectorNumElements() == NumElts;
4904 };
4905
4906 auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
4907 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
4908 return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
4909 (BV && BV->isConstant());
4910 };
4911
4912 // All operands must be vector types with the same number of elements as
4913 // the result type and must be either UNDEF or a build vector of constant
4914 // or UNDEF scalars.
4915 if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
4916 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
4917 return SDValue();
4918
4919 // If we are comparing vectors, then the result needs to be a i1 boolean
4920 // that is then sign-extended back to the legal result type.
4921 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
4922
4923 // Find legal integer scalar type for constant promotion and
4924 // ensure that its scalar size is at least as large as source.
4925 EVT LegalSVT = VT.getScalarType();
4926 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4927 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4928 if (LegalSVT.bitsLT(VT.getScalarType()))
4929 return SDValue();
4930 }
4931
4932 // Constant fold each scalar lane separately.
4933 SmallVector<SDValue, 4> ScalarResults;
4934 for (unsigned i = 0; i != NumElts; i++) {
4935 SmallVector<SDValue, 4> ScalarOps;
4936 for (SDValue Op : Ops) {
4937 EVT InSVT = Op.getValueType().getScalarType();
4938 BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
4939 if (!InBV) {
4940 // We've checked that this is UNDEF or a constant of some kind.
4941 if (Op.isUndef())
4942 ScalarOps.push_back(getUNDEF(InSVT));
4943 else
4944 ScalarOps.push_back(Op);
4945 continue;
4946 }
4947
4948 SDValue ScalarOp = InBV->getOperand(i);
4949 EVT ScalarVT = ScalarOp.getValueType();
4950
4951 // Build vector (integer) scalar operands may need implicit
4952 // truncation - do this before constant folding.
4953 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
4954 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
4955
4956 ScalarOps.push_back(ScalarOp);
4957 }
4958
4959 // Constant fold the scalar operands.
4960 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
4961
4962 // Legalize the (integer) scalar constant if necessary.
4963 if (LegalSVT != SVT)
4964 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4965
4966 // Scalar folding only succeeded if the result is a constant or UNDEF.
4967 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
4968 ScalarResult.getOpcode() != ISD::ConstantFP)
4969 return SDValue();
4970 ScalarResults.push_back(ScalarResult);
4971 }
4972
4973 SDValue V = getBuildVector(VT, DL, ScalarResults);
4974 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
4975 return V;
4976}
4977
4978SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
4979 EVT VT, SDValue N1, SDValue N2) {
4980 // TODO: We don't do any constant folding for strict FP opcodes here, but we
4981 // should. That will require dealing with a potentially non-default
4982 // rounding mode, checking the "opStatus" return value from the APFloat
4983 // math calculations, and possibly other variations.
4984 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
4985 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
4986 if (N1CFP && N2CFP) {
4987 APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
4988 switch (Opcode) {
4989 case ISD::FADD:
4990 C1.add(C2, APFloat::rmNearestTiesToEven);
4991 return getConstantFP(C1, DL, VT);
4992 case ISD::FSUB:
4993 C1.subtract(C2, APFloat::rmNearestTiesToEven);
4994 return getConstantFP(C1, DL, VT);
4995 case ISD::FMUL:
4996 C1.multiply(C2, APFloat::rmNearestTiesToEven);
4997 return getConstantFP(C1, DL, VT);
4998 case ISD::FDIV:
4999 C1.divide(C2, APFloat::rmNearestTiesToEven);
5000 return getConstantFP(C1, DL, VT);
5001 case ISD::FREM:
5002 C1.mod(C2);
5003 return getConstantFP(C1, DL, VT);
5004 case ISD::FCOPYSIGN:
5005 C1.copySign(C2);
5006 return getConstantFP(C1, DL, VT);
5007 default: break;
5008 }
5009 }
5010 if (N1CFP && Opcode == ISD::FP_ROUND) {
5011 APFloat C1 = N1CFP->getValueAPF(); // make copy
5012 bool Unused;
5013 // This can return overflow, underflow, or inexact; we don't care.
5014 // FIXME need to be more flexible about rounding mode.
5015 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
5016 &Unused);
5017 return getConstantFP(C1, DL, VT);
5018 }
5019
5020 switch (Opcode) {
5021 case ISD::FADD:
5022 case ISD::FSUB:
5023 case ISD::FMUL:
5024 case ISD::FDIV:
5025 case ISD::FREM:
5026 // If both operands are undef, the result is undef. If 1 operand is undef,
5027 // the result is NaN. This should match the behavior of the IR optimizer.
5028 if (N1.isUndef() && N2.isUndef())
5029 return getUNDEF(VT);
5030 if (N1.isUndef() || N2.isUndef())
5031 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
5032 }
5033 return SDValue();
5034}
5035
5036SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5037 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5038 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5039 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
5040 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5041 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5042
5043 // Canonicalize constant to RHS if commutative.
5044 if (TLI->isCommutativeBinOp(Opcode)) {
5045 if (N1C && !N2C) {
5046 std::swap(N1C, N2C);
5047 std::swap(N1, N2);
5048 } else if (N1CFP && !N2CFP) {
5049 std::swap(N1CFP, N2CFP);
5050 std::swap(N1, N2);
5051 }
5052 }
5053
5054 switch (Opcode) {
5055 default: break;
5056 case ISD::TokenFactor:
5057 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&((VT == MVT::Other && N1.getValueType() == MVT::Other
&& N2.getValueType() == MVT::Other && "Invalid token factor!"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::Other && N1.getValueType() == MVT::Other && N2.getValueType() == MVT::Other && \"Invalid token factor!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5058, __PRETTY_FUNCTION__))
5058 N2.getValueType() == MVT::Other && "Invalid token factor!")((VT == MVT::Other && N1.getValueType() == MVT::Other
&& N2.getValueType() == MVT::Other && "Invalid token factor!"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::Other && N1.getValueType() == MVT::Other && N2.getValueType() == MVT::Other && \"Invalid token factor!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5058, __PRETTY_FUNCTION__))
;
5059 // Fold trivial token factors.
5060 if (N1.getOpcode() == ISD::EntryToken) return N2;
5061 if (N2.getOpcode() == ISD::EntryToken) return N1;
5062 if (N1 == N2) return N1;
5063 break;
5064 case ISD::BUILD_VECTOR: {
5065 // Attempt to simplify BUILD_VECTOR.
5066 SDValue Ops[] = {N1, N2};
5067 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5068 return V;
5069 break;
5070 }
5071 case ISD::CONCAT_VECTORS: {
5072 SDValue Ops[] = {N1, N2};
5073 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5074 return V;
5075 break;
5076 }
5077 case ISD::AND:
5078 assert(VT.isInteger() && "This operator does not apply to FP types!")((VT.isInteger() && "This operator does not apply to FP types!"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5078, __PRETTY_FUNCTION__))
;
5079 assert(N1.getValueType() == N2.getValueType() &&((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5080, __PRETTY_FUNCTION__))
5080 N1.getValueType() == VT && "Binary operator types must match!")((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5080, __PRETTY_FUNCTION__))
;
5081 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
5082 // worth handling here.
5083 if (N2C && N2C->isNullValue())
5084 return N2;
5085 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
5086 return N1;
5087 break;
5088 case ISD::OR:
5089 case ISD::XOR:
5090 case ISD::ADD:
5091 case ISD::SUB:
5092 assert(VT.isInteger() && "This operator does not apply to FP types!")((VT.isInteger() && "This operator does not apply to FP types!"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5092, __PRETTY_FUNCTION__))
;
5093 assert(N1.getValueType() == N2.getValueType() &&((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5094, __PRETTY_FUNCTION__))
5094 N1.getValueType() == VT && "Binary operator types must match!")((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5094, __PRETTY_FUNCTION__))
;
5095 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
5096 // it's worth handling here.
5097 if (N2C && N2C->isNullValue())
5098 return N1;
5099 break;
5100 case ISD::UDIV:
5101 case ISD::UREM:
5102 case ISD::MULHU:
5103 case ISD::MULHS:
5104 case ISD::MUL:
5105 case ISD::SDIV:
5106 case ISD::SREM:
5107 case ISD::SMIN:
5108 case ISD::SMAX:
5109 case ISD::UMIN:
5110 case ISD::UMAX:
5111 case ISD::SADDSAT:
5112 case ISD::SSUBSAT:
5113 case ISD::UADDSAT:
5114 case ISD::USUBSAT:
5115 assert(VT.isInteger() && "This operator does not apply to FP types!")((VT.isInteger() && "This operator does not apply to FP types!"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5115, __PRETTY_FUNCTION__))
;
5116 assert(N1.getValueType() == N2.getValueType() &&((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5117, __PRETTY_FUNCTION__))
5117 N1.getValueType() == VT && "Binary operator types must match!")((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5117, __PRETTY_FUNCTION__))
;
5118 break;
5119 case ISD::FADD:
5120 case ISD::FSUB:
5121 case ISD::FMUL:
5122 case ISD::FDIV:
5123 case ISD::FREM:
5124 assert(VT.isFloatingPoint() && "This operator only applies to FP types!")((VT.isFloatingPoint() && "This operator only applies to FP types!"
) ? static_cast<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"This operator only applies to FP types!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5124, __PRETTY_FUNCTION__))
;
5125 assert(N1.getValueType() == N2.getValueType() &&((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5126, __PRETTY_FUNCTION__))
5126 N1.getValueType() == VT && "Binary operator types must match!")((N1.getValueType() == N2.getValueType() && N1.getValueType
() == VT && "Binary operator types must match!") ? static_cast
<void> (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5126, __PRETTY_FUNCTION__))
;
5127 if (SDValue V = simplifyFPBinop(Opcode, N1, N2))
5128 return V;
5129 break;
5130 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
5131 assert(N1.getValueType() == VT &&((N1.getValueType() == VT && N1.getValueType().isFloatingPoint
() && N2.getValueType().isFloatingPoint() && "Invalid FCOPYSIGN!"
) ? static_cast<void> (0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5134, __PRETTY_FUNCTION__))
5132 N1.getValueType().isFloatingPoint() &&((N1.getValueType() == VT && N1.getValueType().isFloatingPoint
() && N2.getValueType().isFloatingPoint() && "Invalid FCOPYSIGN!"
) ? static_cast<void> (0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5134, __PRETTY_FUNCTION__))
5133 N2.getValueType().isFloatingPoint() &&((N1.getValueType() == VT && N1.getValueType().isFloatingPoint
() && N2.getValueType().isFloatingPoint() && "Invalid FCOPYSIGN!"
) ? static_cast<void> (0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5134, __PRETTY_FUNCTION__))
5134 "Invalid FCOPYSIGN!")((N1.getValueType() == VT && N1.getValueType().isFloatingPoint
() && N2.getValueType().isFloatingPoint() && "Invalid FCOPYSIGN!"
) ? static_cast<void> (0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5134, __PRETTY_FUNCTION__))
;
5135 break;
5136 case ISD::SHL:
5137 case ISD::SRA:
5138 case ISD::SRL:
5139 if (SDValue V = simplifyShift(N1, N2))
5140 return V;
5141 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5142 case ISD::ROTL:
5143 case ISD::ROTR:
5144 assert(VT == N1.getValueType() &&((VT == N1.getValueType() && "Shift operators return type must be the same as their first arg"
) ? static_cast<void> (0) : __assert_fail ("VT == N1.getValueType() && \"Shift operators return type must be the same as their first arg\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5145, __PRETTY_FUNCTION__))
5145 "Shift operators return type must be the same as their first arg")((VT == N1.getValueType() && "Shift operators return type must be the same as their first arg"
) ? static_cast<void> (0) : __assert_fail ("VT == N1.getValueType() && \"Shift operators return type must be the same as their first arg\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5145, __PRETTY_FUNCTION__))
;
5146 assert(VT.isInteger() && N2.getValueType().isInteger() &&((VT.isInteger() && N2.getValueType().isInteger() &&
"Shifts only work on integers") ? static_cast<void> (0
) : __assert_fail ("VT.isInteger() && N2.getValueType().isInteger() && \"Shifts only work on integers\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5147, __PRETTY_FUNCTION__))
5147 "Shifts only work on integers")((VT.isInteger() && N2.getValueType().isInteger() &&
"Shifts only work on integers") ? static_cast<void> (0
) : __assert_fail ("VT.isInteger() && N2.getValueType().isInteger() && \"Shifts only work on integers\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5147, __PRETTY_FUNCTION__))
;
5148 assert((!VT.isVector() || VT == N2.getValueType()) &&(((!VT.isVector() || VT == N2.getValueType()) && "Vector shift amounts must be in the same as their first arg"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT == N2.getValueType()) && \"Vector shift amounts must be in the same as their first arg\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5149, __PRETTY_FUNCTION__))
5149 "Vector shift amounts must be in the same as their first arg")(((!VT.isVector() || VT == N2.getValueType()) && "Vector shift amounts must be in the same as their first arg"
) ? static_cast<void> (0) : __assert_fail ("(!VT.isVector() || VT == N2.getValueType()) && \"Vector shift amounts must be in the same as their first arg\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5149, __PRETTY_FUNCTION__))
;
5150 // Verify that the shift amount VT is big enough to hold valid shift
5151 // amounts. This catches things like trying to shift an i1024 value by an
5152 // i8, which is easy to fall into in generic code that uses
5153 // TLI.getShiftAmount().
5154 assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&((N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits
()) && "Invalid use of small shift amount with oversized value!"
) ? static_cast<void> (0) : __assert_fail ("N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) && \"Invalid use of small shift amount with oversized value!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5155, __PRETTY_FUNCTION__))
5155 "Invalid use of small shift amount with oversized value!")((N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits
()) && "Invalid use of small shift amount with oversized value!"
) ? static_cast<void> (0) : __assert_fail ("N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) && \"Invalid use of small shift amount with oversized value!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5155, __PRETTY_FUNCTION__))
;
5156
5157 // Always fold shifts of i1 values so the code generator doesn't need to
5158 // handle them. Since we know the size of the shift has to be less than the
5159 // size of the value, the shift/rotate count is guaranteed to be zero.
5160 if (VT == MVT::i1)
5161 return N1;
5162 if (N2C && N2C->isNullValue())
5163 return N1;
5164 break;
5165 case ISD::FP_ROUND:
5166 assert(VT.isFloatingPoint() &&((VT.isFloatingPoint() && N1.getValueType().isFloatingPoint
() && VT.bitsLE(N1.getValueType()) && N2C &&
(N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1)
&& "Invalid FP_ROUND!") ? static_cast<void> (0
) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5170, __PRETTY_FUNCTION__))
5167 N1.getValueType().isFloatingPoint() &&((VT.isFloatingPoint() && N1.getValueType().isFloatingPoint
() && VT.bitsLE(N1.getValueType()) && N2C &&
(N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1)
&& "Invalid FP_ROUND!") ? static_cast<void> (0
) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5170, __PRETTY_FUNCTION__))
5168 VT.bitsLE(N1.getValueType()) &&((VT.isFloatingPoint() && N1.getValueType().isFloatingPoint
() && VT.bitsLE(N1.getValueType()) && N2C &&
(N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1)
&& "Invalid FP_ROUND!") ? static_cast<void> (0
) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5170, __PRETTY_FUNCTION__))
5169 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&((VT.isFloatingPoint() && N1.getValueType().isFloatingPoint
() && VT.bitsLE(N1.getValueType()) && N2C &&
(N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1)
&& "Invalid FP_ROUND!") ? static_cast<void> (0
) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5170, __PRETTY_FUNCTION__))
5170 "Invalid FP_ROUND!")((VT.isFloatingPoint() && N1.getValueType().isFloatingPoint
() && VT.bitsLE(N1.getValueType()) && N2C &&
(N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1)
&& "Invalid FP_ROUND!") ? static_cast<void> (0
) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5170, __PRETTY_FUNCTION__))
;
5171 if (N1.getValueType() == VT) return N1; // noop conversion.
5172 break;
5173 case ISD::AssertSext:
5174 case ISD::AssertZext: {
5175 EVT EVT = cast<VTSDNode>(N2)->getVT();
5176 assert(VT == N1.getValueType() && "Not an inreg extend!")((VT == N1.getValueType() && "Not an inreg extend!") ?
static_cast<void> (0) : __assert_fail ("VT == N1.getValueType() && \"Not an inreg extend!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5176, __PRETTY_FUNCTION__))
;
5177 assert(VT.isInteger() && EVT.isInteger() &&((VT.isInteger() && EVT.isInteger() && "Cannot *_EXTEND_INREG FP types"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5178, __PRETTY_FUNCTION__))
5178 "Cannot *_EXTEND_INREG FP types")((VT.isInteger() && EVT.isInteger() && "Cannot *_EXTEND_INREG FP types"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5178, __PRETTY_FUNCTION__))
;
5179 assert(!EVT.isVector() &&((!EVT.isVector() && "AssertSExt/AssertZExt type should be the vector element type "
"rather than the vector type!") ? static_cast<void> (0
) : __assert_fail ("!EVT.isVector() && \"AssertSExt/AssertZExt type should be the vector element type \" \"rather than the vector type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5181, __PRETTY_FUNCTION__))
5180 "AssertSExt/AssertZExt type should be the vector element type "((!EVT.isVector() && "AssertSExt/AssertZExt type should be the vector element type "
"rather than the vector type!") ? static_cast<void> (0
) : __assert_fail ("!EVT.isVector() && \"AssertSExt/AssertZExt type should be the vector element type \" \"rather than the vector type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5181, __PRETTY_FUNCTION__))
5181 "rather than the vector type!")((!EVT.isVector() && "AssertSExt/AssertZExt type should be the vector element type "
"rather than the vector type!") ? static_cast<void> (0
) : __assert_fail ("!EVT.isVector() && \"AssertSExt/AssertZExt type should be the vector element type \" \"rather than the vector type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5181, __PRETTY_FUNCTION__))
;
5182 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!")((EVT.bitsLE(VT.getScalarType()) && "Not extending!")
? static_cast<void> (0) : __assert_fail ("EVT.bitsLE(VT.getScalarType()) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5182, __PRETTY_FUNCTION__))
;
5183 if (VT.getScalarType() == EVT) return N1; // noop assertion.
5184 break;
5185 }
5186 case ISD::SIGN_EXTEND_INREG: {
5187 EVT EVT = cast<VTSDNode>(N2)->getVT();
5188 assert(VT == N1.getValueType() && "Not an inreg extend!")((VT == N1.getValueType() && "Not an inreg extend!") ?
static_cast<void> (0) : __assert_fail ("VT == N1.getValueType() && \"Not an inreg extend!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5188, __PRETTY_FUNCTION__))
;
5189 assert(VT.isInteger() && EVT.isInteger() &&((VT.isInteger() && EVT.isInteger() && "Cannot *_EXTEND_INREG FP types"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5190, __PRETTY_FUNCTION__))
5190 "Cannot *_EXTEND_INREG FP types")((VT.isInteger() && EVT.isInteger() && "Cannot *_EXTEND_INREG FP types"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5190, __PRETTY_FUNCTION__))
;
5191 assert(EVT.isVector() == VT.isVector() &&((EVT.isVector() == VT.isVector() && "SIGN_EXTEND_INREG type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("EVT.isVector() == VT.isVector() && \"SIGN_EXTEND_INREG type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5193, __PRETTY_FUNCTION__))
5192 "SIGN_EXTEND_INREG type should be vector iff the operand "((EVT.isVector() == VT.isVector() && "SIGN_EXTEND_INREG type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("EVT.isVector() == VT.isVector() && \"SIGN_EXTEND_INREG type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5193, __PRETTY_FUNCTION__))
5193 "type is vector!")((EVT.isVector() == VT.isVector() && "SIGN_EXTEND_INREG type should be vector iff the operand "
"type is vector!") ? static_cast<void> (0) : __assert_fail
("EVT.isVector() == VT.isVector() && \"SIGN_EXTEND_INREG type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5193, __PRETTY_FUNCTION__))
;
5194 assert((!EVT.isVector() ||(((!EVT.isVector() || EVT.getVectorNumElements() == VT.getVectorNumElements
()) && "Vector element counts must match in SIGN_EXTEND_INREG"
) ? static_cast<void> (0) : __assert_fail ("(!EVT.isVector() || EVT.getVectorNumElements() == VT.getVectorNumElements()) && \"Vector element counts must match in SIGN_EXTEND_INREG\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5196, __PRETTY_FUNCTION__))
5195 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&(((!EVT.isVector() || EVT.getVectorNumElements() == VT.getVectorNumElements
()) && "Vector element counts must match in SIGN_EXTEND_INREG"
) ? static_cast<void> (0) : __assert_fail ("(!EVT.isVector() || EVT.getVectorNumElements() == VT.getVectorNumElements()) && \"Vector element counts must match in SIGN_EXTEND_INREG\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5196, __PRETTY_FUNCTION__))
5196 "Vector element counts must match in SIGN_EXTEND_INREG")(((!EVT.isVector() || EVT.getVectorNumElements() == VT.getVectorNumElements
()) && "Vector element counts must match in SIGN_EXTEND_INREG"
) ? static_cast<void> (0) : __assert_fail ("(!EVT.isVector() || EVT.getVectorNumElements() == VT.getVectorNumElements()) && \"Vector element counts must match in SIGN_EXTEND_INREG\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5196, __PRETTY_FUNCTION__))
;
5197 assert(EVT.bitsLE(VT) && "Not extending!")((EVT.bitsLE(VT) && "Not extending!") ? static_cast<
void> (0) : __assert_fail ("EVT.bitsLE(VT) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5197, __PRETTY_FUNCTION__))
;
5198 if (EVT == VT) return N1; // Not actually extending
5199
5200 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
5201 unsigned FromBits = EVT.getScalarSizeInBits();
5202 Val <<= Val.getBitWidth() - FromBits;
5203 Val.ashrInPlace(Val.getBitWidth() - FromBits);
5204 return getConstant(Val, DL, ConstantVT);
5205 };
5206
5207 if (N1C) {
5208 const APInt &Val = N1C->getAPIntValue();
5209 return SignExtendInReg(Val, VT);
5210 }
5211 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
5212 SmallVector<SDValue, 8> Ops;
5213 llvm::EVT OpVT = N1.getOperand(0).getValueType();
5214 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
5215 SDValue Op = N1.getOperand(i);
5216 if (Op.isUndef()) {
5217 Ops.push_back(getUNDEF(OpVT));
5218 continue;
5219 }
5220 ConstantSDNode *C = cast<ConstantSDNode>(Op);
5221 APInt Val = C->getAPIntValue();
5222 Ops.push_back(SignExtendInReg(Val, OpVT));
5223 }
5224 return getBuildVector(VT, DL, Ops);
5225 }
5226 break;
5227 }
5228 case ISD::EXTRACT_VECTOR_ELT:
5229 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&((VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits
() && "The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && \"The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5231, __PRETTY_FUNCTION__))
5230 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \((VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits
() && "The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && \"The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5231, __PRETTY_FUNCTION__))
5231 element type of the vector.")((VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits
() && "The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && \"The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5231, __PRETTY_FUNCTION__))
;
5232
5233 // Extract from an undefined value or using an undefined index is undefined.
5234 if (N1.isUndef() || N2.isUndef())
5235 return getUNDEF(VT);
5236
5237 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
5238 if (N2C && N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
5239 return getUNDEF(VT);
5240
5241 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
5242 // expanding copies of large vectors from registers.
5243 if (N2C &&
5244 N1.getOpcode() == ISD::CONCAT_VECTORS &&
5245 N1.getNumOperands() > 0) {
5246 unsigned Factor =
5247 N1.getOperand(0).getValueType().getVectorNumElements();
5248 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
5249 N1.getOperand(N2C->getZExtValue() / Factor),
5250 getConstant(N2C->getZExtValue() % Factor, DL,
5251 N2.getValueType()));
5252 }
5253
5254 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
5255 // expanding large vector constants.
5256 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
5257 SDValue Elt = N1.getOperand(N2C->getZExtValue());
5258
5259 if (VT != Elt.getValueType())
5260 // If the vector element type is not legal, the BUILD_VECTOR operands
5261 // are promoted and implicitly truncated, and the result implicitly
5262 // extended. Make that explicit here.
5263 Elt = getAnyExtOrTrunc(Elt, DL, VT);
5264
5265 return Elt;
5266 }
5267
5268 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
5269 // operations are lowered to scalars.
5270 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
5271 // If the indices are the same, return the inserted element else
5272 // if the indices are known different, extract the element from
5273 // the original vector.
5274 SDValue N1Op2 = N1.getOperand(2);
5275 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
5276
5277 if (N1Op2C && N2C) {
5278 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
5279 if (VT == N1.getOperand(1).getValueType())
5280 return N1.getOperand(1);
5281 else
5282 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
5283 }
5284
5285 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
5286 }
5287 }
5288
5289 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
5290 // when vector types are scalarized and v1iX is legal.
5291 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx)
5292 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5293 N1.getValueType().getVectorNumElements() == 1) {
5294 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
5295 N1.getOperand(1));
5296 }
5297 break;
5298 case ISD::EXTRACT_ELEMENT:
5299 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!")((N2C && (unsigned)N2C->getZExtValue() < 2 &&
"Bad EXTRACT_ELEMENT!") ? static_cast<void> (0) : __assert_fail
("N2C && (unsigned)N2C->getZExtValue() < 2 && \"Bad EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5299, __PRETTY_FUNCTION__))
;
5300 assert(!N1.getValueType().isVector() && !VT.isVector() &&((!N1.getValueType().isVector() && !VT.isVector() &&
(N1.getValueType().isInteger() == VT.isInteger()) &&
N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? static_cast<void> (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5303, __PRETTY_FUNCTION__))
5301 (N1.getValueType().isInteger() == VT.isInteger()) &&((!N1.getValueType().isVector() && !VT.isVector() &&
(N1.getValueType().isInteger() == VT.isInteger()) &&
N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? static_cast<void> (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5303, __PRETTY_FUNCTION__))
5302 N1.getValueType() != VT &&((!N1.getValueType().isVector() && !VT.isVector() &&
(N1.getValueType().isInteger() == VT.isInteger()) &&
N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? static_cast<void> (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5303, __PRETTY_FUNCTION__))
5303 "Wrong types for EXTRACT_ELEMENT!")((!N1.getValueType().isVector() && !VT.isVector() &&
(N1.getValueType().isInteger() == VT.isInteger()) &&
N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? static_cast<void> (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5303, __PRETTY_FUNCTION__))
;
5304
5305 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
5306 // 64-bit integers into 32-bit parts. Instead of building the extract of
5307 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
5308 if (N1.getOpcode() == ISD::BUILD_PAIR)
5309 return N1.getOperand(N2C->getZExtValue());
5310
5311 // EXTRACT_ELEMENT of a constant int is also very common.
5312 if (N1C) {
5313 unsigned ElementSize = VT.getSizeInBits();
5314 unsigned Shift = ElementSize * N2C->getZExtValue();
5315 APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
5316 return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
5317 }
5318 break;
5319 case ISD::EXTRACT_SUBVECTOR:
5320 if (VT.isSimple() && N1.getValueType().isSimple()) {
5321 assert(VT.isVector() && N1.getValueType().isVector() &&((VT.isVector() && N1.getValueType().isVector() &&
"Extract subvector VTs must be a vectors!") ? static_cast<
void> (0) : __assert_fail ("VT.isVector() && N1.getValueType().isVector() && \"Extract subvector VTs must be a vectors!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5322, __PRETTY_FUNCTION__))
5322 "Extract subvector VTs must be a vectors!")((VT.isVector() && N1.getValueType().isVector() &&
"Extract subvector VTs must be a vectors!") ? static_cast<
void> (0) : __assert_fail ("VT.isVector() && N1.getValueType().isVector() && \"Extract subvector VTs must be a vectors!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5322, __PRETTY_FUNCTION__))
;
5323 assert(VT.getVectorElementType() ==((VT.getVectorElementType() == N1.getValueType().getVectorElementType
() && "Extract subvector VTs must have the same element type!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == N1.getValueType().getVectorElementType() && \"Extract subvector VTs must have the same element type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5325, __PRETTY_FUNCTION__))
5324 N1.getValueType().getVectorElementType() &&((VT.getVectorElementType() == N1.getValueType().getVectorElementType
() && "Extract subvector VTs must have the same element type!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == N1.getValueType().getVectorElementType() && \"Extract subvector VTs must have the same element type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5325, __PRETTY_FUNCTION__))
5325 "Extract subvector VTs must have the same element type!")((VT.getVectorElementType() == N1.getValueType().getVectorElementType
() && "Extract subvector VTs must have the same element type!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == N1.getValueType().getVectorElementType() && \"Extract subvector VTs must have the same element type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5325, __PRETTY_FUNCTION__))
;
5326 assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&((VT.getSimpleVT() <= N1.getSimpleValueType() && "Extract subvector must be from larger vector to smaller vector!"
) ? static_cast<void> (0) : __assert_fail ("VT.getSimpleVT() <= N1.getSimpleValueType() && \"Extract subvector must be from larger vector to smaller vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5327, __PRETTY_FUNCTION__))
5327 "Extract subvector must be from larger vector to smaller vector!")((VT.getSimpleVT() <= N1.getSimpleValueType() && "Extract subvector must be from larger vector to smaller vector!"
) ? static_cast<void> (0) : __assert_fail ("VT.getSimpleVT() <= N1.getSimpleValueType() && \"Extract subvector must be from larger vector to smaller vector!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5327, __PRETTY_FUNCTION__))
;
5328
5329 if (N2C) {
5330 assert((VT.getVectorNumElements() + N2C->getZExtValue()(((VT.getVectorNumElements() + N2C->getZExtValue() <= N1
.getValueType().getVectorNumElements()) && "Extract subvector overflow!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getVectorNumElements() + N2C->getZExtValue() <= N1.getValueType().getVectorNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5332, __PRETTY_FUNCTION__))
5331 <= N1.getValueType().getVectorNumElements())(((VT.getVectorNumElements() + N2C->getZExtValue() <= N1
.getValueType().getVectorNumElements()) && "Extract subvector overflow!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getVectorNumElements() + N2C->getZExtValue() <= N1.getValueType().getVectorNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5332, __PRETTY_FUNCTION__))
5332 && "Extract subvector overflow!")(((VT.getVectorNumElements() + N2C->getZExtValue() <= N1
.getValueType().getVectorNumElements()) && "Extract subvector overflow!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getVectorNumElements() + N2C->getZExtValue() <= N1.getValueType().getVectorNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5332, __PRETTY_FUNCTION__))
;
5333 }
5334
5335 // Trivial extraction.
5336 if (VT.getSimpleVT() == N1.getSimpleValueType())
5337 return N1;
5338
5339 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
5340 if (N1.isUndef())
5341 return getUNDEF(VT);
5342
5343 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
5344 // the concat have the same type as the extract.
5345 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
5346 N1.getNumOperands() > 0 &&
5347 VT == N1.getOperand(0).getValueType()) {
5348 unsigned Factor = VT.getVectorNumElements();
5349 return N1.getOperand(N2C->getZExtValue() / Factor);
5350 }
5351
5352 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
5353 // during shuffle legalization.
5354 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
5355 VT == N1.getOperand(1).getValueType())
5356 return N1.getOperand(1);
5357 }
5358 break;
5359 }
5360
5361 // Perform trivial constant folding.
5362 if (SDValue SV =
5363 FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
5364 return SV;
5365
5366 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
5367 return V;
5368
5369 // Canonicalize an UNDEF to the RHS, even over a constant.
5370 if (N1.isUndef()) {
5371 if (TLI->isCommutativeBinOp(Opcode)) {
5372 std::swap(N1, N2);
5373 } else {
5374 switch (Opcode) {
5375 case ISD::SIGN_EXTEND_INREG:
5376 case ISD::SUB:
5377 return getUNDEF(VT); // fold op(undef, arg2) -> undef
5378 case ISD::UDIV:
5379 case ISD::SDIV:
5380 case ISD::UREM:
5381 case ISD::SREM:
5382 case ISD::SSUBSAT:
5383 case ISD::USUBSAT:
5384 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
5385 }
5386 }
5387 }
5388
5389 // Fold a bunch of operators when the RHS is undef.
5390 if (N2.isUndef()) {
5391 switch (Opcode) {
5392 case ISD::XOR:
5393 if (N1.isUndef())
5394 // Handle undef ^ undef -> 0 special case. This is a common
5395 // idiom (misuse).
5396 return getConstant(0, DL, VT);
5397 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5398 case ISD::ADD:
5399 case ISD::SUB:
5400 case ISD::UDIV:
5401 case ISD::SDIV:
5402 case ISD::UREM: