Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1150, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/CodeGen/SelectionDAG -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-08-28-193554-24367-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/FoldingSet.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Triple.h"
26#include "llvm/ADT/Twine.h"
27#include "llvm/Analysis/BlockFrequencyInfo.h"
28#include "llvm/Analysis/MemoryLocation.h"
29#include "llvm/Analysis/ProfileSummaryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/CodeGen/FunctionLoweringInfo.h"
32#include "llvm/CodeGen/ISDOpcodes.h"
33#include "llvm/CodeGen/MachineBasicBlock.h"
34#include "llvm/CodeGen/MachineConstantPool.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/RuntimeLibcalls.h"
39#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
40#include "llvm/CodeGen/SelectionDAGNodes.h"
41#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
42#include "llvm/CodeGen/TargetFrameLowering.h"
43#include "llvm/CodeGen/TargetLowering.h"
44#include "llvm/CodeGen/TargetRegisterInfo.h"
45#include "llvm/CodeGen/TargetSubtargetInfo.h"
46#include "llvm/CodeGen/ValueTypes.h"
47#include "llvm/IR/Constant.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/DebugLoc.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/GlobalValue.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Type.h"
57#include "llvm/IR/Value.h"
58#include "llvm/Support/Casting.h"
59#include "llvm/Support/CodeGen.h"
60#include "llvm/Support/Compiler.h"
61#include "llvm/Support/Debug.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/KnownBits.h"
64#include "llvm/Support/MachineValueType.h"
65#include "llvm/Support/ManagedStatic.h"
66#include "llvm/Support/MathExtras.h"
67#include "llvm/Support/Mutex.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/Transforms/Utils/SizeOpts.h"
72#include <algorithm>
73#include <cassert>
74#include <cstdint>
75#include <cstdlib>
76#include <limits>
77#include <set>
78#include <string>
79#include <utility>
80#include <vector>
81
82using namespace llvm;
83
84/// makeVTList - Return an instance of the SDVTList struct initialized with the
85/// specified members.
86static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
87 SDVTList Res = {VTs, NumVTs};
88 return Res;
89}
90
91// Default null implementations of the callbacks.
92void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
93void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
94void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
95
96void SelectionDAG::DAGNodeDeletedListener::anchor() {}
97
98#define DEBUG_TYPE"selectiondag" "selectiondag"
99
100static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
101 cl::Hidden, cl::init(true),
102 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
103
104static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
105 cl::desc("Number limit for gluing ld/st of memcpy."),
106 cl::Hidden, cl::init(0));
107
108static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
109 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("selectiondag")) { dbgs() << Msg; V.getNode()->dump
(G);; } } while (false)
;
110}
111
112//===----------------------------------------------------------------------===//
113// ConstantFPSDNode Class
114//===----------------------------------------------------------------------===//
115
116/// isExactlyValue - We don't rely on operator== working on double values, as
117/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
118/// As such, this method can be used to do an exact bit-for-bit comparison of
119/// two floating point values.
120bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
121 return getValueAPF().bitwiseIsEqual(V);
122}
123
124bool ConstantFPSDNode::isValueValidForType(EVT VT,
125 const APFloat& Val) {
126 assert(VT.isFloatingPoint() && "Can only convert between FP types")(static_cast <bool> (VT.isFloatingPoint() && "Can only convert between FP types"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Can only convert between FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 126, __extension__ __PRETTY_FUNCTION__))
;
127
128 // convert modifies in place, so make a copy.
129 APFloat Val2 = APFloat(Val);
130 bool losesInfo;
131 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
132 APFloat::rmNearestTiesToEven,
133 &losesInfo);
134 return !losesInfo;
135}
136
137//===----------------------------------------------------------------------===//
138// ISD Namespace
139//===----------------------------------------------------------------------===//
140
141bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
142 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
143 unsigned EltSize =
144 N->getValueType(0).getVectorElementType().getSizeInBits();
145 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
146 SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize);
147 return true;
148 }
149 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
150 SplatVal = Op0->getValueAPF().bitcastToAPInt().truncOrSelf(EltSize);
151 return true;
152 }
153 }
154
155 auto *BV = dyn_cast<BuildVectorSDNode>(N);
156 if (!BV)
157 return false;
158
159 APInt SplatUndef;
160 unsigned SplatBitSize;
161 bool HasUndefs;
162 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
163 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
164 EltSize) &&
165 EltSize == SplatBitSize;
166}
167
168// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
169// specializations of the more general isConstantSplatVector()?
170
171bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
172 // Look through a bit convert.
173 while (N->getOpcode() == ISD::BITCAST)
174 N = N->getOperand(0).getNode();
175
176 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
177 APInt SplatVal;
178 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue();
179 }
180
181 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
182
183 unsigned i = 0, e = N->getNumOperands();
184
185 // Skip over all of the undef values.
186 while (i != e && N->getOperand(i).isUndef())
187 ++i;
188
189 // Do not accept an all-undef vector.
190 if (i == e) return false;
191
192 // Do not accept build_vectors that aren't all constants or which have non-~0
193 // elements. We have to be a bit careful here, as the type of the constant
194 // may not be the same as the type of the vector elements due to type
195 // legalization (the elements are promoted to a legal type for the target and
196 // a vector of a type may be legal when the base element type is not).
197 // We only want to check enough bits to cover the vector elements, because
198 // we care if the resultant vector is all ones, not whether the individual
199 // constants are.
200 SDValue NotZero = N->getOperand(i);
201 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
202 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
203 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
204 return false;
205 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
206 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
207 return false;
208 } else
209 return false;
210
211 // Okay, we have at least one ~0 value, check to see if the rest match or are
212 // undefs. Even with the above element type twiddling, this should be OK, as
213 // the same type legalization should have applied to all the elements.
214 for (++i; i != e; ++i)
215 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
216 return false;
217 return true;
218}
219
220bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
221 // Look through a bit convert.
222 while (N->getOpcode() == ISD::BITCAST)
223 N = N->getOperand(0).getNode();
224
225 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
226 APInt SplatVal;
227 return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue();
228 }
229
230 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
231
232 bool IsAllUndef = true;
233 for (const SDValue &Op : N->op_values()) {
234 if (Op.isUndef())
235 continue;
236 IsAllUndef = false;
237 // Do not accept build_vectors that aren't all constants or which have non-0
238 // elements. We have to be a bit careful here, as the type of the constant
239 // may not be the same as the type of the vector elements due to type
240 // legalization (the elements are promoted to a legal type for the target
241 // and a vector of a type may be legal when the base element type is not).
242 // We only want to check enough bits to cover the vector elements, because
243 // we care if the resultant vector is all zeros, not whether the individual
244 // constants are.
245 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
246 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
247 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
248 return false;
249 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
250 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
251 return false;
252 } else
253 return false;
254 }
255
256 // Do not accept an all-undef vector.
257 if (IsAllUndef)
258 return false;
259 return true;
260}
261
262bool ISD::isBuildVectorAllOnes(const SDNode *N) {
263 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
264}
265
266bool ISD::isBuildVectorAllZeros(const SDNode *N) {
267 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
268}
269
270bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
271 if (N->getOpcode() != ISD::BUILD_VECTOR)
272 return false;
273
274 for (const SDValue &Op : N->op_values()) {
275 if (Op.isUndef())
276 continue;
277 if (!isa<ConstantSDNode>(Op))
278 return false;
279 }
280 return true;
281}
282
283bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
284 if (N->getOpcode() != ISD::BUILD_VECTOR)
285 return false;
286
287 for (const SDValue &Op : N->op_values()) {
288 if (Op.isUndef())
289 continue;
290 if (!isa<ConstantFPSDNode>(Op))
291 return false;
292 }
293 return true;
294}
295
296bool ISD::allOperandsUndef(const SDNode *N) {
297 // Return false if the node has no operands.
298 // This is "logically inconsistent" with the definition of "all" but
299 // is probably the desired behavior.
300 if (N->getNumOperands() == 0)
301 return false;
302 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
303}
304
305bool ISD::matchUnaryPredicate(SDValue Op,
306 std::function<bool(ConstantSDNode *)> Match,
307 bool AllowUndefs) {
308 // FIXME: Add support for scalar UNDEF cases?
309 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
310 return Match(Cst);
311
312 // FIXME: Add support for vector UNDEF cases?
313 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
314 ISD::SPLAT_VECTOR != Op.getOpcode())
315 return false;
316
317 EVT SVT = Op.getValueType().getScalarType();
318 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
319 if (AllowUndefs && Op.getOperand(i).isUndef()) {
320 if (!Match(nullptr))
321 return false;
322 continue;
323 }
324
325 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
326 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
327 return false;
328 }
329 return true;
330}
331
332bool ISD::matchBinaryPredicate(
333 SDValue LHS, SDValue RHS,
334 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
335 bool AllowUndefs, bool AllowTypeMismatch) {
336 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
337 return false;
338
339 // TODO: Add support for scalar UNDEF cases?
340 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
341 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
342 return Match(LHSCst, RHSCst);
343
344 // TODO: Add support for vector UNDEF cases?
345 if (LHS.getOpcode() != RHS.getOpcode() ||
346 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
347 LHS.getOpcode() != ISD::SPLAT_VECTOR))
348 return false;
349
350 EVT SVT = LHS.getValueType().getScalarType();
351 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
352 SDValue LHSOp = LHS.getOperand(i);
353 SDValue RHSOp = RHS.getOperand(i);
354 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
355 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
356 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
357 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
358 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
359 return false;
360 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
361 LHSOp.getValueType() != RHSOp.getValueType()))
362 return false;
363 if (!Match(LHSCst, RHSCst))
364 return false;
365 }
366 return true;
367}
368
369ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
370 switch (VecReduceOpcode) {
371 default:
372 llvm_unreachable("Expected VECREDUCE opcode")::llvm::llvm_unreachable_internal("Expected VECREDUCE opcode"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 372)
;
373 case ISD::VECREDUCE_FADD:
374 case ISD::VECREDUCE_SEQ_FADD:
375 return ISD::FADD;
376 case ISD::VECREDUCE_FMUL:
377 case ISD::VECREDUCE_SEQ_FMUL:
378 return ISD::FMUL;
379 case ISD::VECREDUCE_ADD:
380 return ISD::ADD;
381 case ISD::VECREDUCE_MUL:
382 return ISD::MUL;
383 case ISD::VECREDUCE_AND:
384 return ISD::AND;
385 case ISD::VECREDUCE_OR:
386 return ISD::OR;
387 case ISD::VECREDUCE_XOR:
388 return ISD::XOR;
389 case ISD::VECREDUCE_SMAX:
390 return ISD::SMAX;
391 case ISD::VECREDUCE_SMIN:
392 return ISD::SMIN;
393 case ISD::VECREDUCE_UMAX:
394 return ISD::UMAX;
395 case ISD::VECREDUCE_UMIN:
396 return ISD::UMIN;
397 case ISD::VECREDUCE_FMAX:
398 return ISD::FMAXNUM;
399 case ISD::VECREDUCE_FMIN:
400 return ISD::FMINNUM;
401 }
402}
403
404bool ISD::isVPOpcode(unsigned Opcode) {
405 switch (Opcode) {
406 default:
407 return false;
408#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
409 case ISD::SDOPC: \
410 return true;
411#include "llvm/IR/VPIntrinsics.def"
412 }
413}
414
415/// The operand position of the vector mask.
416Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
417 switch (Opcode) {
418 default:
419 return None;
420#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \
421 case ISD::SDOPC: \
422 return MASKPOS;
423#include "llvm/IR/VPIntrinsics.def"
424 }
425}
426
427/// The operand position of the explicit vector length parameter.
428Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
429 switch (Opcode) {
430 default:
431 return None;
432#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
433 case ISD::SDOPC: \
434 return EVLPOS;
435#include "llvm/IR/VPIntrinsics.def"
436 }
437}
438
439ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
440 switch (ExtType) {
441 case ISD::EXTLOAD:
442 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
443 case ISD::SEXTLOAD:
444 return ISD::SIGN_EXTEND;
445 case ISD::ZEXTLOAD:
446 return ISD::ZERO_EXTEND;
447 default:
448 break;
449 }
450
451 llvm_unreachable("Invalid LoadExtType")::llvm::llvm_unreachable_internal("Invalid LoadExtType", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 451)
;
452}
453
454ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
455 // To perform this operation, we just need to swap the L and G bits of the
456 // operation.
457 unsigned OldL = (Operation >> 2) & 1;
458 unsigned OldG = (Operation >> 1) & 1;
459 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
460 (OldL << 1) | // New G bit
461 (OldG << 2)); // New L bit.
462}
463
464static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
465 unsigned Operation = Op;
466 if (isIntegerLike)
467 Operation ^= 7; // Flip L, G, E bits, but not U.
468 else
469 Operation ^= 15; // Flip all of the condition bits.
470
471 if (Operation > ISD::SETTRUE2)
472 Operation &= ~8; // Don't let N and U bits get set.
473
474 return ISD::CondCode(Operation);
475}
476
477ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
478 return getSetCCInverseImpl(Op, Type.isInteger());
479}
480
481ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
482 bool isIntegerLike) {
483 return getSetCCInverseImpl(Op, isIntegerLike);
484}
485
486/// For an integer comparison, return 1 if the comparison is a signed operation
487/// and 2 if the result is an unsigned comparison. Return zero if the operation
488/// does not depend on the sign of the input (setne and seteq).
489static int isSignedOp(ISD::CondCode Opcode) {
490 switch (Opcode) {
491 default: llvm_unreachable("Illegal integer setcc operation!")::llvm::llvm_unreachable_internal("Illegal integer setcc operation!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 491)
;
492 case ISD::SETEQ:
493 case ISD::SETNE: return 0;
494 case ISD::SETLT:
495 case ISD::SETLE:
496 case ISD::SETGT:
497 case ISD::SETGE: return 1;
498 case ISD::SETULT:
499 case ISD::SETULE:
500 case ISD::SETUGT:
501 case ISD::SETUGE: return 2;
502 }
503}
504
505ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
506 EVT Type) {
507 bool IsInteger = Type.isInteger();
508 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
509 // Cannot fold a signed integer setcc with an unsigned integer setcc.
510 return ISD::SETCC_INVALID;
511
512 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
513
514 // If the N and U bits get set, then the resultant comparison DOES suddenly
515 // care about orderedness, and it is true when ordered.
516 if (Op > ISD::SETTRUE2)
517 Op &= ~16; // Clear the U bit if the N bit is set.
518
519 // Canonicalize illegal integer setcc's.
520 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
521 Op = ISD::SETNE;
522
523 return ISD::CondCode(Op);
524}
525
526ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
527 EVT Type) {
528 bool IsInteger = Type.isInteger();
529 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
530 // Cannot fold a signed setcc with an unsigned setcc.
531 return ISD::SETCC_INVALID;
532
533 // Combine all of the condition bits.
534 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
535
536 // Canonicalize illegal integer setcc's.
537 if (IsInteger) {
538 switch (Result) {
539 default: break;
540 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
541 case ISD::SETOEQ: // SETEQ & SETU[LG]E
542 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
543 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
544 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
545 }
546 }
547
548 return Result;
549}
550
551//===----------------------------------------------------------------------===//
552// SDNode Profile Support
553//===----------------------------------------------------------------------===//
554
555/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
556static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
557 ID.AddInteger(OpC);
558}
559
560/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
561/// solely with their pointer.
562static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
563 ID.AddPointer(VTList.VTs);
564}
565
566/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
567static void AddNodeIDOperands(FoldingSetNodeID &ID,
568 ArrayRef<SDValue> Ops) {
569 for (auto& Op : Ops) {
570 ID.AddPointer(Op.getNode());
571 ID.AddInteger(Op.getResNo());
572 }
573}
574
575/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
576static void AddNodeIDOperands(FoldingSetNodeID &ID,
577 ArrayRef<SDUse> Ops) {
578 for (auto& Op : Ops) {
579 ID.AddPointer(Op.getNode());
580 ID.AddInteger(Op.getResNo());
581 }
582}
583
584static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
585 SDVTList VTList, ArrayRef<SDValue> OpList) {
586 AddNodeIDOpcode(ID, OpC);
587 AddNodeIDValueTypes(ID, VTList);
588 AddNodeIDOperands(ID, OpList);
589}
590
591/// If this is an SDNode with special info, add this info to the NodeID data.
592static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
593 switch (N->getOpcode()) {
594 case ISD::TargetExternalSymbol:
595 case ISD::ExternalSymbol:
596 case ISD::MCSymbol:
597 llvm_unreachable("Should only be used on nodes with operands")::llvm::llvm_unreachable_internal("Should only be used on nodes with operands"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 597)
;
598 default: break; // Normal nodes don't need extra info.
599 case ISD::TargetConstant:
600 case ISD::Constant: {
601 const ConstantSDNode *C = cast<ConstantSDNode>(N);
602 ID.AddPointer(C->getConstantIntValue());
603 ID.AddBoolean(C->isOpaque());
604 break;
605 }
606 case ISD::TargetConstantFP:
607 case ISD::ConstantFP:
608 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
609 break;
610 case ISD::TargetGlobalAddress:
611 case ISD::GlobalAddress:
612 case ISD::TargetGlobalTLSAddress:
613 case ISD::GlobalTLSAddress: {
614 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
615 ID.AddPointer(GA->getGlobal());
616 ID.AddInteger(GA->getOffset());
617 ID.AddInteger(GA->getTargetFlags());
618 break;
619 }
620 case ISD::BasicBlock:
621 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
622 break;
623 case ISD::Register:
624 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
625 break;
626 case ISD::RegisterMask:
627 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
628 break;
629 case ISD::SRCVALUE:
630 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
631 break;
632 case ISD::FrameIndex:
633 case ISD::TargetFrameIndex:
634 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
635 break;
636 case ISD::LIFETIME_START:
637 case ISD::LIFETIME_END:
638 if (cast<LifetimeSDNode>(N)->hasOffset()) {
639 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
640 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
641 }
642 break;
643 case ISD::PSEUDO_PROBE:
644 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
645 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
646 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
647 break;
648 case ISD::JumpTable:
649 case ISD::TargetJumpTable:
650 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
651 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
652 break;
653 case ISD::ConstantPool:
654 case ISD::TargetConstantPool: {
655 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
656 ID.AddInteger(CP->getAlign().value());
657 ID.AddInteger(CP->getOffset());
658 if (CP->isMachineConstantPoolEntry())
659 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
660 else
661 ID.AddPointer(CP->getConstVal());
662 ID.AddInteger(CP->getTargetFlags());
663 break;
664 }
665 case ISD::TargetIndex: {
666 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
667 ID.AddInteger(TI->getIndex());
668 ID.AddInteger(TI->getOffset());
669 ID.AddInteger(TI->getTargetFlags());
670 break;
671 }
672 case ISD::LOAD: {
673 const LoadSDNode *LD = cast<LoadSDNode>(N);
674 ID.AddInteger(LD->getMemoryVT().getRawBits());
675 ID.AddInteger(LD->getRawSubclassData());
676 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
677 break;
678 }
679 case ISD::STORE: {
680 const StoreSDNode *ST = cast<StoreSDNode>(N);
681 ID.AddInteger(ST->getMemoryVT().getRawBits());
682 ID.AddInteger(ST->getRawSubclassData());
683 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
684 break;
685 }
686 case ISD::MLOAD: {
687 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
688 ID.AddInteger(MLD->getMemoryVT().getRawBits());
689 ID.AddInteger(MLD->getRawSubclassData());
690 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
691 break;
692 }
693 case ISD::MSTORE: {
694 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
695 ID.AddInteger(MST->getMemoryVT().getRawBits());
696 ID.AddInteger(MST->getRawSubclassData());
697 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
698 break;
699 }
700 case ISD::MGATHER: {
701 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
702 ID.AddInteger(MG->getMemoryVT().getRawBits());
703 ID.AddInteger(MG->getRawSubclassData());
704 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
705 break;
706 }
707 case ISD::MSCATTER: {
708 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
709 ID.AddInteger(MS->getMemoryVT().getRawBits());
710 ID.AddInteger(MS->getRawSubclassData());
711 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
712 break;
713 }
714 case ISD::ATOMIC_CMP_SWAP:
715 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
716 case ISD::ATOMIC_SWAP:
717 case ISD::ATOMIC_LOAD_ADD:
718 case ISD::ATOMIC_LOAD_SUB:
719 case ISD::ATOMIC_LOAD_AND:
720 case ISD::ATOMIC_LOAD_CLR:
721 case ISD::ATOMIC_LOAD_OR:
722 case ISD::ATOMIC_LOAD_XOR:
723 case ISD::ATOMIC_LOAD_NAND:
724 case ISD::ATOMIC_LOAD_MIN:
725 case ISD::ATOMIC_LOAD_MAX:
726 case ISD::ATOMIC_LOAD_UMIN:
727 case ISD::ATOMIC_LOAD_UMAX:
728 case ISD::ATOMIC_LOAD:
729 case ISD::ATOMIC_STORE: {
730 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
731 ID.AddInteger(AT->getMemoryVT().getRawBits());
732 ID.AddInteger(AT->getRawSubclassData());
733 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
734 break;
735 }
736 case ISD::PREFETCH: {
737 const MemSDNode *PF = cast<MemSDNode>(N);
738 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
739 break;
740 }
741 case ISD::VECTOR_SHUFFLE: {
742 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
743 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
744 i != e; ++i)
745 ID.AddInteger(SVN->getMaskElt(i));
746 break;
747 }
748 case ISD::TargetBlockAddress:
749 case ISD::BlockAddress: {
750 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
751 ID.AddPointer(BA->getBlockAddress());
752 ID.AddInteger(BA->getOffset());
753 ID.AddInteger(BA->getTargetFlags());
754 break;
755 }
756 } // end switch (N->getOpcode())
757
758 // Target specific memory nodes could also have address spaces to check.
759 if (N->isTargetMemoryOpcode())
760 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
761}
762
763/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
764/// data.
765static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
766 AddNodeIDOpcode(ID, N->getOpcode());
767 // Add the return value info.
768 AddNodeIDValueTypes(ID, N->getVTList());
769 // Add the operand info.
770 AddNodeIDOperands(ID, N->ops());
771
772 // Handle SDNode leafs with special info.
773 AddNodeIDCustom(ID, N);
774}
775
776//===----------------------------------------------------------------------===//
777// SelectionDAG Class
778//===----------------------------------------------------------------------===//
779
780/// doNotCSE - Return true if CSE should not be performed for this node.
781static bool doNotCSE(SDNode *N) {
782 if (N->getValueType(0) == MVT::Glue)
783 return true; // Never CSE anything that produces a flag.
784
785 switch (N->getOpcode()) {
786 default: break;
787 case ISD::HANDLENODE:
788 case ISD::EH_LABEL:
789 return true; // Never CSE these nodes.
790 }
791
792 // Check that remaining values produced are not flags.
793 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
794 if (N->getValueType(i) == MVT::Glue)
795 return true; // Never CSE anything that produces a flag.
796
797 return false;
798}
799
800/// RemoveDeadNodes - This method deletes all unreachable nodes in the
801/// SelectionDAG.
802void SelectionDAG::RemoveDeadNodes() {
803 // Create a dummy node (which is not added to allnodes), that adds a reference
804 // to the root node, preventing it from being deleted.
805 HandleSDNode Dummy(getRoot());
806
807 SmallVector<SDNode*, 128> DeadNodes;
808
809 // Add all obviously-dead nodes to the DeadNodes worklist.
810 for (SDNode &Node : allnodes())
811 if (Node.use_empty())
812 DeadNodes.push_back(&Node);
813
814 RemoveDeadNodes(DeadNodes);
815
816 // If the root changed (e.g. it was a dead load, update the root).
817 setRoot(Dummy.getValue());
818}
819
820/// RemoveDeadNodes - This method deletes the unreachable nodes in the
821/// given list, and any nodes that become unreachable as a result.
822void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
823
824 // Process the worklist, deleting the nodes and adding their uses to the
825 // worklist.
826 while (!DeadNodes.empty()) {
827 SDNode *N = DeadNodes.pop_back_val();
828 // Skip to next node if we've already managed to delete the node. This could
829 // happen if replacing a node causes a node previously added to the node to
830 // be deleted.
831 if (N->getOpcode() == ISD::DELETED_NODE)
832 continue;
833
834 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
835 DUL->NodeDeleted(N, nullptr);
836
837 // Take the node out of the appropriate CSE map.
838 RemoveNodeFromCSEMaps(N);
839
840 // Next, brutally remove the operand list. This is safe to do, as there are
841 // no cycles in the graph.
842 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
843 SDUse &Use = *I++;
844 SDNode *Operand = Use.getNode();
845 Use.set(SDValue());
846
847 // Now that we removed this operand, see if there are no uses of it left.
848 if (Operand->use_empty())
849 DeadNodes.push_back(Operand);
850 }
851
852 DeallocateNode(N);
853 }
854}
855
856void SelectionDAG::RemoveDeadNode(SDNode *N){
857 SmallVector<SDNode*, 16> DeadNodes(1, N);
858
859 // Create a dummy node that adds a reference to the root node, preventing
860 // it from being deleted. (This matters if the root is an operand of the
861 // dead node.)
862 HandleSDNode Dummy(getRoot());
863
864 RemoveDeadNodes(DeadNodes);
865}
866
867void SelectionDAG::DeleteNode(SDNode *N) {
868 // First take this out of the appropriate CSE map.
869 RemoveNodeFromCSEMaps(N);
870
871 // Finally, remove uses due to operands of this node, remove from the
872 // AllNodes list, and delete the node.
873 DeleteNodeNotInCSEMaps(N);
874}
875
876void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
877 assert(N->getIterator() != AllNodes.begin() &&(static_cast <bool> (N->getIterator() != AllNodes.begin
() && "Cannot delete the entry node!") ? void (0) : __assert_fail
("N->getIterator() != AllNodes.begin() && \"Cannot delete the entry node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 878, __extension__ __PRETTY_FUNCTION__))
878 "Cannot delete the entry node!")(static_cast <bool> (N->getIterator() != AllNodes.begin
() && "Cannot delete the entry node!") ? void (0) : __assert_fail
("N->getIterator() != AllNodes.begin() && \"Cannot delete the entry node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 878, __extension__ __PRETTY_FUNCTION__))
;
879 assert(N->use_empty() && "Cannot delete a node that is not dead!")(static_cast <bool> (N->use_empty() && "Cannot delete a node that is not dead!"
) ? void (0) : __assert_fail ("N->use_empty() && \"Cannot delete a node that is not dead!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 879, __extension__ __PRETTY_FUNCTION__))
;
880
881 // Drop all of the operands and decrement used node's use counts.
882 N->DropOperands();
883
884 DeallocateNode(N);
885}
886
887void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
888 assert(!(V->isVariadic() && isParameter))(static_cast <bool> (!(V->isVariadic() && isParameter
)) ? void (0) : __assert_fail ("!(V->isVariadic() && isParameter)"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 888, __extension__ __PRETTY_FUNCTION__))
;
889 if (isParameter)
890 ByvalParmDbgValues.push_back(V);
891 else
892 DbgValues.push_back(V);
893 for (const SDNode *Node : V->getSDNodes())
894 if (Node)
895 DbgValMap[Node].push_back(V);
896}
897
898void SDDbgInfo::erase(const SDNode *Node) {
899 DbgValMapType::iterator I = DbgValMap.find(Node);
900 if (I == DbgValMap.end())
901 return;
902 for (auto &Val: I->second)
903 Val->setIsInvalidated();
904 DbgValMap.erase(I);
905}
906
907void SelectionDAG::DeallocateNode(SDNode *N) {
908 // If we have operands, deallocate them.
909 removeOperands(N);
910
911 NodeAllocator.Deallocate(AllNodes.remove(N));
912
913 // Set the opcode to DELETED_NODE to help catch bugs when node
914 // memory is reallocated.
915 // FIXME: There are places in SDag that have grown a dependency on the opcode
916 // value in the released node.
917 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
918 N->NodeType = ISD::DELETED_NODE;
919
920 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
921 // them and forget about that node.
922 DbgInfo->erase(N);
923}
924
925#ifndef NDEBUG
926/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
927static void VerifySDNode(SDNode *N) {
928 switch (N->getOpcode()) {
929 default:
930 break;
931 case ISD::BUILD_PAIR: {
932 EVT VT = N->getValueType(0);
933 assert(N->getNumValues() == 1 && "Too many results!")(static_cast <bool> (N->getNumValues() == 1 &&
"Too many results!") ? void (0) : __assert_fail ("N->getNumValues() == 1 && \"Too many results!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 933, __extension__ __PRETTY_FUNCTION__))
;
934 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&(static_cast <bool> (!VT.isVector() && (VT.isInteger
() || VT.isFloatingPoint()) && "Wrong return type!") ?
void (0) : __assert_fail ("!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 935, __extension__ __PRETTY_FUNCTION__))
935 "Wrong return type!")(static_cast <bool> (!VT.isVector() && (VT.isInteger
() || VT.isFloatingPoint()) && "Wrong return type!") ?
void (0) : __assert_fail ("!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 935, __extension__ __PRETTY_FUNCTION__))
;
936 assert(N->getNumOperands() == 2 && "Wrong number of operands!")(static_cast <bool> (N->getNumOperands() == 2 &&
"Wrong number of operands!") ? void (0) : __assert_fail ("N->getNumOperands() == 2 && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 936, __extension__ __PRETTY_FUNCTION__))
;
937 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&(static_cast <bool> (N->getOperand(0).getValueType()
== N->getOperand(1).getValueType() && "Mismatched operand types!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && \"Mismatched operand types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 938, __extension__ __PRETTY_FUNCTION__))
938 "Mismatched operand types!")(static_cast <bool> (N->getOperand(0).getValueType()
== N->getOperand(1).getValueType() && "Mismatched operand types!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && \"Mismatched operand types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 938, __extension__ __PRETTY_FUNCTION__))
;
939 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&(static_cast <bool> (N->getOperand(0).getValueType()
.isInteger() == VT.isInteger() && "Wrong operand type!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType().isInteger() == VT.isInteger() && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 940, __extension__ __PRETTY_FUNCTION__))
940 "Wrong operand type!")(static_cast <bool> (N->getOperand(0).getValueType()
.isInteger() == VT.isInteger() && "Wrong operand type!"
) ? void (0) : __assert_fail ("N->getOperand(0).getValueType().isInteger() == VT.isInteger() && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 940, __extension__ __PRETTY_FUNCTION__))
;
941 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&(static_cast <bool> (VT.getSizeInBits() == 2 * N->getOperand
(0).getValueSizeInBits() && "Wrong return type size")
? void (0) : __assert_fail ("VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && \"Wrong return type size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 942, __extension__ __PRETTY_FUNCTION__))
942 "Wrong return type size")(static_cast <bool> (VT.getSizeInBits() == 2 * N->getOperand
(0).getValueSizeInBits() && "Wrong return type size")
? void (0) : __assert_fail ("VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && \"Wrong return type size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 942, __extension__ __PRETTY_FUNCTION__))
;
943 break;
944 }
945 case ISD::BUILD_VECTOR: {
946 assert(N->getNumValues() == 1 && "Too many results!")(static_cast <bool> (N->getNumValues() == 1 &&
"Too many results!") ? void (0) : __assert_fail ("N->getNumValues() == 1 && \"Too many results!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 946, __extension__ __PRETTY_FUNCTION__))
;
947 assert(N->getValueType(0).isVector() && "Wrong return type!")(static_cast <bool> (N->getValueType(0).isVector() &&
"Wrong return type!") ? void (0) : __assert_fail ("N->getValueType(0).isVector() && \"Wrong return type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 947, __extension__ __PRETTY_FUNCTION__))
;
948 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&(static_cast <bool> (N->getNumOperands() == N->getValueType
(0).getVectorNumElements() && "Wrong number of operands!"
) ? void (0) : __assert_fail ("N->getNumOperands() == N->getValueType(0).getVectorNumElements() && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 949, __extension__ __PRETTY_FUNCTION__))
949 "Wrong number of operands!")(static_cast <bool> (N->getNumOperands() == N->getValueType
(0).getVectorNumElements() && "Wrong number of operands!"
) ? void (0) : __assert_fail ("N->getNumOperands() == N->getValueType(0).getVectorNumElements() && \"Wrong number of operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 949, __extension__ __PRETTY_FUNCTION__))
;
950 EVT EltVT = N->getValueType(0).getVectorElementType();
951 for (const SDUse &Op : N->ops()) {
952 assert((Op.getValueType() == EltVT ||(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
953 (EltVT.isInteger() && Op.getValueType().isInteger() &&(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
954 EltVT.bitsLE(Op.getValueType()))) &&(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
955 "Wrong operand type!")(static_cast <bool> ((Op.getValueType() == EltVT || (EltVT
.isInteger() && Op.getValueType().isInteger() &&
EltVT.bitsLE(Op.getValueType()))) && "Wrong operand type!"
) ? void (0) : __assert_fail ("(Op.getValueType() == EltVT || (EltVT.isInteger() && Op.getValueType().isInteger() && EltVT.bitsLE(Op.getValueType()))) && \"Wrong operand type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
;
956 assert(Op.getValueType() == N->getOperand(0).getValueType() &&(static_cast <bool> (Op.getValueType() == N->getOperand
(0).getValueType() && "Operands must all have the same type"
) ? void (0) : __assert_fail ("Op.getValueType() == N->getOperand(0).getValueType() && \"Operands must all have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 957, __extension__ __PRETTY_FUNCTION__))
957 "Operands must all have the same type")(static_cast <bool> (Op.getValueType() == N->getOperand
(0).getValueType() && "Operands must all have the same type"
) ? void (0) : __assert_fail ("Op.getValueType() == N->getOperand(0).getValueType() && \"Operands must all have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 957, __extension__ __PRETTY_FUNCTION__))
;
958 }
959 break;
960 }
961 }
962}
963#endif // NDEBUG
964
965/// Insert a newly allocated node into the DAG.
966///
967/// Handles insertion into the all nodes list and CSE map, as well as
968/// verification and other common operations when a new node is allocated.
969void SelectionDAG::InsertNode(SDNode *N) {
970 AllNodes.push_back(N);
971#ifndef NDEBUG
972 N->PersistentId = NextPersistentId++;
973 VerifySDNode(N);
974#endif
975 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
976 DUL->NodeInserted(N);
977}
978
979/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
980/// correspond to it. This is useful when we're about to delete or repurpose
981/// the node. We don't want future request for structurally identical nodes
982/// to return N anymore.
983bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
984 bool Erased = false;
985 switch (N->getOpcode()) {
986 case ISD::HANDLENODE: return false; // noop.
987 case ISD::CONDCODE:
988 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&(static_cast <bool> (CondCodeNodes[cast<CondCodeSDNode
>(N)->get()] && "Cond code doesn't exist!") ? void
(0) : __assert_fail ("CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && \"Cond code doesn't exist!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 989, __extension__ __PRETTY_FUNCTION__))
989 "Cond code doesn't exist!")(static_cast <bool> (CondCodeNodes[cast<CondCodeSDNode
>(N)->get()] && "Cond code doesn't exist!") ? void
(0) : __assert_fail ("CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && \"Cond code doesn't exist!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 989, __extension__ __PRETTY_FUNCTION__))
;
990 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
991 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
992 break;
993 case ISD::ExternalSymbol:
994 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
995 break;
996 case ISD::TargetExternalSymbol: {
997 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
998 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
999 ESN->getSymbol(), ESN->getTargetFlags()));
1000 break;
1001 }
1002 case ISD::MCSymbol: {
1003 auto *MCSN = cast<MCSymbolSDNode>(N);
1004 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1005 break;
1006 }
1007 case ISD::VALUETYPE: {
1008 EVT VT = cast<VTSDNode>(N)->getVT();
1009 if (VT.isExtended()) {
1010 Erased = ExtendedValueTypeNodes.erase(VT);
1011 } else {
1012 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1013 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1014 }
1015 break;
1016 }
1017 default:
1018 // Remove it from the CSE Map.
1019 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "DELETED_NODE in CSEMap!") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"DELETED_NODE in CSEMap!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1019, __extension__ __PRETTY_FUNCTION__))
;
1020 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!")(static_cast <bool> (N->getOpcode() != ISD::EntryToken
&& "EntryToken in CSEMap!") ? void (0) : __assert_fail
("N->getOpcode() != ISD::EntryToken && \"EntryToken in CSEMap!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1020, __extension__ __PRETTY_FUNCTION__))
;
1021 Erased = CSEMap.RemoveNode(N);
1022 break;
1023 }
1024#ifndef NDEBUG
1025 // Verify that the node was actually in one of the CSE maps, unless it has a
1026 // flag result (which cannot be CSE'd) or is one of the special cases that are
1027 // not subject to CSE.
1028 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1029 !N->isMachineOpcode() && !doNotCSE(N)) {
1030 N->dump(this);
1031 dbgs() << "\n";
1032 llvm_unreachable("Node is not in map!")::llvm::llvm_unreachable_internal("Node is not in map!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1032)
;
1033 }
1034#endif
1035 return Erased;
1036}
1037
1038/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1039/// maps and modified in place. Add it back to the CSE maps, unless an identical
1040/// node already exists, in which case transfer all its users to the existing
1041/// node. This transfer can potentially trigger recursive merging.
1042void
1043SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1044 // For node types that aren't CSE'd, just act as if no identical node
1045 // already exists.
1046 if (!doNotCSE(N)) {
1047 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1048 if (Existing != N) {
1049 // If there was already an existing matching node, use ReplaceAllUsesWith
1050 // to replace the dead one with the existing one. This can cause
1051 // recursive merging of other unrelated nodes down the line.
1052 ReplaceAllUsesWith(N, Existing);
1053
1054 // N is now dead. Inform the listeners and delete it.
1055 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1056 DUL->NodeDeleted(N, Existing);
1057 DeleteNodeNotInCSEMaps(N);
1058 return;
1059 }
1060 }
1061
1062 // If the node doesn't already exist, we updated it. Inform listeners.
1063 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1064 DUL->NodeUpdated(N);
1065}
1066
1067/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1068/// were replaced with those specified. If this node is never memoized,
1069/// return null, otherwise return a pointer to the slot it would take. If a
1070/// node already exists with these operands, the slot will be non-null.
1071SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1072 void *&InsertPos) {
1073 if (doNotCSE(N))
1074 return nullptr;
1075
1076 SDValue Ops[] = { Op };
1077 FoldingSetNodeID ID;
1078 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1079 AddNodeIDCustom(ID, N);
1080 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1081 if (Node)
1082 Node->intersectFlagsWith(N->getFlags());
1083 return Node;
1084}
1085
1086/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1087/// were replaced with those specified. If this node is never memoized,
1088/// return null, otherwise return a pointer to the slot it would take. If a
1089/// node already exists with these operands, the slot will be non-null.
1090SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1091 SDValue Op1, SDValue Op2,
1092 void *&InsertPos) {
1093 if (doNotCSE(N))
1094 return nullptr;
1095
1096 SDValue Ops[] = { Op1, Op2 };
1097 FoldingSetNodeID ID;
1098 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1099 AddNodeIDCustom(ID, N);
1100 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1101 if (Node)
1102 Node->intersectFlagsWith(N->getFlags());
1103 return Node;
1104}
1105
1106/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1107/// were replaced with those specified. If this node is never memoized,
1108/// return null, otherwise return a pointer to the slot it would take. If a
1109/// node already exists with these operands, the slot will be non-null.
1110SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1111 void *&InsertPos) {
1112 if (doNotCSE(N))
1113 return nullptr;
1114
1115 FoldingSetNodeID ID;
1116 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1117 AddNodeIDCustom(ID, N);
1118 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1119 if (Node)
1120 Node->intersectFlagsWith(N->getFlags());
1121 return Node;
1122}
1123
1124Align SelectionDAG::getEVTAlign(EVT VT) const {
1125 Type *Ty = VT == MVT::iPTR ?
1126 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
1127 VT.getTypeForEVT(*getContext());
1128
1129 return getDataLayout().getABITypeAlign(Ty);
1130}
1131
1132// EntryNode could meaningfully have debug info if we can find it...
1133SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
1134 : TM(tm), OptLevel(OL),
1135 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1136 Root(getEntryNode()) {
1137 InsertNode(&EntryNode);
1138 DbgInfo = new SDDbgInfo();
1139}
1140
1141void SelectionDAG::init(MachineFunction &NewMF,
1142 OptimizationRemarkEmitter &NewORE,
1143 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1144 LegacyDivergenceAnalysis * Divergence,
1145 ProfileSummaryInfo *PSIin,
1146 BlockFrequencyInfo *BFIin) {
1147 MF = &NewMF;
1148 SDAGISelPass = PassPtr;
1149 ORE = &NewORE;
1150 TLI = getSubtarget().getTargetLowering();
1151 TSI = getSubtarget().getSelectionDAGInfo();
1152 LibInfo = LibraryInfo;
1153 Context = &MF->getFunction().getContext();
1154 DA = Divergence;
1155 PSI = PSIin;
1156 BFI = BFIin;
1157}
1158
1159SelectionDAG::~SelectionDAG() {
1160 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners")(static_cast <bool> (!UpdateListeners && "Dangling registered DAGUpdateListeners"
) ? void (0) : __assert_fail ("!UpdateListeners && \"Dangling registered DAGUpdateListeners\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1160, __extension__ __PRETTY_FUNCTION__))
;
1161 allnodes_clear();
1162 OperandRecycler.clear(OperandAllocator);
1163 delete DbgInfo;
1164}
1165
1166bool SelectionDAG::shouldOptForSize() const {
1167 return MF->getFunction().hasOptSize() ||
1168 llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1169}
1170
1171void SelectionDAG::allnodes_clear() {
1172 assert(&*AllNodes.begin() == &EntryNode)(static_cast <bool> (&*AllNodes.begin() == &EntryNode
) ? void (0) : __assert_fail ("&*AllNodes.begin() == &EntryNode"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1172, __extension__ __PRETTY_FUNCTION__))
;
1173 AllNodes.remove(AllNodes.begin());
1174 while (!AllNodes.empty())
1175 DeallocateNode(&AllNodes.front());
1176#ifndef NDEBUG
1177 NextPersistentId = 0;
1178#endif
1179}
1180
1181SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1182 void *&InsertPos) {
1183 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1184 if (N) {
1185 switch (N->getOpcode()) {
1186 default: break;
1187 case ISD::Constant:
1188 case ISD::ConstantFP:
1189 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "::llvm::llvm_unreachable_internal("Querying for Constant and ConstantFP nodes requires "
"debug location. Use another overload.", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1190)
1190 "debug location. Use another overload.")::llvm::llvm_unreachable_internal("Querying for Constant and ConstantFP nodes requires "
"debug location. Use another overload.", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1190)
;
1191 }
1192 }
1193 return N;
1194}
1195
1196SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1197 const SDLoc &DL, void *&InsertPos) {
1198 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1199 if (N) {
1200 switch (N->getOpcode()) {
1201 case ISD::Constant:
1202 case ISD::ConstantFP:
1203 // Erase debug location from the node if the node is used at several
1204 // different places. Do not propagate one location to all uses as it
1205 // will cause a worse single stepping debugging experience.
1206 if (N->getDebugLoc() != DL.getDebugLoc())
1207 N->setDebugLoc(DebugLoc());
1208 break;
1209 default:
1210 // When the node's point of use is located earlier in the instruction
1211 // sequence than its prior point of use, update its debug info to the
1212 // earlier location.
1213 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1214 N->setDebugLoc(DL.getDebugLoc());
1215 break;
1216 }
1217 }
1218 return N;
1219}
1220
1221void SelectionDAG::clear() {
1222 allnodes_clear();
1223 OperandRecycler.clear(OperandAllocator);
1224 OperandAllocator.Reset();
1225 CSEMap.clear();
1226
1227 ExtendedValueTypeNodes.clear();
1228 ExternalSymbols.clear();
1229 TargetExternalSymbols.clear();
1230 MCSymbols.clear();
1231 SDCallSiteDbgInfo.clear();
1232 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1233 static_cast<CondCodeSDNode*>(nullptr));
1234 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1235 static_cast<SDNode*>(nullptr));
1236
1237 EntryNode.UseList = nullptr;
1238 InsertNode(&EntryNode);
1239 Root = getEntryNode();
1240 DbgInfo->clear();
1241}
1242
1243SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1244 return VT.bitsGT(Op.getValueType())
1245 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1246 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1247}
1248
1249std::pair<SDValue, SDValue>
1250SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1251 const SDLoc &DL, EVT VT) {
1252 assert(!VT.bitsEq(Op.getValueType()) &&(static_cast <bool> (!VT.bitsEq(Op.getValueType()) &&
"Strict no-op FP extend/round not allowed.") ? void (0) : __assert_fail
("!VT.bitsEq(Op.getValueType()) && \"Strict no-op FP extend/round not allowed.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1253, __extension__ __PRETTY_FUNCTION__))
1253 "Strict no-op FP extend/round not allowed.")(static_cast <bool> (!VT.bitsEq(Op.getValueType()) &&
"Strict no-op FP extend/round not allowed.") ? void (0) : __assert_fail
("!VT.bitsEq(Op.getValueType()) && \"Strict no-op FP extend/round not allowed.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1253, __extension__ __PRETTY_FUNCTION__))
;
1254 SDValue Res =
1255 VT.bitsGT(Op.getValueType())
1256 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1257 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1258 {Chain, Op, getIntPtrConstant(0, DL)});
1259
1260 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1261}
1262
1263SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1264 return VT.bitsGT(Op.getValueType()) ?
1265 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1266 getNode(ISD::TRUNCATE, DL, VT, Op);
1267}
1268
1269SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1270 return VT.bitsGT(Op.getValueType()) ?
1271 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1272 getNode(ISD::TRUNCATE, DL, VT, Op);
1273}
1274
1275SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1276 return VT.bitsGT(Op.getValueType()) ?
1277 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1278 getNode(ISD::TRUNCATE, DL, VT, Op);
1279}
1280
1281SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1282 EVT OpVT) {
1283 if (VT.bitsLE(Op.getValueType()))
1284 return getNode(ISD::TRUNCATE, SL, VT, Op);
1285
1286 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1287 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1288}
1289
1290SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1291 EVT OpVT = Op.getValueType();
1292 assert(VT.isInteger() && OpVT.isInteger() &&(static_cast <bool> (VT.isInteger() && OpVT.isInteger
() && "Cannot getZeroExtendInReg FP types") ? void (0
) : __assert_fail ("VT.isInteger() && OpVT.isInteger() && \"Cannot getZeroExtendInReg FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1293, __extension__ __PRETTY_FUNCTION__))
1293 "Cannot getZeroExtendInReg FP types")(static_cast <bool> (VT.isInteger() && OpVT.isInteger
() && "Cannot getZeroExtendInReg FP types") ? void (0
) : __assert_fail ("VT.isInteger() && OpVT.isInteger() && \"Cannot getZeroExtendInReg FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1293, __extension__ __PRETTY_FUNCTION__))
;
1294 assert(VT.isVector() == OpVT.isVector() &&(static_cast <bool> (VT.isVector() == OpVT.isVector() &&
"getZeroExtendInReg type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == OpVT.isVector() && \"getZeroExtendInReg type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1296, __extension__ __PRETTY_FUNCTION__))
1295 "getZeroExtendInReg type should be vector iff the operand "(static_cast <bool> (VT.isVector() == OpVT.isVector() &&
"getZeroExtendInReg type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == OpVT.isVector() && \"getZeroExtendInReg type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1296, __extension__ __PRETTY_FUNCTION__))
1296 "type is vector!")(static_cast <bool> (VT.isVector() == OpVT.isVector() &&
"getZeroExtendInReg type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == OpVT.isVector() && \"getZeroExtendInReg type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1296, __extension__ __PRETTY_FUNCTION__))
;
1297 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == OpVT.getVectorElementCount()) && "Vector element counts must match in getZeroExtendInReg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == OpVT.getVectorElementCount()) && \"Vector element counts must match in getZeroExtendInReg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1299, __extension__ __PRETTY_FUNCTION__))
1298 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == OpVT.getVectorElementCount()) && "Vector element counts must match in getZeroExtendInReg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == OpVT.getVectorElementCount()) && \"Vector element counts must match in getZeroExtendInReg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1299, __extension__ __PRETTY_FUNCTION__))
1299 "Vector element counts must match in getZeroExtendInReg")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == OpVT.getVectorElementCount()) && "Vector element counts must match in getZeroExtendInReg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == OpVT.getVectorElementCount()) && \"Vector element counts must match in getZeroExtendInReg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1299, __extension__ __PRETTY_FUNCTION__))
;
1300 assert(VT.bitsLE(OpVT) && "Not extending!")(static_cast <bool> (VT.bitsLE(OpVT) && "Not extending!"
) ? void (0) : __assert_fail ("VT.bitsLE(OpVT) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1300, __extension__ __PRETTY_FUNCTION__))
;
1301 if (OpVT == VT)
1302 return Op;
1303 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),
1304 VT.getScalarSizeInBits());
1305 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1306}
1307
1308SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1309 // Only unsigned pointer semantics are supported right now. In the future this
1310 // might delegate to TLI to check pointer signedness.
1311 return getZExtOrTrunc(Op, DL, VT);
1312}
1313
1314SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1315 // Only unsigned pointer semantics are supported right now. In the future this
1316 // might delegate to TLI to check pointer signedness.
1317 return getZeroExtendInReg(Op, DL, VT);
1318}
1319
1320/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1321SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1322 EVT EltVT = VT.getScalarType();
1323 SDValue NegOne =
1324 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1325 return getNode(ISD::XOR, DL, VT, Val, NegOne);
1326}
1327
1328SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1329 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1330 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1331}
1332
1333SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1334 EVT OpVT) {
1335 if (!V)
1336 return getConstant(0, DL, VT);
1337
1338 switch (TLI->getBooleanContents(OpVT)) {
1339 case TargetLowering::ZeroOrOneBooleanContent:
1340 case TargetLowering::UndefinedBooleanContent:
1341 return getConstant(1, DL, VT);
1342 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1343 return getAllOnesConstant(DL, VT);
1344 }
1345 llvm_unreachable("Unexpected boolean content enum!")::llvm::llvm_unreachable_internal("Unexpected boolean content enum!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1345)
;
1346}
1347
1348SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1349 bool isT, bool isO) {
1350 EVT EltVT = VT.getScalarType();
1351 assert((EltVT.getSizeInBits() >= 64 ||(static_cast <bool> ((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 <
2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? void (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1353, __extension__ __PRETTY_FUNCTION__))
1352 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&(static_cast <bool> ((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 <
2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? void (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1353, __extension__ __PRETTY_FUNCTION__))
1353 "getConstant with a uint64_t value that doesn't fit in the type!")(static_cast <bool> ((EltVT.getSizeInBits() >= 64 ||
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 <
2) && "getConstant with a uint64_t value that doesn't fit in the type!"
) ? void (0) : __assert_fail ("(EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && \"getConstant with a uint64_t value that doesn't fit in the type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1353, __extension__ __PRETTY_FUNCTION__))
;
1354 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1355}
1356
1357SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1358 bool isT, bool isO) {
1359 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1360}
1361
1362SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1363 EVT VT, bool isT, bool isO) {
1364 assert(VT.isInteger() && "Cannot create FP integer constant!")(static_cast <bool> (VT.isInteger() && "Cannot create FP integer constant!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Cannot create FP integer constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1364, __extension__ __PRETTY_FUNCTION__))
;
1365
1366 EVT EltVT = VT.getScalarType();
1367 const ConstantInt *Elt = &Val;
1368
1369 // In some cases the vector type is legal but the element type is illegal and
1370 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1371 // inserted value (the type does not need to match the vector element type).
1372 // Any extra bits introduced will be truncated away.
1373 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1374 TargetLowering::TypePromoteInteger) {
1375 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1376 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1377 Elt = ConstantInt::get(*getContext(), NewVal);
1378 }
1379 // In other cases the element type is illegal and needs to be expanded, for
1380 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1381 // the value into n parts and use a vector type with n-times the elements.
1382 // Then bitcast to the type requested.
1383 // Legalizing constants too early makes the DAGCombiner's job harder so we
1384 // only legalize if the DAG tells us we must produce legal types.
1385 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1386 TLI->getTypeAction(*getContext(), EltVT) ==
1387 TargetLowering::TypeExpandInteger) {
1388 const APInt &NewVal = Elt->getValue();
1389 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1390 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1391
1392 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1393 if (VT.isScalableVector()) {
1394 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&(static_cast <bool> (EltVT.getSizeInBits() % ViaEltSizeInBits
== 0 && "Can only handle an even split!") ? void (0)
: __assert_fail ("EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && \"Can only handle an even split!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1395, __extension__ __PRETTY_FUNCTION__))
1395 "Can only handle an even split!")(static_cast <bool> (EltVT.getSizeInBits() % ViaEltSizeInBits
== 0 && "Can only handle an even split!") ? void (0)
: __assert_fail ("EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && \"Can only handle an even split!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1395, __extension__ __PRETTY_FUNCTION__))
;
1396 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1397
1398 SmallVector<SDValue, 2> ScalarParts;
1399 for (unsigned i = 0; i != Parts; ++i)
1400 ScalarParts.push_back(getConstant(
1401 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1402 ViaEltVT, isT, isO));
1403
1404 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1405 }
1406
1407 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1408 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1409
1410 // Check the temporary vector is the correct size. If this fails then
1411 // getTypeToTransformTo() probably returned a type whose size (in bits)
1412 // isn't a power-of-2 factor of the requested type size.
1413 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits())(static_cast <bool> (ViaVecVT.getSizeInBits() == VT.getSizeInBits
()) ? void (0) : __assert_fail ("ViaVecVT.getSizeInBits() == VT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1413, __extension__ __PRETTY_FUNCTION__))
;
1414
1415 SmallVector<SDValue, 2> EltParts;
1416 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1417 EltParts.push_back(getConstant(
1418 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1419 ViaEltVT, isT, isO));
1420
1421 // EltParts is currently in little endian order. If we actually want
1422 // big-endian order then reverse it now.
1423 if (getDataLayout().isBigEndian())
1424 std::reverse(EltParts.begin(), EltParts.end());
1425
1426 // The elements must be reversed when the element order is different
1427 // to the endianness of the elements (because the BITCAST is itself a
1428 // vector shuffle in this situation). However, we do not need any code to
1429 // perform this reversal because getConstant() is producing a vector
1430 // splat.
1431 // This situation occurs in MIPS MSA.
1432
1433 SmallVector<SDValue, 8> Ops;
1434 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1435 llvm::append_range(Ops, EltParts);
1436
1437 SDValue V =
1438 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1439 return V;
1440 }
1441
1442 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&(static_cast <bool> (Elt->getBitWidth() == EltVT.getSizeInBits
() && "APInt size does not match type size!") ? void (
0) : __assert_fail ("Elt->getBitWidth() == EltVT.getSizeInBits() && \"APInt size does not match type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1443, __extension__ __PRETTY_FUNCTION__))
1443 "APInt size does not match type size!")(static_cast <bool> (Elt->getBitWidth() == EltVT.getSizeInBits
() && "APInt size does not match type size!") ? void (
0) : __assert_fail ("Elt->getBitWidth() == EltVT.getSizeInBits() && \"APInt size does not match type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1443, __extension__ __PRETTY_FUNCTION__))
;
1444 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1445 FoldingSetNodeID ID;
1446 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1447 ID.AddPointer(Elt);
1448 ID.AddBoolean(isO);
1449 void *IP = nullptr;
1450 SDNode *N = nullptr;
1451 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1452 if (!VT.isVector())
1453 return SDValue(N, 0);
1454
1455 if (!N) {
1456 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1457 CSEMap.InsertNode(N, IP);
1458 InsertNode(N);
1459 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1460 }
1461
1462 SDValue Result(N, 0);
1463 if (VT.isScalableVector())
1464 Result = getSplatVector(VT, DL, Result);
1465 else if (VT.isVector())
1466 Result = getSplatBuildVector(VT, DL, Result);
1467
1468 return Result;
1469}
1470
1471SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1472 bool isTarget) {
1473 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1474}
1475
1476SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1477 const SDLoc &DL, bool LegalTypes) {
1478 assert(VT.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (VT.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1478, __extension__ __PRETTY_FUNCTION__))
;
1479 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1480 return getConstant(Val, DL, ShiftVT);
1481}
1482
1483SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1484 bool isTarget) {
1485 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1486}
1487
1488SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1489 bool isTarget) {
1490 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1491}
1492
1493SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1494 EVT VT, bool isTarget) {
1495 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!")(static_cast <bool> (VT.isFloatingPoint() && "Cannot create integer FP constant!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Cannot create integer FP constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1495, __extension__ __PRETTY_FUNCTION__))
;
1496
1497 EVT EltVT = VT.getScalarType();
1498
1499 // Do the map lookup using the actual bit pattern for the floating point
1500 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1501 // we don't have issues with SNANs.
1502 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1503 FoldingSetNodeID ID;
1504 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1505 ID.AddPointer(&V);
1506 void *IP = nullptr;
1507 SDNode *N = nullptr;
1508 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1509 if (!VT.isVector())
1510 return SDValue(N, 0);
1511
1512 if (!N) {
1513 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1514 CSEMap.InsertNode(N, IP);
1515 InsertNode(N);
1516 }
1517
1518 SDValue Result(N, 0);
1519 if (VT.isScalableVector())
1520 Result = getSplatVector(VT, DL, Result);
1521 else if (VT.isVector())
1522 Result = getSplatBuildVector(VT, DL, Result);
1523 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1524 return Result;
1525}
1526
1527SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1528 bool isTarget) {
1529 EVT EltVT = VT.getScalarType();
1530 if (EltVT == MVT::f32)
1531 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1532 if (EltVT == MVT::f64)
1533 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1534 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1535 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1536 bool Ignored;
1537 APFloat APF = APFloat(Val);
1538 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1539 &Ignored);
1540 return getConstantFP(APF, DL, VT, isTarget);
1541 }
1542 llvm_unreachable("Unsupported type in getConstantFP")::llvm::llvm_unreachable_internal("Unsupported type in getConstantFP"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1542)
;
1543}
1544
1545SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1546 EVT VT, int64_t Offset, bool isTargetGA,
1547 unsigned TargetFlags) {
1548 assert((TargetFlags == 0 || isTargetGA) &&(static_cast <bool> ((TargetFlags == 0 || isTargetGA) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTargetGA) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1549, __extension__ __PRETTY_FUNCTION__))
1549 "Cannot set target flags on target-independent globals")(static_cast <bool> ((TargetFlags == 0 || isTargetGA) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTargetGA) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1549, __extension__ __PRETTY_FUNCTION__))
;
1550
1551 // Truncate (with sign-extension) the offset value to the pointer size.
1552 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1553 if (BitWidth < 64)
1554 Offset = SignExtend64(Offset, BitWidth);
1555
1556 unsigned Opc;
1557 if (GV->isThreadLocal())
1558 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1559 else
1560 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1561
1562 FoldingSetNodeID ID;
1563 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1564 ID.AddPointer(GV);
1565 ID.AddInteger(Offset);
1566 ID.AddInteger(TargetFlags);
1567 void *IP = nullptr;
1568 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1569 return SDValue(E, 0);
1570
1571 auto *N = newSDNode<GlobalAddressSDNode>(
1572 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1573 CSEMap.InsertNode(N, IP);
1574 InsertNode(N);
1575 return SDValue(N, 0);
1576}
1577
1578SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1579 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1580 FoldingSetNodeID ID;
1581 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1582 ID.AddInteger(FI);
1583 void *IP = nullptr;
1584 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1585 return SDValue(E, 0);
1586
1587 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1588 CSEMap.InsertNode(N, IP);
1589 InsertNode(N);
1590 return SDValue(N, 0);
1591}
1592
1593SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1594 unsigned TargetFlags) {
1595 assert((TargetFlags == 0 || isTarget) &&(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent jump tables")
? void (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent jump tables\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1596, __extension__ __PRETTY_FUNCTION__))
1596 "Cannot set target flags on target-independent jump tables")(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent jump tables")
? void (0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent jump tables\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1596, __extension__ __PRETTY_FUNCTION__))
;
1597 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1598 FoldingSetNodeID ID;
1599 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1600 ID.AddInteger(JTI);
1601 ID.AddInteger(TargetFlags);
1602 void *IP = nullptr;
1603 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1604 return SDValue(E, 0);
1605
1606 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1607 CSEMap.InsertNode(N, IP);
1608 InsertNode(N);
1609 return SDValue(N, 0);
1610}
1611
1612SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1613 MaybeAlign Alignment, int Offset,
1614 bool isTarget, unsigned TargetFlags) {
1615 assert((TargetFlags == 0 || isTarget) &&(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1616, __extension__ __PRETTY_FUNCTION__))
1616 "Cannot set target flags on target-independent globals")(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1616, __extension__ __PRETTY_FUNCTION__))
;
1617 if (!Alignment)
1618 Alignment = shouldOptForSize()
1619 ? getDataLayout().getABITypeAlign(C->getType())
1620 : getDataLayout().getPrefTypeAlign(C->getType());
1621 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1622 FoldingSetNodeID ID;
1623 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1624 ID.AddInteger(Alignment->value());
1625 ID.AddInteger(Offset);
1626 ID.AddPointer(C);
1627 ID.AddInteger(TargetFlags);
1628 void *IP = nullptr;
1629 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1630 return SDValue(E, 0);
1631
1632 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1633 TargetFlags);
1634 CSEMap.InsertNode(N, IP);
1635 InsertNode(N);
1636 SDValue V = SDValue(N, 0);
1637 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1638 return V;
1639}
1640
1641SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1642 MaybeAlign Alignment, int Offset,
1643 bool isTarget, unsigned TargetFlags) {
1644 assert((TargetFlags == 0 || isTarget) &&(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1645, __extension__ __PRETTY_FUNCTION__))
1645 "Cannot set target flags on target-independent globals")(static_cast <bool> ((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals") ? void
(0) : __assert_fail ("(TargetFlags == 0 || isTarget) && \"Cannot set target flags on target-independent globals\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1645, __extension__ __PRETTY_FUNCTION__))
;
1646 if (!Alignment)
1647 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1648 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1649 FoldingSetNodeID ID;
1650 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1651 ID.AddInteger(Alignment->value());
1652 ID.AddInteger(Offset);
1653 C->addSelectionDAGCSEId(ID);
1654 ID.AddInteger(TargetFlags);
1655 void *IP = nullptr;
1656 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1657 return SDValue(E, 0);
1658
1659 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1660 TargetFlags);
1661 CSEMap.InsertNode(N, IP);
1662 InsertNode(N);
1663 return SDValue(N, 0);
1664}
1665
1666SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1667 unsigned TargetFlags) {
1668 FoldingSetNodeID ID;
1669 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1670 ID.AddInteger(Index);
1671 ID.AddInteger(Offset);
1672 ID.AddInteger(TargetFlags);
1673 void *IP = nullptr;
1674 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1675 return SDValue(E, 0);
1676
1677 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1678 CSEMap.InsertNode(N, IP);
1679 InsertNode(N);
1680 return SDValue(N, 0);
1681}
1682
1683SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1684 FoldingSetNodeID ID;
1685 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1686 ID.AddPointer(MBB);
1687 void *IP = nullptr;
1688 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1689 return SDValue(E, 0);
1690
1691 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1692 CSEMap.InsertNode(N, IP);
1693 InsertNode(N);
1694 return SDValue(N, 0);
1695}
1696
1697SDValue SelectionDAG::getValueType(EVT VT) {
1698 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1699 ValueTypeNodes.size())
1700 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1701
1702 SDNode *&N = VT.isExtended() ?
1703 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1704
1705 if (N) return SDValue(N, 0);
1706 N = newSDNode<VTSDNode>(VT);
1707 InsertNode(N);
1708 return SDValue(N, 0);
1709}
1710
1711SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1712 SDNode *&N = ExternalSymbols[Sym];
1713 if (N) return SDValue(N, 0);
1714 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1715 InsertNode(N);
1716 return SDValue(N, 0);
1717}
1718
1719SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1720 SDNode *&N = MCSymbols[Sym];
1721 if (N)
1722 return SDValue(N, 0);
1723 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1724 InsertNode(N);
1725 return SDValue(N, 0);
1726}
1727
1728SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1729 unsigned TargetFlags) {
1730 SDNode *&N =
1731 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1732 if (N) return SDValue(N, 0);
1733 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1734 InsertNode(N);
1735 return SDValue(N, 0);
1736}
1737
1738SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1739 if ((unsigned)Cond >= CondCodeNodes.size())
1740 CondCodeNodes.resize(Cond+1);
1741
1742 if (!CondCodeNodes[Cond]) {
1743 auto *N = newSDNode<CondCodeSDNode>(Cond);
1744 CondCodeNodes[Cond] = N;
1745 InsertNode(N);
1746 }
1747
1748 return SDValue(CondCodeNodes[Cond], 0);
1749}
1750
1751SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
1752 APInt One(ResVT.getScalarSizeInBits(), 1);
1753 return getStepVector(DL, ResVT, One);
1754}
1755
1756SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
1757 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth())(static_cast <bool> (ResVT.getScalarSizeInBits() == StepVal
.getBitWidth()) ? void (0) : __assert_fail ("ResVT.getScalarSizeInBits() == StepVal.getBitWidth()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1757, __extension__ __PRETTY_FUNCTION__))
;
1758 if (ResVT.isScalableVector())
1759 return getNode(
1760 ISD::STEP_VECTOR, DL, ResVT,
1761 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
1762
1763 SmallVector<SDValue, 16> OpsStepConstants;
1764 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
1765 OpsStepConstants.push_back(
1766 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
1767 return getBuildVector(ResVT, DL, OpsStepConstants);
1768}
1769
1770/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1771/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1772static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1773 std::swap(N1, N2);
1774 ShuffleVectorSDNode::commuteMask(M);
1775}
1776
1777SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1778 SDValue N2, ArrayRef<int> Mask) {
1779 assert(VT.getVectorNumElements() == Mask.size() &&(static_cast <bool> (VT.getVectorNumElements() == Mask.
size() && "Must have the same number of vector elements as mask elements!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == Mask.size() && \"Must have the same number of vector elements as mask elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1780, __extension__ __PRETTY_FUNCTION__))
1780 "Must have the same number of vector elements as mask elements!")(static_cast <bool> (VT.getVectorNumElements() == Mask.
size() && "Must have the same number of vector elements as mask elements!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == Mask.size() && \"Must have the same number of vector elements as mask elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1780, __extension__ __PRETTY_FUNCTION__))
;
1781 assert(VT == N1.getValueType() && VT == N2.getValueType() &&(static_cast <bool> (VT == N1.getValueType() &&
VT == N2.getValueType() && "Invalid VECTOR_SHUFFLE")
? void (0) : __assert_fail ("VT == N1.getValueType() && VT == N2.getValueType() && \"Invalid VECTOR_SHUFFLE\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1782, __extension__ __PRETTY_FUNCTION__))
1782 "Invalid VECTOR_SHUFFLE")(static_cast <bool> (VT == N1.getValueType() &&
VT == N2.getValueType() && "Invalid VECTOR_SHUFFLE")
? void (0) : __assert_fail ("VT == N1.getValueType() && VT == N2.getValueType() && \"Invalid VECTOR_SHUFFLE\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1782, __extension__ __PRETTY_FUNCTION__))
;
1783
1784 // Canonicalize shuffle undef, undef -> undef
1785 if (N1.isUndef() && N2.isUndef())
1786 return getUNDEF(VT);
1787
1788 // Validate that all indices in Mask are within the range of the elements
1789 // input to the shuffle.
1790 int NElts = Mask.size();
1791 assert(llvm::all_of(Mask,(static_cast <bool> (llvm::all_of(Mask, [&](int M) {
return M < (NElts * 2) && M >= -1; }) &&
"Index out of range") ? void (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1793, __extension__ __PRETTY_FUNCTION__))
1792 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&(static_cast <bool> (llvm::all_of(Mask, [&](int M) {
return M < (NElts * 2) && M >= -1; }) &&
"Index out of range") ? void (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1793, __extension__ __PRETTY_FUNCTION__))
1793 "Index out of range")(static_cast <bool> (llvm::all_of(Mask, [&](int M) {
return M < (NElts * 2) && M >= -1; }) &&
"Index out of range") ? void (0) : __assert_fail ("llvm::all_of(Mask, [&](int M) { return M < (NElts * 2) && M >= -1; }) && \"Index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 1793, __extension__ __PRETTY_FUNCTION__))
;
1794
1795 // Copy the mask so we can do any needed cleanup.
1796 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1797
1798 // Canonicalize shuffle v, v -> v, undef
1799 if (N1 == N2) {
1800 N2 = getUNDEF(VT);
1801 for (int i = 0; i != NElts; ++i)
1802 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1803 }
1804
1805 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1806 if (N1.isUndef())
1807 commuteShuffle(N1, N2, MaskVec);
1808
1809 if (TLI->hasVectorBlend()) {
1810 // If shuffling a splat, try to blend the splat instead. We do this here so
1811 // that even when this arises during lowering we don't have to re-handle it.
1812 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1813 BitVector UndefElements;
1814 SDValue Splat = BV->getSplatValue(&UndefElements);
1815 if (!Splat)
1816 return;
1817
1818 for (int i = 0; i < NElts; ++i) {
1819 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1820 continue;
1821
1822 // If this input comes from undef, mark it as such.
1823 if (UndefElements[MaskVec[i] - Offset]) {
1824 MaskVec[i] = -1;
1825 continue;
1826 }
1827
1828 // If we can blend a non-undef lane, use that instead.
1829 if (!UndefElements[i])
1830 MaskVec[i] = i + Offset;
1831 }
1832 };
1833 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1834 BlendSplat(N1BV, 0);
1835 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1836 BlendSplat(N2BV, NElts);
1837 }
1838
1839 // Canonicalize all index into lhs, -> shuffle lhs, undef
1840 // Canonicalize all index into rhs, -> shuffle rhs, undef
1841 bool AllLHS = true, AllRHS = true;
1842 bool N2Undef = N2.isUndef();
1843 for (int i = 0; i != NElts; ++i) {
1844 if (MaskVec[i] >= NElts) {
1845 if (N2Undef)
1846 MaskVec[i] = -1;
1847 else
1848 AllLHS = false;
1849 } else if (MaskVec[i] >= 0) {
1850 AllRHS = false;
1851 }
1852 }
1853 if (AllLHS && AllRHS)
1854 return getUNDEF(VT);
1855 if (AllLHS && !N2Undef)
1856 N2 = getUNDEF(VT);
1857 if (AllRHS) {
1858 N1 = getUNDEF(VT);
1859 commuteShuffle(N1, N2, MaskVec);
1860 }
1861 // Reset our undef status after accounting for the mask.
1862 N2Undef = N2.isUndef();
1863 // Re-check whether both sides ended up undef.
1864 if (N1.isUndef() && N2Undef)
1865 return getUNDEF(VT);
1866
1867 // If Identity shuffle return that node.
1868 bool Identity = true, AllSame = true;
1869 for (int i = 0; i != NElts; ++i) {
1870 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1871 if (MaskVec[i] != MaskVec[0]) AllSame = false;
1872 }
1873 if (Identity && NElts)
1874 return N1;
1875
1876 // Shuffling a constant splat doesn't change the result.
1877 if (N2Undef) {
1878 SDValue V = N1;
1879
1880 // Look through any bitcasts. We check that these don't change the number
1881 // (and size) of elements and just changes their types.
1882 while (V.getOpcode() == ISD::BITCAST)
1883 V = V->getOperand(0);
1884
1885 // A splat should always show up as a build vector node.
1886 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1887 BitVector UndefElements;
1888 SDValue Splat = BV->getSplatValue(&UndefElements);
1889 // If this is a splat of an undef, shuffling it is also undef.
1890 if (Splat && Splat.isUndef())
1891 return getUNDEF(VT);
1892
1893 bool SameNumElts =
1894 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1895
1896 // We only have a splat which can skip shuffles if there is a splatted
1897 // value and no undef lanes rearranged by the shuffle.
1898 if (Splat && UndefElements.none()) {
1899 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1900 // number of elements match or the value splatted is a zero constant.
1901 if (SameNumElts)
1902 return N1;
1903 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1904 if (C->isNullValue())
1905 return N1;
1906 }
1907
1908 // If the shuffle itself creates a splat, build the vector directly.
1909 if (AllSame && SameNumElts) {
1910 EVT BuildVT = BV->getValueType(0);
1911 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1912 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1913
1914 // We may have jumped through bitcasts, so the type of the
1915 // BUILD_VECTOR may not match the type of the shuffle.
1916 if (BuildVT != VT)
1917 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1918 return NewBV;
1919 }
1920 }
1921 }
1922
1923 FoldingSetNodeID ID;
1924 SDValue Ops[2] = { N1, N2 };
1925 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1926 for (int i = 0; i != NElts; ++i)
1927 ID.AddInteger(MaskVec[i]);
1928
1929 void* IP = nullptr;
1930 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1931 return SDValue(E, 0);
1932
1933 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1934 // SDNode doesn't have access to it. This memory will be "leaked" when
1935 // the node is deallocated, but recovered when the NodeAllocator is released.
1936 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1937 llvm::copy(MaskVec, MaskAlloc);
1938
1939 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1940 dl.getDebugLoc(), MaskAlloc);
1941 createOperands(N, Ops);
1942
1943 CSEMap.InsertNode(N, IP);
1944 InsertNode(N);
1945 SDValue V = SDValue(N, 0);
1946 NewSDValueDbgMsg(V, "Creating new node: ", this);
1947 return V;
1948}
1949
1950SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1951 EVT VT = SV.getValueType(0);
1952 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1953 ShuffleVectorSDNode::commuteMask(MaskVec);
1954
1955 SDValue Op0 = SV.getOperand(0);
1956 SDValue Op1 = SV.getOperand(1);
1957 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1958}
1959
1960SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1961 FoldingSetNodeID ID;
1962 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1963 ID.AddInteger(RegNo);
1964 void *IP = nullptr;
1965 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1966 return SDValue(E, 0);
1967
1968 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1969 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1970 CSEMap.InsertNode(N, IP);
1971 InsertNode(N);
1972 return SDValue(N, 0);
1973}
1974
1975SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1976 FoldingSetNodeID ID;
1977 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1978 ID.AddPointer(RegMask);
1979 void *IP = nullptr;
1980 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1981 return SDValue(E, 0);
1982
1983 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1984 CSEMap.InsertNode(N, IP);
1985 InsertNode(N);
1986 return SDValue(N, 0);
1987}
1988
1989SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1990 MCSymbol *Label) {
1991 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1992}
1993
1994SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1995 SDValue Root, MCSymbol *Label) {
1996 FoldingSetNodeID ID;
1997 SDValue Ops[] = { Root };
1998 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1999 ID.AddPointer(Label);
2000 void *IP = nullptr;
2001 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2002 return SDValue(E, 0);
2003
2004 auto *N =
2005 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2006 createOperands(N, Ops);
2007
2008 CSEMap.InsertNode(N, IP);
2009 InsertNode(N);
2010 return SDValue(N, 0);
2011}
2012
2013SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2014 int64_t Offset, bool isTarget,
2015 unsigned TargetFlags) {
2016 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2017
2018 FoldingSetNodeID ID;
2019 AddNodeIDNode(ID, Opc, getVTList(VT), None);
2020 ID.AddPointer(BA);
2021 ID.AddInteger(Offset);
2022 ID.AddInteger(TargetFlags);
2023 void *IP = nullptr;
2024 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2025 return SDValue(E, 0);
2026
2027 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2028 CSEMap.InsertNode(N, IP);
2029 InsertNode(N);
2030 return SDValue(N, 0);
2031}
2032
2033SDValue SelectionDAG::getSrcValue(const Value *V) {
2034 FoldingSetNodeID ID;
2035 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
2036 ID.AddPointer(V);
2037
2038 void *IP = nullptr;
2039 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2040 return SDValue(E, 0);
2041
2042 auto *N = newSDNode<SrcValueSDNode>(V);
2043 CSEMap.InsertNode(N, IP);
2044 InsertNode(N);
2045 return SDValue(N, 0);
2046}
2047
2048SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2049 FoldingSetNodeID ID;
2050 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
2051 ID.AddPointer(MD);
2052
2053 void *IP = nullptr;
2054 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2055 return SDValue(E, 0);
2056
2057 auto *N = newSDNode<MDNodeSDNode>(MD);
2058 CSEMap.InsertNode(N, IP);
2059 InsertNode(N);
2060 return SDValue(N, 0);
2061}
2062
2063SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2064 if (VT == V.getValueType())
2065 return V;
2066
2067 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2068}
2069
2070SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2071 unsigned SrcAS, unsigned DestAS) {
2072 SDValue Ops[] = {Ptr};
2073 FoldingSetNodeID ID;
2074 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
2075 ID.AddInteger(SrcAS);
2076 ID.AddInteger(DestAS);
2077
2078 void *IP = nullptr;
2079 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2080 return SDValue(E, 0);
2081
2082 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2083 VT, SrcAS, DestAS);
2084 createOperands(N, Ops);
2085
2086 CSEMap.InsertNode(N, IP);
2087 InsertNode(N);
2088 return SDValue(N, 0);
2089}
2090
2091SDValue SelectionDAG::getFreeze(SDValue V) {
2092 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2093}
2094
2095/// getShiftAmountOperand - Return the specified value casted to
2096/// the target's desired shift amount type.
2097SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2098 EVT OpTy = Op.getValueType();
2099 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2100 if (OpTy == ShTy || OpTy.isVector()) return Op;
2101
2102 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2103}
2104
2105SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2106 SDLoc dl(Node);
2107 const TargetLowering &TLI = getTargetLoweringInfo();
2108 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2109 EVT VT = Node->getValueType(0);
2110 SDValue Tmp1 = Node->getOperand(0);
2111 SDValue Tmp2 = Node->getOperand(1);
2112 const MaybeAlign MA(Node->getConstantOperandVal(3));
2113
2114 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1
Calling 'SelectionDAG::getLoad'
2115 Tmp2, MachinePointerInfo(V));
2116 SDValue VAList = VAListLoad;
2117
2118 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2119 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2120 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2121
2122 VAList =
2123 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2124 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2125 }
2126
2127 // Increment the pointer, VAList, to the next vaarg
2128 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2129 getConstant(getDataLayout().getTypeAllocSize(
2130 VT.getTypeForEVT(*getContext())),
2131 dl, VAList.getValueType()));
2132 // Store the incremented VAList to the legalized pointer
2133 Tmp1 =
2134 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2135 // Load the actual argument out of the pointer VAList
2136 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2137}
2138
2139SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2140 SDLoc dl(Node);
2141 const TargetLowering &TLI = getTargetLoweringInfo();
2142 // This defaults to loading a pointer from the input and storing it to the
2143 // output, returning the chain.
2144 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2145 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2146 SDValue Tmp1 =
2147 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2148 Node->getOperand(2), MachinePointerInfo(VS));
2149 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2150 MachinePointerInfo(VD));
2151}
2152
2153Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2154 const DataLayout &DL = getDataLayout();
2155 Type *Ty = VT.getTypeForEVT(*getContext());
2156 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2157
2158 if (TLI->isTypeLegal(VT) || !VT.isVector())
2159 return RedAlign;
2160
2161 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2162 const Align StackAlign = TFI->getStackAlign();
2163
2164 // See if we can choose a smaller ABI alignment in cases where it's an
2165 // illegal vector type that will get broken down.
2166 if (RedAlign > StackAlign) {
2167 EVT IntermediateVT;
2168 MVT RegisterVT;
2169 unsigned NumIntermediates;
2170 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2171 NumIntermediates, RegisterVT);
2172 Ty = IntermediateVT.getTypeForEVT(*getContext());
2173 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2174 if (RedAlign2 < RedAlign)
2175 RedAlign = RedAlign2;
2176 }
2177
2178 return RedAlign;
2179}
2180
2181SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2182 MachineFrameInfo &MFI = MF->getFrameInfo();
2183 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2184 int StackID = 0;
2185 if (Bytes.isScalable())
2186 StackID = TFI->getStackIDForScalableVectors();
2187 // The stack id gives an indication of whether the object is scalable or
2188 // not, so it's safe to pass in the minimum size here.
2189 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment,
2190 false, nullptr, StackID);
2191 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2192}
2193
2194SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2195 Type *Ty = VT.getTypeForEVT(*getContext());
2196 Align StackAlign =
2197 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2198 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2199}
2200
2201SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2202 TypeSize VT1Size = VT1.getStoreSize();
2203 TypeSize VT2Size = VT2.getStoreSize();
2204 assert(VT1Size.isScalable() == VT2Size.isScalable() &&(static_cast <bool> (VT1Size.isScalable() == VT2Size.isScalable
() && "Don't know how to choose the maximum size when creating a stack "
"temporary") ? void (0) : __assert_fail ("VT1Size.isScalable() == VT2Size.isScalable() && \"Don't know how to choose the maximum size when creating a stack \" \"temporary\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2206, __extension__ __PRETTY_FUNCTION__))
2205 "Don't know how to choose the maximum size when creating a stack "(static_cast <bool> (VT1Size.isScalable() == VT2Size.isScalable
() && "Don't know how to choose the maximum size when creating a stack "
"temporary") ? void (0) : __assert_fail ("VT1Size.isScalable() == VT2Size.isScalable() && \"Don't know how to choose the maximum size when creating a stack \" \"temporary\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2206, __extension__ __PRETTY_FUNCTION__))
2206 "temporary")(static_cast <bool> (VT1Size.isScalable() == VT2Size.isScalable
() && "Don't know how to choose the maximum size when creating a stack "
"temporary") ? void (0) : __assert_fail ("VT1Size.isScalable() == VT2Size.isScalable() && \"Don't know how to choose the maximum size when creating a stack \" \"temporary\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2206, __extension__ __PRETTY_FUNCTION__))
;
2207 TypeSize Bytes =
2208 VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size;
2209
2210 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2211 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2212 const DataLayout &DL = getDataLayout();
2213 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2214 return CreateStackTemporary(Bytes, Align);
2215}
2216
2217SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2218 ISD::CondCode Cond, const SDLoc &dl) {
2219 EVT OpVT = N1.getValueType();
2220
2221 // These setcc operations always fold.
2222 switch (Cond) {
2223 default: break;
2224 case ISD::SETFALSE:
2225 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2226 case ISD::SETTRUE:
2227 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2228
2229 case ISD::SETOEQ:
2230 case ISD::SETOGT:
2231 case ISD::SETOGE:
2232 case ISD::SETOLT:
2233 case ISD::SETOLE:
2234 case ISD::SETONE:
2235 case ISD::SETO:
2236 case ISD::SETUO:
2237 case ISD::SETUEQ:
2238 case ISD::SETUNE:
2239 assert(!OpVT.isInteger() && "Illegal setcc for integer!")(static_cast <bool> (!OpVT.isInteger() && "Illegal setcc for integer!"
) ? void (0) : __assert_fail ("!OpVT.isInteger() && \"Illegal setcc for integer!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2239, __extension__ __PRETTY_FUNCTION__))
;
2240 break;
2241 }
2242
2243 if (OpVT.isInteger()) {
2244 // For EQ and NE, we can always pick a value for the undef to make the
2245 // predicate pass or fail, so we can return undef.
2246 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2247 // icmp eq/ne X, undef -> undef.
2248 if ((N1.isUndef() || N2.isUndef()) &&
2249 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2250 return getUNDEF(VT);
2251
2252 // If both operands are undef, we can return undef for int comparison.
2253 // icmp undef, undef -> undef.
2254 if (N1.isUndef() && N2.isUndef())
2255 return getUNDEF(VT);
2256
2257 // icmp X, X -> true/false
2258 // icmp X, undef -> true/false because undef could be X.
2259 if (N1 == N2)
2260 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2261 }
2262
2263 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2264 const APInt &C2 = N2C->getAPIntValue();
2265 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2266 const APInt &C1 = N1C->getAPIntValue();
2267
2268 switch (Cond) {
2269 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2269)
;
2270 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT);
2271 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT);
2272 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
2273 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
2274 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
2275 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2276 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2277 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2278 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2279 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2280 }
2281 }
2282 }
2283
2284 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2285 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2286
2287 if (N1CFP && N2CFP) {
2288 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2289 switch (Cond) {
2290 default: break;
2291 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2292 return getUNDEF(VT);
2293 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2294 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2295 OpVT);
2296 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2297 return getUNDEF(VT);
2298 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2299 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2300 R==APFloat::cmpLessThan, dl, VT,
2301 OpVT);
2302 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2303 return getUNDEF(VT);
2304 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2305 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2306 OpVT);
2307 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2308 return getUNDEF(VT);
2309 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2310 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2311 VT, OpVT);
2312 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2313 return getUNDEF(VT);
2314 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2315 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2316 R==APFloat::cmpEqual, dl, VT,
2317 OpVT);
2318 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2319 return getUNDEF(VT);
2320 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2321 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2322 R==APFloat::cmpEqual, dl, VT, OpVT);
2323 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2324 OpVT);
2325 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2326 OpVT);
2327 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2328 R==APFloat::cmpEqual, dl, VT,
2329 OpVT);
2330 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2331 OpVT);
2332 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2333 R==APFloat::cmpLessThan, dl, VT,
2334 OpVT);
2335 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2336 R==APFloat::cmpUnordered, dl, VT,
2337 OpVT);
2338 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2339 VT, OpVT);
2340 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2341 OpVT);
2342 }
2343 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2344 // Ensure that the constant occurs on the RHS.
2345 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2346 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2347 return SDValue();
2348 return getSetCC(dl, VT, N2, N1, SwappedCond);
2349 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2350 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2351 // If an operand is known to be a nan (or undef that could be a nan), we can
2352 // fold it.
2353 // Choosing NaN for the undef will always make unordered comparison succeed
2354 // and ordered comparison fails.
2355 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2356 switch (ISD::getUnorderedFlavor(Cond)) {
2357 default:
2358 llvm_unreachable("Unknown flavor!")::llvm::llvm_unreachable_internal("Unknown flavor!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2358)
;
2359 case 0: // Known false.
2360 return getBoolConstant(false, dl, VT, OpVT);
2361 case 1: // Known true.
2362 return getBoolConstant(true, dl, VT, OpVT);
2363 case 2: // Undefined.
2364 return getUNDEF(VT);
2365 }
2366 }
2367
2368 // Could not fold it.
2369 return SDValue();
2370}
2371
2372/// See if the specified operand can be simplified with the knowledge that only
2373/// the bits specified by DemandedBits are used.
2374/// TODO: really we should be making this into the DAG equivalent of
2375/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2376SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2377 EVT VT = V.getValueType();
2378
2379 if (VT.isScalableVector())
2380 return SDValue();
2381
2382 APInt DemandedElts = VT.isVector()
2383 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2384 : APInt(1, 1);
2385 return GetDemandedBits(V, DemandedBits, DemandedElts);
2386}
2387
2388/// See if the specified operand can be simplified with the knowledge that only
2389/// the bits specified by DemandedBits are used in the elements specified by
2390/// DemandedElts.
2391/// TODO: really we should be making this into the DAG equivalent of
2392/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2393SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2394 const APInt &DemandedElts) {
2395 switch (V.getOpcode()) {
2396 default:
2397 return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts,
2398 *this, 0);
2399 case ISD::Constant: {
2400 const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue();
2401 APInt NewVal = CVal & DemandedBits;
2402 if (NewVal != CVal)
2403 return getConstant(NewVal, SDLoc(V), V.getValueType());
2404 break;
2405 }
2406 case ISD::SRL:
2407 // Only look at single-use SRLs.
2408 if (!V.getNode()->hasOneUse())
2409 break;
2410 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2411 // See if we can recursively simplify the LHS.
2412 unsigned Amt = RHSC->getZExtValue();
2413
2414 // Watch out for shift count overflow though.
2415 if (Amt >= DemandedBits.getBitWidth())
2416 break;
2417 APInt SrcDemandedBits = DemandedBits << Amt;
2418 if (SDValue SimplifyLHS =
2419 GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2420 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2421 V.getOperand(1));
2422 }
2423 break;
2424 }
2425 return SDValue();
2426}
2427
2428/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2429/// use this predicate to simplify operations downstream.
2430bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2431 unsigned BitWidth = Op.getScalarValueSizeInBits();
2432 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2433}
2434
2435/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2436/// this predicate to simplify operations downstream. Mask is known to be zero
2437/// for bits that V cannot have.
2438bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2439 unsigned Depth) const {
2440 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2441}
2442
2443/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2444/// DemandedElts. We use this predicate to simplify operations downstream.
2445/// Mask is known to be zero for bits that V cannot have.
2446bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2447 const APInt &DemandedElts,
2448 unsigned Depth) const {
2449 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2450}
2451
2452/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2453bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2454 unsigned Depth) const {
2455 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2456}
2457
2458/// isSplatValue - Return true if the vector V has the same value
2459/// across all DemandedElts. For scalable vectors it does not make
2460/// sense to specify which elements are demanded or undefined, therefore
2461/// they are simply ignored.
2462bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2463 APInt &UndefElts, unsigned Depth) {
2464 EVT VT = V.getValueType();
2465 assert(VT.isVector() && "Vector type expected")(static_cast <bool> (VT.isVector() && "Vector type expected"
) ? void (0) : __assert_fail ("VT.isVector() && \"Vector type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2465, __extension__ __PRETTY_FUNCTION__))
;
2466
2467 if (!VT.isScalableVector() && !DemandedElts)
2468 return false; // No demanded elts, better to assume we don't know anything.
2469
2470 if (Depth >= MaxRecursionDepth)
2471 return false; // Limit search depth.
2472
2473 // Deal with some common cases here that work for both fixed and scalable
2474 // vector types.
2475 switch (V.getOpcode()) {
2476 case ISD::SPLAT_VECTOR:
2477 UndefElts = V.getOperand(0).isUndef()
2478 ? APInt::getAllOnesValue(DemandedElts.getBitWidth())
2479 : APInt(DemandedElts.getBitWidth(), 0);
2480 return true;
2481 case ISD::ADD:
2482 case ISD::SUB:
2483 case ISD::AND:
2484 case ISD::XOR:
2485 case ISD::OR: {
2486 APInt UndefLHS, UndefRHS;
2487 SDValue LHS = V.getOperand(0);
2488 SDValue RHS = V.getOperand(1);
2489 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2490 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2491 UndefElts = UndefLHS | UndefRHS;
2492 return true;
2493 }
2494 return false;
2495 }
2496 case ISD::ABS:
2497 case ISD::TRUNCATE:
2498 case ISD::SIGN_EXTEND:
2499 case ISD::ZERO_EXTEND:
2500 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2501 }
2502
2503 // We don't support other cases than those above for scalable vectors at
2504 // the moment.
2505 if (VT.isScalableVector())
2506 return false;
2507
2508 unsigned NumElts = VT.getVectorNumElements();
2509 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch")(static_cast <bool> (NumElts == DemandedElts.getBitWidth
() && "Vector size mismatch") ? void (0) : __assert_fail
("NumElts == DemandedElts.getBitWidth() && \"Vector size mismatch\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2509, __extension__ __PRETTY_FUNCTION__))
;
2510 UndefElts = APInt::getNullValue(NumElts);
2511
2512 switch (V.getOpcode()) {
2513 case ISD::BUILD_VECTOR: {
2514 SDValue Scl;
2515 for (unsigned i = 0; i != NumElts; ++i) {
2516 SDValue Op = V.getOperand(i);
2517 if (Op.isUndef()) {
2518 UndefElts.setBit(i);
2519 continue;
2520 }
2521 if (!DemandedElts[i])
2522 continue;
2523 if (Scl && Scl != Op)
2524 return false;
2525 Scl = Op;
2526 }
2527 return true;
2528 }
2529 case ISD::VECTOR_SHUFFLE: {
2530 // Check if this is a shuffle node doing a splat.
2531 // TODO: Do we need to handle shuffle(splat, undef, mask)?
2532 int SplatIndex = -1;
2533 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2534 for (int i = 0; i != (int)NumElts; ++i) {
2535 int M = Mask[i];
2536 if (M < 0) {
2537 UndefElts.setBit(i);
2538 continue;
2539 }
2540 if (!DemandedElts[i])
2541 continue;
2542 if (0 <= SplatIndex && SplatIndex != M)
2543 return false;
2544 SplatIndex = M;
2545 }
2546 return true;
2547 }
2548 case ISD::EXTRACT_SUBVECTOR: {
2549 // Offset the demanded elts by the subvector index.
2550 SDValue Src = V.getOperand(0);
2551 // We don't support scalable vectors at the moment.
2552 if (Src.getValueType().isScalableVector())
2553 return false;
2554 uint64_t Idx = V.getConstantOperandVal(1);
2555 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2556 APInt UndefSrcElts;
2557 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2558 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2559 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2560 return true;
2561 }
2562 break;
2563 }
2564 }
2565
2566 return false;
2567}
2568
2569/// Helper wrapper to main isSplatValue function.
2570bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2571 EVT VT = V.getValueType();
2572 assert(VT.isVector() && "Vector type expected")(static_cast <bool> (VT.isVector() && "Vector type expected"
) ? void (0) : __assert_fail ("VT.isVector() && \"Vector type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2572, __extension__ __PRETTY_FUNCTION__))
;
2573
2574 APInt UndefElts;
2575 APInt DemandedElts;
2576
2577 // For now we don't support this with scalable vectors.
2578 if (!VT.isScalableVector())
2579 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2580 return isSplatValue(V, DemandedElts, UndefElts) &&
2581 (AllowUndefs || !UndefElts);
2582}
2583
2584SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2585 V = peekThroughExtractSubvectors(V);
2586
2587 EVT VT = V.getValueType();
2588 unsigned Opcode = V.getOpcode();
2589 switch (Opcode) {
2590 default: {
2591 APInt UndefElts;
2592 APInt DemandedElts;
2593
2594 if (!VT.isScalableVector())
2595 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2596
2597 if (isSplatValue(V, DemandedElts, UndefElts)) {
2598 if (VT.isScalableVector()) {
2599 // DemandedElts and UndefElts are ignored for scalable vectors, since
2600 // the only supported cases are SPLAT_VECTOR nodes.
2601 SplatIdx = 0;
2602 } else {
2603 // Handle case where all demanded elements are UNDEF.
2604 if (DemandedElts.isSubsetOf(UndefElts)) {
2605 SplatIdx = 0;
2606 return getUNDEF(VT);
2607 }
2608 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2609 }
2610 return V;
2611 }
2612 break;
2613 }
2614 case ISD::SPLAT_VECTOR:
2615 SplatIdx = 0;
2616 return V;
2617 case ISD::VECTOR_SHUFFLE: {
2618 if (VT.isScalableVector())
2619 return SDValue();
2620
2621 // Check if this is a shuffle node doing a splat.
2622 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2623 // getTargetVShiftNode currently struggles without the splat source.
2624 auto *SVN = cast<ShuffleVectorSDNode>(V);
2625 if (!SVN->isSplat())
2626 break;
2627 int Idx = SVN->getSplatIndex();
2628 int NumElts = V.getValueType().getVectorNumElements();
2629 SplatIdx = Idx % NumElts;
2630 return V.getOperand(Idx / NumElts);
2631 }
2632 }
2633
2634 return SDValue();
2635}
2636
2637SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2638 int SplatIdx;
2639 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2640 EVT SVT = SrcVector.getValueType().getScalarType();
2641 EVT LegalSVT = SVT;
2642 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2643 if (!SVT.isInteger())
2644 return SDValue();
2645 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2646 if (LegalSVT.bitsLT(SVT))
2647 return SDValue();
2648 }
2649 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2650 getVectorIdxConstant(SplatIdx, SDLoc(V)));
2651 }
2652 return SDValue();
2653}
2654
2655const APInt *
2656SelectionDAG::getValidShiftAmountConstant(SDValue V,
2657 const APInt &DemandedElts) const {
2658 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2660, __extension__ __PRETTY_FUNCTION__))
2659 V.getOpcode() == ISD::SRA) &&(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2660, __extension__ __PRETTY_FUNCTION__))
2660 "Unknown shift node")(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2660, __extension__ __PRETTY_FUNCTION__))
;
2661 unsigned BitWidth = V.getScalarValueSizeInBits();
2662 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2663 // Shifting more than the bitwidth is not valid.
2664 const APInt &ShAmt = SA->getAPIntValue();
2665 if (ShAmt.ult(BitWidth))
2666 return &ShAmt;
2667 }
2668 return nullptr;
2669}
2670
2671const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2672 SDValue V, const APInt &DemandedElts) const {
2673 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2675, __extension__ __PRETTY_FUNCTION__))
2674 V.getOpcode() == ISD::SRA) &&(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2675, __extension__ __PRETTY_FUNCTION__))
2675 "Unknown shift node")(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2675, __extension__ __PRETTY_FUNCTION__))
;
2676 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2677 return ValidAmt;
2678 unsigned BitWidth = V.getScalarValueSizeInBits();
2679 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2680 if (!BV)
2681 return nullptr;
2682 const APInt *MinShAmt = nullptr;
2683 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2684 if (!DemandedElts[i])
2685 continue;
2686 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2687 if (!SA)
2688 return nullptr;
2689 // Shifting more than the bitwidth is not valid.
2690 const APInt &ShAmt = SA->getAPIntValue();
2691 if (ShAmt.uge(BitWidth))
2692 return nullptr;
2693 if (MinShAmt && MinShAmt->ule(ShAmt))
2694 continue;
2695 MinShAmt = &ShAmt;
2696 }
2697 return MinShAmt;
2698}
2699
2700const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
2701 SDValue V, const APInt &DemandedElts) const {
2702 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2704, __extension__ __PRETTY_FUNCTION__))
2703 V.getOpcode() == ISD::SRA) &&(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2704, __extension__ __PRETTY_FUNCTION__))
2704 "Unknown shift node")(static_cast <bool> ((V.getOpcode() == ISD::SHL || V.getOpcode
() == ISD::SRL || V.getOpcode() == ISD::SRA) && "Unknown shift node"
) ? void (0) : __assert_fail ("(V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || V.getOpcode() == ISD::SRA) && \"Unknown shift node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2704, __extension__ __PRETTY_FUNCTION__))
;
2705 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2706 return ValidAmt;
2707 unsigned BitWidth = V.getScalarValueSizeInBits();
2708 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2709 if (!BV)
2710 return nullptr;
2711 const APInt *MaxShAmt = nullptr;
2712 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2713 if (!DemandedElts[i])
2714 continue;
2715 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2716 if (!SA)
2717 return nullptr;
2718 // Shifting more than the bitwidth is not valid.
2719 const APInt &ShAmt = SA->getAPIntValue();
2720 if (ShAmt.uge(BitWidth))
2721 return nullptr;
2722 if (MaxShAmt && MaxShAmt->uge(ShAmt))
2723 continue;
2724 MaxShAmt = &ShAmt;
2725 }
2726 return MaxShAmt;
2727}
2728
2729/// Determine which bits of Op are known to be either zero or one and return
2730/// them in Known. For vectors, the known bits are those that are shared by
2731/// every vector element.
2732KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2733 EVT VT = Op.getValueType();
2734
2735 // TOOD: Until we have a plan for how to represent demanded elements for
2736 // scalable vectors, we can just bail out for now.
2737 if (Op.getValueType().isScalableVector()) {
2738 unsigned BitWidth = Op.getScalarValueSizeInBits();
2739 return KnownBits(BitWidth);
2740 }
2741
2742 APInt DemandedElts = VT.isVector()
2743 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2744 : APInt(1, 1);
2745 return computeKnownBits(Op, DemandedElts, Depth);
2746}
2747
2748/// Determine which bits of Op are known to be either zero or one and return
2749/// them in Known. The DemandedElts argument allows us to only collect the known
2750/// bits that are shared by the requested vector elements.
2751KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2752 unsigned Depth) const {
2753 unsigned BitWidth = Op.getScalarValueSizeInBits();
2754
2755 KnownBits Known(BitWidth); // Don't know anything.
2756
2757 // TOOD: Until we have a plan for how to represent demanded elements for
2758 // scalable vectors, we can just bail out for now.
2759 if (Op.getValueType().isScalableVector())
2760 return Known;
2761
2762 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2763 // We know all of the bits for a constant!
2764 return KnownBits::makeConstant(C->getAPIntValue());
2765 }
2766 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2767 // We know all of the bits for a constant fp!
2768 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
2769 }
2770
2771 if (Depth >= MaxRecursionDepth)
2772 return Known; // Limit search depth.
2773
2774 KnownBits Known2;
2775 unsigned NumElts = DemandedElts.getBitWidth();
2776 assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__))
2777 NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__))
2778 "Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__))
;
2779
2780 if (!DemandedElts)
2781 return Known; // No demanded elts, better to assume we don't know anything.
2782
2783 unsigned Opcode = Op.getOpcode();
2784 switch (Opcode) {
2785 case ISD::BUILD_VECTOR:
2786 // Collect the known bits that are shared by every demanded vector element.
2787 Known.Zero.setAllBits(); Known.One.setAllBits();
2788 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2789 if (!DemandedElts[i])
2790 continue;
2791
2792 SDValue SrcOp = Op.getOperand(i);
2793 Known2 = computeKnownBits(SrcOp, Depth + 1);
2794
2795 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2796 if (SrcOp.getValueSizeInBits() != BitWidth) {
2797 assert(SrcOp.getValueSizeInBits() > BitWidth &&(static_cast <bool> (SrcOp.getValueSizeInBits() > BitWidth
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > BitWidth && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2798, __extension__ __PRETTY_FUNCTION__))
2798 "Expected BUILD_VECTOR implicit truncation")(static_cast <bool> (SrcOp.getValueSizeInBits() > BitWidth
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > BitWidth && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2798, __extension__ __PRETTY_FUNCTION__))
;
2799 Known2 = Known2.trunc(BitWidth);
2800 }
2801
2802 // Known bits are the values that are shared by every demanded element.
2803 Known = KnownBits::commonBits(Known, Known2);
2804
2805 // If we don't know any bits, early out.
2806 if (Known.isUnknown())
2807 break;
2808 }
2809 break;
2810 case ISD::VECTOR_SHUFFLE: {
2811 // Collect the known bits that are shared by every vector element referenced
2812 // by the shuffle.
2813 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2814 Known.Zero.setAllBits(); Known.One.setAllBits();
2815 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2816 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")(static_cast <bool> (NumElts == SVN->getMask().size(
) && "Unexpected vector size") ? void (0) : __assert_fail
("NumElts == SVN->getMask().size() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2816, __extension__ __PRETTY_FUNCTION__))
;
2817 for (unsigned i = 0; i != NumElts; ++i) {
2818 if (!DemandedElts[i])
2819 continue;
2820
2821 int M = SVN->getMaskElt(i);
2822 if (M < 0) {
2823 // For UNDEF elements, we don't know anything about the common state of
2824 // the shuffle result.
2825 Known.resetAll();
2826 DemandedLHS.clearAllBits();
2827 DemandedRHS.clearAllBits();
2828 break;
2829 }
2830
2831 if ((unsigned)M < NumElts)
2832 DemandedLHS.setBit((unsigned)M % NumElts);
2833 else
2834 DemandedRHS.setBit((unsigned)M % NumElts);
2835 }
2836 // Known bits are the values that are shared by every demanded element.
2837 if (!!DemandedLHS) {
2838 SDValue LHS = Op.getOperand(0);
2839 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2840 Known = KnownBits::commonBits(Known, Known2);
2841 }
2842 // If we don't know any bits, early out.
2843 if (Known.isUnknown())
2844 break;
2845 if (!!DemandedRHS) {
2846 SDValue RHS = Op.getOperand(1);
2847 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2848 Known = KnownBits::commonBits(Known, Known2);
2849 }
2850 break;
2851 }
2852 case ISD::CONCAT_VECTORS: {
2853 // Split DemandedElts and test each of the demanded subvectors.
2854 Known.Zero.setAllBits(); Known.One.setAllBits();
2855 EVT SubVectorVT = Op.getOperand(0).getValueType();
2856 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2857 unsigned NumSubVectors = Op.getNumOperands();
2858 for (unsigned i = 0; i != NumSubVectors; ++i) {
2859 APInt DemandedSub =
2860 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
2861 if (!!DemandedSub) {
2862 SDValue Sub = Op.getOperand(i);
2863 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2864 Known = KnownBits::commonBits(Known, Known2);
2865 }
2866 // If we don't know any bits, early out.
2867 if (Known.isUnknown())
2868 break;
2869 }
2870 break;
2871 }
2872 case ISD::INSERT_SUBVECTOR: {
2873 // Demand any elements from the subvector and the remainder from the src its
2874 // inserted into.
2875 SDValue Src = Op.getOperand(0);
2876 SDValue Sub = Op.getOperand(1);
2877 uint64_t Idx = Op.getConstantOperandVal(2);
2878 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2879 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2880 APInt DemandedSrcElts = DemandedElts;
2881 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2882
2883 Known.One.setAllBits();
2884 Known.Zero.setAllBits();
2885 if (!!DemandedSubElts) {
2886 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2887 if (Known.isUnknown())
2888 break; // early-out.
2889 }
2890 if (!!DemandedSrcElts) {
2891 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2892 Known = KnownBits::commonBits(Known, Known2);
2893 }
2894 break;
2895 }
2896 case ISD::EXTRACT_SUBVECTOR: {
2897 // Offset the demanded elts by the subvector index.
2898 SDValue Src = Op.getOperand(0);
2899 // Bail until we can represent demanded elements for scalable vectors.
2900 if (Src.getValueType().isScalableVector())
2901 break;
2902 uint64_t Idx = Op.getConstantOperandVal(1);
2903 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2904 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2905 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2906 break;
2907 }
2908 case ISD::SCALAR_TO_VECTOR: {
2909 // We know about scalar_to_vector as much as we know about it source,
2910 // which becomes the first element of otherwise unknown vector.
2911 if (DemandedElts != 1)
2912 break;
2913
2914 SDValue N0 = Op.getOperand(0);
2915 Known = computeKnownBits(N0, Depth + 1);
2916 if (N0.getValueSizeInBits() != BitWidth)
2917 Known = Known.trunc(BitWidth);
2918
2919 break;
2920 }
2921 case ISD::BITCAST: {
2922 SDValue N0 = Op.getOperand(0);
2923 EVT SubVT = N0.getValueType();
2924 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2925
2926 // Ignore bitcasts from unsupported types.
2927 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2928 break;
2929
2930 // Fast handling of 'identity' bitcasts.
2931 if (BitWidth == SubBitWidth) {
2932 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2933 break;
2934 }
2935
2936 bool IsLE = getDataLayout().isLittleEndian();
2937
2938 // Bitcast 'small element' vector to 'large element' scalar/vector.
2939 if ((BitWidth % SubBitWidth) == 0) {
2940 assert(N0.getValueType().isVector() && "Expected bitcast from vector")(static_cast <bool> (N0.getValueType().isVector() &&
"Expected bitcast from vector") ? void (0) : __assert_fail (
"N0.getValueType().isVector() && \"Expected bitcast from vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2940, __extension__ __PRETTY_FUNCTION__))
;
2941
2942 // Collect known bits for the (larger) output by collecting the known
2943 // bits from each set of sub elements and shift these into place.
2944 // We need to separately call computeKnownBits for each set of
2945 // sub elements as the knownbits for each is likely to be different.
2946 unsigned SubScale = BitWidth / SubBitWidth;
2947 APInt SubDemandedElts(NumElts * SubScale, 0);
2948 for (unsigned i = 0; i != NumElts; ++i)
2949 if (DemandedElts[i])
2950 SubDemandedElts.setBit(i * SubScale);
2951
2952 for (unsigned i = 0; i != SubScale; ++i) {
2953 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2954 Depth + 1);
2955 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2956 Known.insertBits(Known2, SubBitWidth * Shifts);
2957 }
2958 }
2959
2960 // Bitcast 'large element' scalar/vector to 'small element' vector.
2961 if ((SubBitWidth % BitWidth) == 0) {
2962 assert(Op.getValueType().isVector() && "Expected bitcast to vector")(static_cast <bool> (Op.getValueType().isVector() &&
"Expected bitcast to vector") ? void (0) : __assert_fail ("Op.getValueType().isVector() && \"Expected bitcast to vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 2962, __extension__ __PRETTY_FUNCTION__))
;
2963
2964 // Collect known bits for the (smaller) output by collecting the known
2965 // bits from the overlapping larger input elements and extracting the
2966 // sub sections we actually care about.
2967 unsigned SubScale = SubBitWidth / BitWidth;
2968 APInt SubDemandedElts(NumElts / SubScale, 0);
2969 for (unsigned i = 0; i != NumElts; ++i)
2970 if (DemandedElts[i])
2971 SubDemandedElts.setBit(i / SubScale);
2972
2973 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2974
2975 Known.Zero.setAllBits(); Known.One.setAllBits();
2976 for (unsigned i = 0; i != NumElts; ++i)
2977 if (DemandedElts[i]) {
2978 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
2979 unsigned Offset = (Shifts % SubScale) * BitWidth;
2980 Known = KnownBits::commonBits(Known,
2981 Known2.extractBits(BitWidth, Offset));
2982 // If we don't know any bits, early out.
2983 if (Known.isUnknown())
2984 break;
2985 }
2986 }
2987 break;
2988 }
2989 case ISD::AND:
2990 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2991 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2992
2993 Known &= Known2;
2994 break;
2995 case ISD::OR:
2996 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2997 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2998
2999 Known |= Known2;
3000 break;
3001 case ISD::XOR:
3002 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3003 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3004
3005 Known ^= Known2;
3006 break;
3007 case ISD::MUL: {
3008 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3009 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3010 Known = KnownBits::mul(Known, Known2);
3011 break;
3012 }
3013 case ISD::MULHU: {
3014 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3015 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3016 Known = KnownBits::mulhu(Known, Known2);
3017 break;
3018 }
3019 case ISD::MULHS: {
3020 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3021 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3022 Known = KnownBits::mulhs(Known, Known2);
3023 break;
3024 }
3025 case ISD::UMUL_LOHI: {
3026 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result")(static_cast <bool> ((Op.getResNo() == 0 || Op.getResNo
() == 1) && "Unknown result") ? void (0) : __assert_fail
("(Op.getResNo() == 0 || Op.getResNo() == 1) && \"Unknown result\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3026, __extension__ __PRETTY_FUNCTION__))
;
3027 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3028 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3029 if (Op.getResNo() == 0)
3030 Known = KnownBits::mul(Known, Known2);
3031 else
3032 Known = KnownBits::mulhu(Known, Known2);
3033 break;
3034 }
3035 case ISD::SMUL_LOHI: {
3036 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result")(static_cast <bool> ((Op.getResNo() == 0 || Op.getResNo
() == 1) && "Unknown result") ? void (0) : __assert_fail
("(Op.getResNo() == 0 || Op.getResNo() == 1) && \"Unknown result\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3036, __extension__ __PRETTY_FUNCTION__))
;
3037 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3038 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3039 if (Op.getResNo() == 0)
3040 Known = KnownBits::mul(Known, Known2);
3041 else
3042 Known = KnownBits::mulhs(Known, Known2);
3043 break;
3044 }
3045 case ISD::UDIV: {
3046 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3047 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3048 Known = KnownBits::udiv(Known, Known2);
3049 break;
3050 }
3051 case ISD::SELECT:
3052 case ISD::VSELECT:
3053 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3054 // If we don't know any bits, early out.
3055 if (Known.isUnknown())
3056 break;
3057 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3058
3059 // Only known if known in both the LHS and RHS.
3060 Known = KnownBits::commonBits(Known, Known2);
3061 break;
3062 case ISD::SELECT_CC:
3063 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3064 // If we don't know any bits, early out.
3065 if (Known.isUnknown())
3066 break;
3067 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3068
3069 // Only known if known in both the LHS and RHS.
3070 Known = KnownBits::commonBits(Known, Known2);
3071 break;
3072 case ISD::SMULO:
3073 case ISD::UMULO:
3074 if (Op.getResNo() != 1)
3075 break;
3076 // The boolean result conforms to getBooleanContents.
3077 // If we know the result of a setcc has the top bits zero, use this info.
3078 // We know that we have an integer-based boolean since these operations
3079 // are only available for integer.
3080 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3081 TargetLowering::ZeroOrOneBooleanContent &&
3082 BitWidth > 1)
3083 Known.Zero.setBitsFrom(1);
3084 break;
3085 case ISD::SETCC:
3086 case ISD::STRICT_FSETCC:
3087 case ISD::STRICT_FSETCCS: {
3088 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3089 // If we know the result of a setcc has the top bits zero, use this info.
3090 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3091 TargetLowering::ZeroOrOneBooleanContent &&
3092 BitWidth > 1)
3093 Known.Zero.setBitsFrom(1);
3094 break;
3095 }
3096 case ISD::SHL:
3097 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3098 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3099 Known = KnownBits::shl(Known, Known2);
3100
3101 // Minimum shift low bits are known zero.
3102 if (const APInt *ShMinAmt =
3103 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3104 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3105 break;
3106 case ISD::SRL:
3107 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3108 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3109 Known = KnownBits::lshr(Known, Known2);
3110
3111 // Minimum shift high bits are known zero.
3112 if (const APInt *ShMinAmt =
3113 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3114 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3115 break;
3116 case ISD::SRA:
3117 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3118 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3119 Known = KnownBits::ashr(Known, Known2);
3120 // TODO: Add minimum shift high known sign bits.
3121 break;
3122 case ISD::FSHL:
3123 case ISD::FSHR:
3124 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3125 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3126
3127 // For fshl, 0-shift returns the 1st arg.
3128 // For fshr, 0-shift returns the 2nd arg.
3129 if (Amt == 0) {
3130 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3131 DemandedElts, Depth + 1);
3132 break;
3133 }
3134
3135 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3136 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3137 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3138 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3139 if (Opcode == ISD::FSHL) {
3140 Known.One <<= Amt;
3141 Known.Zero <<= Amt;
3142 Known2.One.lshrInPlace(BitWidth - Amt);
3143 Known2.Zero.lshrInPlace(BitWidth - Amt);
3144 } else {
3145 Known.One <<= BitWidth - Amt;
3146 Known.Zero <<= BitWidth - Amt;
3147 Known2.One.lshrInPlace(Amt);
3148 Known2.Zero.lshrInPlace(Amt);
3149 }
3150 Known.One |= Known2.One;
3151 Known.Zero |= Known2.Zero;
3152 }
3153 break;
3154 case ISD::SIGN_EXTEND_INREG: {
3155 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3156 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3157 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3158 break;
3159 }
3160 case ISD::CTTZ:
3161 case ISD::CTTZ_ZERO_UNDEF: {
3162 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3163 // If we have a known 1, its position is our upper bound.
3164 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3165 unsigned LowBits = Log2_32(PossibleTZ) + 1;
3166 Known.Zero.setBitsFrom(LowBits);
3167 break;
3168 }
3169 case ISD::CTLZ:
3170 case ISD::CTLZ_ZERO_UNDEF: {
3171 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3172 // If we have a known 1, its position is our upper bound.
3173 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3174 unsigned LowBits = Log2_32(PossibleLZ) + 1;
3175 Known.Zero.setBitsFrom(LowBits);
3176 break;
3177 }
3178 case ISD::CTPOP: {
3179 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3180 // If we know some of the bits are zero, they can't be one.
3181 unsigned PossibleOnes = Known2.countMaxPopulation();
3182 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
3183 break;
3184 }
3185 case ISD::PARITY: {
3186 // Parity returns 0 everywhere but the LSB.
3187 Known.Zero.setBitsFrom(1);
3188 break;
3189 }
3190 case ISD::LOAD: {
3191 LoadSDNode *LD = cast<LoadSDNode>(Op);
3192 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3193 if (ISD::isNON_EXTLoad(LD) && Cst) {
3194 // Determine any common known bits from the loaded constant pool value.
3195 Type *CstTy = Cst->getType();
3196 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
3197 // If its a vector splat, then we can (quickly) reuse the scalar path.
3198 // NOTE: We assume all elements match and none are UNDEF.
3199 if (CstTy->isVectorTy()) {
3200 if (const Constant *Splat = Cst->getSplatValue()) {
3201 Cst = Splat;
3202 CstTy = Cst->getType();
3203 }
3204 }
3205 // TODO - do we need to handle different bitwidths?
3206 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3207 // Iterate across all vector elements finding common known bits.
3208 Known.One.setAllBits();
3209 Known.Zero.setAllBits();
3210 for (unsigned i = 0; i != NumElts; ++i) {
3211 if (!DemandedElts[i])
3212 continue;
3213 if (Constant *Elt = Cst->getAggregateElement(i)) {
3214 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3215 const APInt &Value = CInt->getValue();
3216 Known.One &= Value;
3217 Known.Zero &= ~Value;
3218 continue;
3219 }
3220 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3221 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3222 Known.One &= Value;
3223 Known.Zero &= ~Value;
3224 continue;
3225 }
3226 }
3227 Known.One.clearAllBits();
3228 Known.Zero.clearAllBits();
3229 break;
3230 }
3231 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3232 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3233 Known = KnownBits::makeConstant(CInt->getValue());
3234 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3235 Known =
3236 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3237 }
3238 }
3239 }
3240 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3241 // If this is a ZEXTLoad and we are looking at the loaded value.
3242 EVT VT = LD->getMemoryVT();
3243 unsigned MemBits = VT.getScalarSizeInBits();
3244 Known.Zero.setBitsFrom(MemBits);
3245 } else if (const MDNode *Ranges = LD->getRanges()) {
3246 if (LD->getExtensionType() == ISD::NON_EXTLOAD)
3247 computeKnownBitsFromRangeMetadata(*Ranges, Known);
3248 }
3249 break;
3250 }
3251 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3252 EVT InVT = Op.getOperand(0).getValueType();
3253 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3254 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3255 Known = Known.zext(BitWidth);
3256 break;
3257 }
3258 case ISD::ZERO_EXTEND: {
3259 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3260 Known = Known.zext(BitWidth);
3261 break;
3262 }
3263 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3264 EVT InVT = Op.getOperand(0).getValueType();
3265 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3266 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3267 // If the sign bit is known to be zero or one, then sext will extend
3268 // it to the top bits, else it will just zext.
3269 Known = Known.sext(BitWidth);
3270 break;
3271 }
3272 case ISD::SIGN_EXTEND: {
3273 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3274 // If the sign bit is known to be zero or one, then sext will extend
3275 // it to the top bits, else it will just zext.
3276 Known = Known.sext(BitWidth);
3277 break;
3278 }
3279 case ISD::ANY_EXTEND_VECTOR_INREG: {
3280 EVT InVT = Op.getOperand(0).getValueType();
3281 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3282 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3283 Known = Known.anyext(BitWidth);
3284 break;
3285 }
3286 case ISD::ANY_EXTEND: {
3287 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3288 Known = Known.anyext(BitWidth);
3289 break;
3290 }
3291 case ISD::TRUNCATE: {
3292 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3293 Known = Known.trunc(BitWidth);
3294 break;
3295 }
3296 case ISD::AssertZext: {
3297 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3298 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3299 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3300 Known.Zero |= (~InMask);
3301 Known.One &= (~Known.Zero);
3302 break;
3303 }
3304 case ISD::AssertAlign: {
3305 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3306 assert(LogOfAlign != 0)(static_cast <bool> (LogOfAlign != 0) ? void (0) : __assert_fail
("LogOfAlign != 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3306, __extension__ __PRETTY_FUNCTION__))
;
3307 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3308 // well as clearing one bits.
3309 Known.Zero.setLowBits(LogOfAlign);
3310 Known.One.clearLowBits(LogOfAlign);
3311 break;
3312 }
3313 case ISD::FGETSIGN:
3314 // All bits are zero except the low bit.
3315 Known.Zero.setBitsFrom(1);
3316 break;
3317 case ISD::USUBO:
3318 case ISD::SSUBO:
3319 if (Op.getResNo() == 1) {
3320 // If we know the result of a setcc has the top bits zero, use this info.
3321 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3322 TargetLowering::ZeroOrOneBooleanContent &&
3323 BitWidth > 1)
3324 Known.Zero.setBitsFrom(1);
3325 break;
3326 }
3327 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3328 case ISD::SUB:
3329 case ISD::SUBC: {
3330 assert(Op.getResNo() == 0 &&(static_cast <bool> (Op.getResNo() == 0 && "We only compute knownbits for the difference here."
) ? void (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the difference here.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3331, __extension__ __PRETTY_FUNCTION__))
3331 "We only compute knownbits for the difference here.")(static_cast <bool> (Op.getResNo() == 0 && "We only compute knownbits for the difference here."
) ? void (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the difference here.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3331, __extension__ __PRETTY_FUNCTION__))
;
3332
3333 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3334 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3335 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3336 Known, Known2);
3337 break;
3338 }
3339 case ISD::UADDO:
3340 case ISD::SADDO:
3341 case ISD::ADDCARRY:
3342 if (Op.getResNo() == 1) {
3343 // If we know the result of a setcc has the top bits zero, use this info.
3344 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3345 TargetLowering::ZeroOrOneBooleanContent &&
3346 BitWidth > 1)
3347 Known.Zero.setBitsFrom(1);
3348 break;
3349 }
3350 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3351 case ISD::ADD:
3352 case ISD::ADDC:
3353 case ISD::ADDE: {
3354 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.")(static_cast <bool> (Op.getResNo() == 0 && "We only compute knownbits for the sum here."
) ? void (0) : __assert_fail ("Op.getResNo() == 0 && \"We only compute knownbits for the sum here.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3354, __extension__ __PRETTY_FUNCTION__))
;
3355
3356 // With ADDE and ADDCARRY, a carry bit may be added in.
3357 KnownBits Carry(1);
3358 if (Opcode == ISD::ADDE)
3359 // Can't track carry from glue, set carry to unknown.
3360 Carry.resetAll();
3361 else if (Opcode == ISD::ADDCARRY)
3362 // TODO: Compute known bits for the carry operand. Not sure if it is worth
3363 // the trouble (how often will we find a known carry bit). And I haven't
3364 // tested this very much yet, but something like this might work:
3365 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3366 // Carry = Carry.zextOrTrunc(1, false);
3367 Carry.resetAll();
3368 else
3369 Carry.setAllZero();
3370
3371 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3372 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3373 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3374 break;
3375 }
3376 case ISD::SREM: {
3377 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3378 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3379 Known = KnownBits::srem(Known, Known2);
3380 break;
3381 }
3382 case ISD::UREM: {
3383 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3384 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3385 Known = KnownBits::urem(Known, Known2);
3386 break;
3387 }
3388 case ISD::EXTRACT_ELEMENT: {
3389 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3390 const unsigned Index = Op.getConstantOperandVal(1);
3391 const unsigned EltBitWidth = Op.getValueSizeInBits();
3392
3393 // Remove low part of known bits mask
3394 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3395 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3396
3397 // Remove high part of known bit mask
3398 Known = Known.trunc(EltBitWidth);
3399 break;
3400 }
3401 case ISD::EXTRACT_VECTOR_ELT: {
3402 SDValue InVec = Op.getOperand(0);
3403 SDValue EltNo = Op.getOperand(1);
3404 EVT VecVT = InVec.getValueType();
3405 // computeKnownBits not yet implemented for scalable vectors.
3406 if (VecVT.isScalableVector())
3407 break;
3408 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3409 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3410
3411 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3412 // anything about the extended bits.
3413 if (BitWidth > EltBitWidth)
3414 Known = Known.trunc(EltBitWidth);
3415
3416 // If we know the element index, just demand that vector element, else for
3417 // an unknown element index, ignore DemandedElts and demand them all.
3418 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3419 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3420 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3421 DemandedSrcElts =
3422 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3423
3424 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3425 if (BitWidth > EltBitWidth)
3426 Known = Known.anyext(BitWidth);
3427 break;
3428 }
3429 case ISD::INSERT_VECTOR_ELT: {
3430 // If we know the element index, split the demand between the
3431 // source vector and the inserted element, otherwise assume we need
3432 // the original demanded vector elements and the value.
3433 SDValue InVec = Op.getOperand(0);
3434 SDValue InVal = Op.getOperand(1);
3435 SDValue EltNo = Op.getOperand(2);
3436 bool DemandedVal = true;
3437 APInt DemandedVecElts = DemandedElts;
3438 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3439 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3440 unsigned EltIdx = CEltNo->getZExtValue();
3441 DemandedVal = !!DemandedElts[EltIdx];
3442 DemandedVecElts.clearBit(EltIdx);
3443 }
3444 Known.One.setAllBits();
3445 Known.Zero.setAllBits();
3446 if (DemandedVal) {
3447 Known2 = computeKnownBits(InVal, Depth + 1);
3448 Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth));
3449 }
3450 if (!!DemandedVecElts) {
3451 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3452 Known = KnownBits::commonBits(Known, Known2);
3453 }
3454 break;
3455 }
3456 case ISD::BITREVERSE: {
3457 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3458 Known = Known2.reverseBits();
3459 break;
3460 }
3461 case ISD::BSWAP: {
3462 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3463 Known = Known2.byteSwap();
3464 break;
3465 }
3466 case ISD::ABS: {
3467 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3468 Known = Known2.abs();
3469 break;
3470 }
3471 case ISD::USUBSAT: {
3472 // The result of usubsat will never be larger than the LHS.
3473 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3474 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
3475 break;
3476 }
3477 case ISD::UMIN: {
3478 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3479 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3480 Known = KnownBits::umin(Known, Known2);
3481 break;
3482 }
3483 case ISD::UMAX: {
3484 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3485 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3486 Known = KnownBits::umax(Known, Known2);
3487 break;
3488 }
3489 case ISD::SMIN:
3490 case ISD::SMAX: {
3491 // If we have a clamp pattern, we know that the number of sign bits will be
3492 // the minimum of the clamp min/max range.
3493 bool IsMax = (Opcode == ISD::SMAX);
3494 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3495 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3496 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3497 CstHigh =
3498 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3499 if (CstLow && CstHigh) {
3500 if (!IsMax)
3501 std::swap(CstLow, CstHigh);
3502
3503 const APInt &ValueLow = CstLow->getAPIntValue();
3504 const APInt &ValueHigh = CstHigh->getAPIntValue();
3505 if (ValueLow.sle(ValueHigh)) {
3506 unsigned LowSignBits = ValueLow.getNumSignBits();
3507 unsigned HighSignBits = ValueHigh.getNumSignBits();
3508 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3509 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3510 Known.One.setHighBits(MinSignBits);
3511 break;
3512 }
3513 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3514 Known.Zero.setHighBits(MinSignBits);
3515 break;
3516 }
3517 }
3518 }
3519
3520 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3521 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3522 if (IsMax)
3523 Known = KnownBits::smax(Known, Known2);
3524 else
3525 Known = KnownBits::smin(Known, Known2);
3526 break;
3527 }
3528 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
3529 if (Op.getResNo() == 1) {
3530 // The boolean result conforms to getBooleanContents.
3531 // If we know the result of a setcc has the top bits zero, use this info.
3532 // We know that we have an integer-based boolean since these operations
3533 // are only available for integer.
3534 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3535 TargetLowering::ZeroOrOneBooleanContent &&
3536 BitWidth > 1)
3537 Known.Zero.setBitsFrom(1);
3538 break;
3539 }
3540 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3541 case ISD::ATOMIC_CMP_SWAP:
3542 case ISD::ATOMIC_SWAP:
3543 case ISD::ATOMIC_LOAD_ADD:
3544 case ISD::ATOMIC_LOAD_SUB:
3545 case ISD::ATOMIC_LOAD_AND:
3546 case ISD::ATOMIC_LOAD_CLR:
3547 case ISD::ATOMIC_LOAD_OR:
3548 case ISD::ATOMIC_LOAD_XOR:
3549 case ISD::ATOMIC_LOAD_NAND:
3550 case ISD::ATOMIC_LOAD_MIN:
3551 case ISD::ATOMIC_LOAD_MAX:
3552 case ISD::ATOMIC_LOAD_UMIN:
3553 case ISD::ATOMIC_LOAD_UMAX:
3554 case ISD::ATOMIC_LOAD: {
3555 unsigned MemBits =
3556 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
3557 // If we are looking at the loaded value.
3558 if (Op.getResNo() == 0) {
3559 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
3560 Known.Zero.setBitsFrom(MemBits);
3561 }
3562 break;
3563 }
3564 case ISD::FrameIndex:
3565 case ISD::TargetFrameIndex:
3566 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
3567 Known, getMachineFunction());
3568 break;
3569
3570 default:
3571 if (Opcode < ISD::BUILTIN_OP_END)
3572 break;
3573 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3574 case ISD::INTRINSIC_WO_CHAIN:
3575 case ISD::INTRINSIC_W_CHAIN:
3576 case ISD::INTRINSIC_VOID:
3577 // Allow the target to implement this method for its nodes.
3578 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3579 break;
3580 }
3581
3582 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3582, __extension__ __PRETTY_FUNCTION__))
;
3583 return Known;
3584}
3585
3586SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3587 SDValue N1) const {
3588 // X + 0 never overflow
3589 if (isNullConstant(N1))
3590 return OFK_Never;
3591
3592 KnownBits N1Known = computeKnownBits(N1);
3593 if (N1Known.Zero.getBoolValue()) {
3594 KnownBits N0Known = computeKnownBits(N0);
3595
3596 bool overflow;
3597 (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow);
3598 if (!overflow)
3599 return OFK_Never;
3600 }
3601
3602 // mulhi + 1 never overflow
3603 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3604 (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue())
3605 return OFK_Never;
3606
3607 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3608 KnownBits N0Known = computeKnownBits(N0);
3609
3610 if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue())
3611 return OFK_Never;
3612 }
3613
3614 return OFK_Sometime;
3615}
3616
3617bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3618 EVT OpVT = Val.getValueType();
3619 unsigned BitWidth = OpVT.getScalarSizeInBits();
3620
3621 // Is the constant a known power of 2?
3622 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3623 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3624
3625 // A left-shift of a constant one will have exactly one bit set because
3626 // shifting the bit off the end is undefined.
3627 if (Val.getOpcode() == ISD::SHL) {
3628 auto *C = isConstOrConstSplat(Val.getOperand(0));
3629 if (C && C->getAPIntValue() == 1)
3630 return true;
3631 }
3632
3633 // Similarly, a logical right-shift of a constant sign-bit will have exactly
3634 // one bit set.
3635 if (Val.getOpcode() == ISD::SRL) {
3636 auto *C = isConstOrConstSplat(Val.getOperand(0));
3637 if (C && C->getAPIntValue().isSignMask())
3638 return true;
3639 }
3640
3641 // Are all operands of a build vector constant powers of two?
3642 if (Val.getOpcode() == ISD::BUILD_VECTOR)
3643 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3644 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3645 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3646 return false;
3647 }))
3648 return true;
3649
3650 // Is the operand of a splat vector a constant power of two?
3651 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
3652 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
3653 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
3654 return true;
3655
3656 // More could be done here, though the above checks are enough
3657 // to handle some common cases.
3658
3659 // Fall back to computeKnownBits to catch other known cases.
3660 KnownBits Known = computeKnownBits(Val);
3661 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3662}
3663
3664unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3665 EVT VT = Op.getValueType();
3666
3667 // TODO: Assume we don't know anything for now.
3668 if (VT.isScalableVector())
3669 return 1;
3670
3671 APInt DemandedElts = VT.isVector()
3672 ? APInt::getAllOnesValue(VT.getVectorNumElements())
3673 : APInt(1, 1);
3674 return ComputeNumSignBits(Op, DemandedElts, Depth);
3675}
3676
3677unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3678 unsigned Depth) const {
3679 EVT VT = Op.getValueType();
3680 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!")(static_cast <bool> ((VT.isInteger() || VT.isFloatingPoint
()) && "Invalid VT!") ? void (0) : __assert_fail ("(VT.isInteger() || VT.isFloatingPoint()) && \"Invalid VT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3680, __extension__ __PRETTY_FUNCTION__))
;
3681 unsigned VTBits = VT.getScalarSizeInBits();
3682 unsigned NumElts = DemandedElts.getBitWidth();
3683 unsigned Tmp, Tmp2;
3684 unsigned FirstAnswer = 1;
3685
3686 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3687 const APInt &Val = C->getAPIntValue();
3688 return Val.getNumSignBits();
3689 }
3690
3691 if (Depth >= MaxRecursionDepth)
3692 return 1; // Limit search depth.
3693
3694 if (!DemandedElts || VT.isScalableVector())
3695 return 1; // No demanded elts, better to assume we don't know anything.
3696
3697 unsigned Opcode = Op.getOpcode();
3698 switch (Opcode) {
3699 default: break;
3700 case ISD::AssertSext:
3701 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3702 return VTBits-Tmp+1;
3703 case ISD::AssertZext:
3704 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3705 return VTBits-Tmp;
3706
3707 case ISD::BUILD_VECTOR:
3708 Tmp = VTBits;
3709 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3710 if (!DemandedElts[i])
3711 continue;
3712
3713 SDValue SrcOp = Op.getOperand(i);
3714 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
3715
3716 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3717 if (SrcOp.getValueSizeInBits() != VTBits) {
3718 assert(SrcOp.getValueSizeInBits() > VTBits &&(static_cast <bool> (SrcOp.getValueSizeInBits() > VTBits
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > VTBits && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3719, __extension__ __PRETTY_FUNCTION__))
3719 "Expected BUILD_VECTOR implicit truncation")(static_cast <bool> (SrcOp.getValueSizeInBits() > VTBits
&& "Expected BUILD_VECTOR implicit truncation") ? void
(0) : __assert_fail ("SrcOp.getValueSizeInBits() > VTBits && \"Expected BUILD_VECTOR implicit truncation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3719, __extension__ __PRETTY_FUNCTION__))
;
3720 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3721 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3722 }
3723 Tmp = std::min(Tmp, Tmp2);
3724 }
3725 return Tmp;
3726
3727 case ISD::VECTOR_SHUFFLE: {
3728 // Collect the minimum number of sign bits that are shared by every vector
3729 // element referenced by the shuffle.
3730 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3731 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3732 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")(static_cast <bool> (NumElts == SVN->getMask().size(
) && "Unexpected vector size") ? void (0) : __assert_fail
("NumElts == SVN->getMask().size() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3732, __extension__ __PRETTY_FUNCTION__))
;
3733 for (unsigned i = 0; i != NumElts; ++i) {
3734 int M = SVN->getMaskElt(i);
3735 if (!DemandedElts[i])
3736 continue;
3737 // For UNDEF elements, we don't know anything about the common state of
3738 // the shuffle result.
3739 if (M < 0)
3740 return 1;
3741 if ((unsigned)M < NumElts)
3742 DemandedLHS.setBit((unsigned)M % NumElts);
3743 else
3744 DemandedRHS.setBit((unsigned)M % NumElts);
3745 }
3746 Tmp = std::numeric_limits<unsigned>::max();
3747 if (!!DemandedLHS)
3748 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3749 if (!!DemandedRHS) {
3750 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3751 Tmp = std::min(Tmp, Tmp2);
3752 }
3753 // If we don't know anything, early out and try computeKnownBits fall-back.
3754 if (Tmp == 1)
3755 break;
3756 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3756, __extension__ __PRETTY_FUNCTION__))
;
3757 return Tmp;
3758 }
3759
3760 case ISD::BITCAST: {
3761 SDValue N0 = Op.getOperand(0);
3762 EVT SrcVT = N0.getValueType();
3763 unsigned SrcBits = SrcVT.getScalarSizeInBits();
3764
3765 // Ignore bitcasts from unsupported types..
3766 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3767 break;
3768
3769 // Fast handling of 'identity' bitcasts.
3770 if (VTBits == SrcBits)
3771 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3772
3773 bool IsLE = getDataLayout().isLittleEndian();
3774
3775 // Bitcast 'large element' scalar/vector to 'small element' vector.
3776 if ((SrcBits % VTBits) == 0) {
3777 assert(VT.isVector() && "Expected bitcast to vector")(static_cast <bool> (VT.isVector() && "Expected bitcast to vector"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected bitcast to vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 3777, __extension__ __PRETTY_FUNCTION__))
;
3778
3779 unsigned Scale = SrcBits / VTBits;
3780 APInt SrcDemandedElts(NumElts / Scale, 0);
3781 for (unsigned i = 0; i != NumElts; ++i)
3782 if (DemandedElts[i])
3783 SrcDemandedElts.setBit(i / Scale);
3784
3785 // Fast case - sign splat can be simply split across the small elements.
3786 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3787 if (Tmp == SrcBits)
3788 return VTBits;
3789
3790 // Slow case - determine how far the sign extends into each sub-element.
3791 Tmp2 = VTBits;
3792 for (unsigned i = 0; i != NumElts; ++i)
3793 if (DemandedElts[i]) {
3794 unsigned SubOffset = i % Scale;
3795 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3796 SubOffset = SubOffset * VTBits;
3797 if (Tmp <= SubOffset)
3798 return 1;
3799 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3800 }
3801 return Tmp2;
3802 }
3803 break;
3804 }
3805
3806 case ISD::SIGN_EXTEND:
3807 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3808 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3809 case ISD::SIGN_EXTEND_INREG:
3810 // Max of the input and what this extends.
3811 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3812 Tmp = VTBits-Tmp+1;
3813 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3814 return std::max(Tmp, Tmp2);
3815 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3816 SDValue Src = Op.getOperand(0);
3817 EVT SrcVT = Src.getValueType();
3818 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3819 Tmp = VTBits - SrcVT.getScalarSizeInBits();
3820 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3821 }
3822 case ISD::SRA:
3823 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3824 // SRA X, C -> adds C sign bits.
3825 if (const APInt *ShAmt =
3826 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3827 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
3828 return Tmp;
3829 case ISD::SHL:
3830 if (const APInt *ShAmt =
3831 getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
3832 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
3833 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3834 if (ShAmt->ult(Tmp))
3835 return Tmp - ShAmt->getZExtValue();
3836 }
3837 break;
3838 case ISD::AND:
3839 case ISD::OR:
3840 case ISD::XOR: // NOT is handled here.
3841 // Logical binary ops preserve the number of sign bits at the worst.
3842 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3843 if (Tmp != 1) {
3844 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3845 FirstAnswer = std::min(Tmp, Tmp2);
3846 // We computed what we know about the sign bits as our first
3847 // answer. Now proceed to the generic code that uses
3848 // computeKnownBits, and pick whichever answer is better.
3849 }
3850 break;
3851
3852 case ISD::SELECT:
3853 case ISD::VSELECT:
3854 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3855 if (Tmp == 1) return 1; // Early out.
3856 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3857 return std::min(Tmp, Tmp2);
3858 case ISD::SELECT_CC:
3859 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3860 if (Tmp == 1) return 1; // Early out.
3861 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3862 return std::min(Tmp, Tmp2);
3863
3864 case ISD::SMIN:
3865 case ISD::SMAX: {
3866 // If we have a clamp pattern, we know that the number of sign bits will be
3867 // the minimum of the clamp min/max range.
3868 bool IsMax = (Opcode == ISD::SMAX);
3869 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3870 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3871 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3872 CstHigh =
3873 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3874 if (CstLow && CstHigh) {
3875 if (!IsMax)
3876 std::swap(CstLow, CstHigh);
3877 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3878 Tmp = CstLow->getAPIntValue().getNumSignBits();
3879 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3880 return std::min(Tmp, Tmp2);
3881 }
3882 }
3883
3884 // Fallback - just get the minimum number of sign bits of the operands.
3885 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3886 if (Tmp == 1)
3887 return 1; // Early out.
3888 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3889 return std::min(Tmp, Tmp2);
3890 }
3891 case ISD::UMIN:
3892 case ISD::UMAX:
3893 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3894 if (Tmp == 1)
3895 return 1; // Early out.
3896 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3897 return std::min(Tmp, Tmp2);
3898 case ISD::SADDO:
3899 case ISD::UADDO:
3900 case ISD::SSUBO:
3901 case ISD::USUBO:
3902 case ISD::SMULO:
3903 case ISD::UMULO:
3904 if (Op.getResNo() != 1)
3905 break;
3906 // The boolean result conforms to getBooleanContents. Fall through.
3907 // If setcc returns 0/-1, all bits are sign bits.
3908 // We know that we have an integer-based boolean since these operations
3909 // are only available for integer.
3910 if (TLI->getBooleanContents(VT.isVector(), false) ==
3911 TargetLowering::ZeroOrNegativeOneBooleanContent)
3912 return VTBits;
3913 break;
3914 case ISD::SETCC:
3915 case ISD::STRICT_FSETCC:
3916 case ISD::STRICT_FSETCCS: {
3917 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3918 // If setcc returns 0/-1, all bits are sign bits.
3919 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3920 TargetLowering::ZeroOrNegativeOneBooleanContent)
3921 return VTBits;
3922 break;
3923 }
3924 case ISD::ROTL:
3925 case ISD::ROTR:
3926 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3927
3928 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
3929 if (Tmp == VTBits)
3930 return VTBits;
3931
3932 if (ConstantSDNode *C =
3933 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3934 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3935
3936 // Handle rotate right by N like a rotate left by 32-N.
3937 if (Opcode == ISD::ROTR)
3938 RotAmt = (VTBits - RotAmt) % VTBits;
3939
3940 // If we aren't rotating out all of the known-in sign bits, return the
3941 // number that are left. This handles rotl(sext(x), 1) for example.
3942 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3943 }
3944 break;
3945 case ISD::ADD:
3946 case ISD::ADDC:
3947 // Add can have at most one carry bit. Thus we know that the output
3948 // is, at worst, one more bit than the inputs.
3949 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3950 if (Tmp == 1) return 1; // Early out.
3951
3952 // Special case decrementing a value (ADD X, -1):
3953 if (ConstantSDNode *CRHS =
3954 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
3955 if (CRHS->isAllOnesValue()) {
3956 KnownBits Known =
3957 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3958
3959 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3960 // sign bits set.
3961 if ((Known.Zero | 1).isAllOnesValue())
3962 return VTBits;
3963
3964 // If we are subtracting one from a positive number, there is no carry
3965 // out of the result.
3966 if (Known.isNonNegative())
3967 return Tmp;
3968 }
3969
3970 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3971 if (Tmp2 == 1) return 1; // Early out.
3972 return std::min(Tmp, Tmp2) - 1;
3973 case ISD::SUB:
3974 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3975 if (Tmp2 == 1) return 1; // Early out.
3976
3977 // Handle NEG.
3978 if (ConstantSDNode *CLHS =
3979 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
3980 if (CLHS->isNullValue()) {
3981 KnownBits Known =
3982 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3983 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3984 // sign bits set.
3985 if ((Known.Zero | 1).isAllOnesValue())
3986 return VTBits;
3987
3988 // If the input is known to be positive (the sign bit is known clear),
3989 // the output of the NEG has the same number of sign bits as the input.
3990 if (Known.isNonNegative())
3991 return Tmp2;
3992
3993 // Otherwise, we treat this like a SUB.
3994 }
3995
3996 // Sub can have at most one carry bit. Thus we know that the output
3997 // is, at worst, one more bit than the inputs.
3998 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3999 if (Tmp == 1) return 1; // Early out.
4000 return std::min(Tmp, Tmp2) - 1;
4001 case ISD::MUL: {
4002 // The output of the Mul can be at most twice the valid bits in the inputs.
4003 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4004 if (SignBitsOp0 == 1)
4005 break;
4006 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4007 if (SignBitsOp1 == 1)
4008 break;
4009 unsigned OutValidBits =
4010 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4011 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4012 }
4013 case ISD::SREM:
4014 // The sign bit is the LHS's sign bit, except when the result of the
4015 // remainder is zero. The magnitude of the result should be less than or
4016 // equal to the magnitude of the LHS. Therefore, the result should have
4017 // at least as many sign bits as the left hand side.
4018 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4019 case ISD::TRUNCATE: {
4020 // Check if the sign bits of source go down as far as the truncated value.
4021 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4022 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4023 if (NumSrcSignBits > (NumSrcBits - VTBits))
4024 return NumSrcSignBits - (NumSrcBits - VTBits);
4025 break;
4026 }
4027 case ISD::EXTRACT_ELEMENT: {
4028 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4029 const int BitWidth = Op.getValueSizeInBits();
4030 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4031
4032 // Get reverse index (starting from 1), Op1 value indexes elements from
4033 // little end. Sign starts at big end.
4034 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4035
4036 // If the sign portion ends in our element the subtraction gives correct
4037 // result. Otherwise it gives either negative or > bitwidth result
4038 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
4039 }
4040 case ISD::INSERT_VECTOR_ELT: {
4041 // If we know the element index, split the demand between the
4042 // source vector and the inserted element, otherwise assume we need
4043 // the original demanded vector elements and the value.
4044 SDValue InVec = Op.getOperand(0);
4045 SDValue InVal = Op.getOperand(1);
4046 SDValue EltNo = Op.getOperand(2);
4047 bool DemandedVal = true;
4048 APInt DemandedVecElts = DemandedElts;
4049 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4050 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4051 unsigned EltIdx = CEltNo->getZExtValue();
4052 DemandedVal = !!DemandedElts[EltIdx];
4053 DemandedVecElts.clearBit(EltIdx);
4054 }
4055 Tmp = std::numeric_limits<unsigned>::max();
4056 if (DemandedVal) {
4057 // TODO - handle implicit truncation of inserted elements.
4058 if (InVal.getScalarValueSizeInBits() != VTBits)
4059 break;
4060 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4061 Tmp = std::min(Tmp, Tmp2);
4062 }
4063 if (!!DemandedVecElts) {
4064 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4065 Tmp = std::min(Tmp, Tmp2);
4066 }
4067 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4067, __extension__ __PRETTY_FUNCTION__))
;
4068 return Tmp;
4069 }
4070 case ISD::EXTRACT_VECTOR_ELT: {
4071 SDValue InVec = Op.getOperand(0);
4072 SDValue EltNo = Op.getOperand(1);
4073 EVT VecVT = InVec.getValueType();
4074 // ComputeNumSignBits not yet implemented for scalable vectors.
4075 if (VecVT.isScalableVector())
4076 break;
4077 const unsigned BitWidth = Op.getValueSizeInBits();
4078 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4079 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4080
4081 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4082 // anything about sign bits. But if the sizes match we can derive knowledge
4083 // about sign bits from the vector operand.
4084 if (BitWidth != EltBitWidth)
4085 break;
4086
4087 // If we know the element index, just demand that vector element, else for
4088 // an unknown element index, ignore DemandedElts and demand them all.
4089 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
4090 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4091 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4092 DemandedSrcElts =
4093 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4094
4095 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4096 }
4097 case ISD::EXTRACT_SUBVECTOR: {
4098 // Offset the demanded elts by the subvector index.
4099 SDValue Src = Op.getOperand(0);
4100 // Bail until we can represent demanded elements for scalable vectors.
4101 if (Src.getValueType().isScalableVector())
4102 break;
4103 uint64_t Idx = Op.getConstantOperandVal(1);
4104 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4105 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
4106 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4107 }
4108 case ISD::CONCAT_VECTORS: {
4109 // Determine the minimum number of sign bits across all demanded
4110 // elts of the input vectors. Early out if the result is already 1.
4111 Tmp = std::numeric_limits<unsigned>::max();
4112 EVT SubVectorVT = Op.getOperand(0).getValueType();
4113 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4114 unsigned NumSubVectors = Op.getNumOperands();
4115 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4116 APInt DemandedSub =
4117 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4118 if (!DemandedSub)
4119 continue;
4120 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4121 Tmp = std::min(Tmp, Tmp2);
4122 }
4123 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4123, __extension__ __PRETTY_FUNCTION__))
;
4124 return Tmp;
4125 }
4126 case ISD::INSERT_SUBVECTOR: {
4127 // Demand any elements from the subvector and the remainder from the src its
4128 // inserted into.
4129 SDValue Src = Op.getOperand(0);
4130 SDValue Sub = Op.getOperand(1);
4131 uint64_t Idx = Op.getConstantOperandVal(2);
4132 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4133 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4134 APInt DemandedSrcElts = DemandedElts;
4135 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
4136
4137 Tmp = std::numeric_limits<unsigned>::max();
4138 if (!!DemandedSubElts) {
4139 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4140 if (Tmp == 1)
4141 return 1; // early-out
4142 }
4143 if (!!DemandedSrcElts) {
4144 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4145 Tmp = std::min(Tmp, Tmp2);
4146 }
4147 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")(static_cast <bool> (Tmp <= VTBits && "Failed to determine minimum sign bits"
) ? void (0) : __assert_fail ("Tmp <= VTBits && \"Failed to determine minimum sign bits\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4147, __extension__ __PRETTY_FUNCTION__))
;
4148 return Tmp;
4149 }
4150 case ISD::ATOMIC_CMP_SWAP:
4151 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4152 case ISD::ATOMIC_SWAP:
4153 case ISD::ATOMIC_LOAD_ADD:
4154 case ISD::ATOMIC_LOAD_SUB:
4155 case ISD::ATOMIC_LOAD_AND:
4156 case ISD::ATOMIC_LOAD_CLR:
4157 case ISD::ATOMIC_LOAD_OR:
4158 case ISD::ATOMIC_LOAD_XOR:
4159 case ISD::ATOMIC_LOAD_NAND:
4160 case ISD::ATOMIC_LOAD_MIN:
4161 case ISD::ATOMIC_LOAD_MAX:
4162 case ISD::ATOMIC_LOAD_UMIN:
4163 case ISD::ATOMIC_LOAD_UMAX:
4164 case ISD::ATOMIC_LOAD: {
4165 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4166 // If we are looking at the loaded value.
4167 if (Op.getResNo() == 0) {
4168 if (Tmp == VTBits)
4169 return 1; // early-out
4170 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4171 return VTBits - Tmp + 1;
4172 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4173 return VTBits - Tmp;
4174 }
4175 break;
4176 }
4177 }
4178
4179 // If we are looking at the loaded value of the SDNode.
4180 if (Op.getResNo() == 0) {
4181 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4182 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4183 unsigned ExtType = LD->getExtensionType();
4184 switch (ExtType) {
4185 default: break;
4186 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4187 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4188 return VTBits - Tmp + 1;
4189 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4190 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4191 return VTBits - Tmp;
4192 case ISD::NON_EXTLOAD:
4193 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4194 // We only need to handle vectors - computeKnownBits should handle
4195 // scalar cases.
4196 Type *CstTy = Cst->getType();
4197 if (CstTy->isVectorTy() &&
4198 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
4199 Tmp = VTBits;
4200 for (unsigned i = 0; i != NumElts; ++i) {
4201 if (!DemandedElts[i])
4202 continue;
4203 if (Constant *Elt = Cst->getAggregateElement(i)) {
4204 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4205 const APInt &Value = CInt->getValue();
4206 Tmp = std::min(Tmp, Value.getNumSignBits());
4207 continue;
4208 }
4209 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4210 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4211 Tmp = std::min(Tmp, Value.getNumSignBits());
4212 continue;
4213 }
4214 }
4215 // Unknown type. Conservatively assume no bits match sign bit.
4216 return 1;
4217 }
4218 return Tmp;
4219 }
4220 }
4221 break;
4222 }
4223 }
4224 }
4225
4226 // Allow the target to implement this method for its nodes.
4227 if (Opcode >= ISD::BUILTIN_OP_END ||
4228 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4229 Opcode == ISD::INTRINSIC_W_CHAIN ||
4230 Opcode == ISD::INTRINSIC_VOID) {
4231 unsigned NumBits =
4232 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4233 if (NumBits > 1)
4234 FirstAnswer = std::max(FirstAnswer, NumBits);
4235 }
4236
4237 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4238 // use this information.
4239 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4240
4241 APInt Mask;
4242 if (Known.isNonNegative()) { // sign bit is 0
4243 Mask = Known.Zero;
4244 } else if (Known.isNegative()) { // sign bit is 1;
4245 Mask = Known.One;
4246 } else {
4247 // Nothing known.
4248 return FirstAnswer;
4249 }
4250
4251 // Okay, we know that the sign bit in Mask is set. Use CLO to determine
4252 // the number of identical bits in the top of the input value.
4253 Mask <<= Mask.getBitWidth()-VTBits;
4254 return std::max(FirstAnswer, Mask.countLeadingOnes());
4255}
4256
4257bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
4258 unsigned Depth) const {
4259 // Early out for FREEZE.
4260 if (Op.getOpcode() == ISD::FREEZE)
4261 return true;
4262
4263 // TODO: Assume we don't know anything for now.
4264 EVT VT = Op.getValueType();
4265 if (VT.isScalableVector())
4266 return false;
4267
4268 APInt DemandedElts = VT.isVector()
4269 ? APInt::getAllOnesValue(VT.getVectorNumElements())
4270 : APInt(1, 1);
4271 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4272}
4273
4274bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
4275 const APInt &DemandedElts,
4276 bool PoisonOnly,
4277 unsigned Depth) const {
4278 unsigned Opcode = Op.getOpcode();
4279
4280 // Early out for FREEZE.
4281 if (Opcode == ISD::FREEZE)
4282 return true;
4283
4284 if (Depth >= MaxRecursionDepth)
4285 return false; // Limit search depth.
4286
4287 if (isIntOrFPConstant(Op))
4288 return true;
4289
4290 switch (Opcode) {
4291 case ISD::UNDEF:
4292 return PoisonOnly;
4293
4294 case ISD::BUILD_VECTOR:
4295 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
4296 // this shouldn't affect the result.
4297 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
4298 if (!DemandedElts[i])
4299 continue;
4300 if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly,
4301 Depth + 1))
4302 return false;
4303 }
4304 return true;
4305
4306 // TODO: Search for noundef attributes from library functions.
4307
4308 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
4309
4310 default:
4311 // Allow the target to implement this method for its nodes.
4312 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
4313 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
4314 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4315 Op, DemandedElts, *this, PoisonOnly, Depth);
4316 break;
4317 }
4318
4319 return false;
4320}
4321
4322bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
4323 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
4324 !isa<ConstantSDNode>(Op.getOperand(1)))
4325 return false;
4326
4327 if (Op.getOpcode() == ISD::OR &&
4328 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
4329 return false;
4330
4331 return true;
4332}
4333
4334bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
4335 // If we're told that NaNs won't happen, assume they won't.
4336 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
4337 return true;
4338
4339 if (Depth >= MaxRecursionDepth)
4340 return false; // Limit search depth.
4341
4342 // TODO: Handle vectors.
4343 // If the value is a constant, we can obviously see if it is a NaN or not.
4344 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
4345 return !C->getValueAPF().isNaN() ||
4346 (SNaN && !C->getValueAPF().isSignaling());
4347 }
4348
4349 unsigned Opcode = Op.getOpcode();
4350 switch (Opcode) {
4351 case ISD::FADD:
4352 case ISD::FSUB:
4353 case ISD::FMUL:
4354 case ISD::FDIV:
4355 case ISD::FREM:
4356 case ISD::FSIN:
4357 case ISD::FCOS: {
4358 if (SNaN)
4359 return true;
4360 // TODO: Need isKnownNeverInfinity
4361 return false;
4362 }
4363 case ISD::FCANONICALIZE:
4364 case ISD::FEXP:
4365 case ISD::FEXP2:
4366 case ISD::FTRUNC:
4367 case ISD::FFLOOR:
4368 case ISD::FCEIL:
4369 case ISD::FROUND:
4370 case ISD::FROUNDEVEN:
4371 case ISD::FRINT:
4372 case ISD::FNEARBYINT: {
4373 if (SNaN)
4374 return true;
4375 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4376 }
4377 case ISD::FABS:
4378 case ISD::FNEG:
4379 case ISD::FCOPYSIGN: {
4380 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4381 }
4382 case ISD::SELECT:
4383 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4384 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4385 case ISD::FP_EXTEND:
4386 case ISD::FP_ROUND: {
4387 if (SNaN)
4388 return true;
4389 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4390 }
4391 case ISD::SINT_TO_FP:
4392 case ISD::UINT_TO_FP:
4393 return true;
4394 case ISD::FMA:
4395 case ISD::FMAD: {
4396 if (SNaN)
4397 return true;
4398 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4399 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4400 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4401 }
4402 case ISD::FSQRT: // Need is known positive
4403 case ISD::FLOG:
4404 case ISD::FLOG2:
4405 case ISD::FLOG10:
4406 case ISD::FPOWI:
4407 case ISD::FPOW: {
4408 if (SNaN)
4409 return true;
4410 // TODO: Refine on operand
4411 return false;
4412 }
4413 case ISD::FMINNUM:
4414 case ISD::FMAXNUM: {
4415 // Only one needs to be known not-nan, since it will be returned if the
4416 // other ends up being one.
4417 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4418 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4419 }
4420 case ISD::FMINNUM_IEEE:
4421 case ISD::FMAXNUM_IEEE: {
4422 if (SNaN)
4423 return true;
4424 // This can return a NaN if either operand is an sNaN, or if both operands
4425 // are NaN.
4426 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4427 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4428 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4429 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4430 }
4431 case ISD::FMINIMUM:
4432 case ISD::FMAXIMUM: {
4433 // TODO: Does this quiet or return the origina NaN as-is?
4434 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4435 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4436 }
4437 case ISD::EXTRACT_VECTOR_ELT: {
4438 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4439 }
4440 default:
4441 if (Opcode >= ISD::BUILTIN_OP_END ||
4442 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4443 Opcode == ISD::INTRINSIC_W_CHAIN ||
4444 Opcode == ISD::INTRINSIC_VOID) {
4445 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4446 }
4447
4448 return false;
4449 }
4450}
4451
4452bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4453 assert(Op.getValueType().isFloatingPoint() &&(static_cast <bool> (Op.getValueType().isFloatingPoint(
) && "Floating point type expected") ? void (0) : __assert_fail
("Op.getValueType().isFloatingPoint() && \"Floating point type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4454, __extension__ __PRETTY_FUNCTION__))
4454 "Floating point type expected")(static_cast <bool> (Op.getValueType().isFloatingPoint(
) && "Floating point type expected") ? void (0) : __assert_fail
("Op.getValueType().isFloatingPoint() && \"Floating point type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4454, __extension__ __PRETTY_FUNCTION__))
;
4455
4456 // If the value is a constant, we can obviously see if it is a zero or not.
4457 // TODO: Add BuildVector support.
4458 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4459 return !C->isZero();
4460 return false;
4461}
4462
4463bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4464 assert(!Op.getValueType().isFloatingPoint() &&(static_cast <bool> (!Op.getValueType().isFloatingPoint
() && "Floating point types unsupported - use isKnownNeverZeroFloat"
) ? void (0) : __assert_fail ("!Op.getValueType().isFloatingPoint() && \"Floating point types unsupported - use isKnownNeverZeroFloat\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4465, __extension__ __PRETTY_FUNCTION__))
4465 "Floating point types unsupported - use isKnownNeverZeroFloat")(static_cast <bool> (!Op.getValueType().isFloatingPoint
() && "Floating point types unsupported - use isKnownNeverZeroFloat"
) ? void (0) : __assert_fail ("!Op.getValueType().isFloatingPoint() && \"Floating point types unsupported - use isKnownNeverZeroFloat\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4465, __extension__ __PRETTY_FUNCTION__))
;
4466
4467 // If the value is a constant, we can obviously see if it is a zero or not.
4468 if (ISD::matchUnaryPredicate(
4469 Op, [](ConstantSDNode *C) { return !C->isNullValue(); }))
4470 return true;
4471
4472 // TODO: Recognize more cases here.
4473 switch (Op.getOpcode()) {
4474 default: break;
4475 case ISD::OR:
4476 if (isKnownNeverZero(Op.getOperand(1)) ||
4477 isKnownNeverZero(Op.getOperand(0)))
4478 return true;
4479 break;
4480 }
4481
4482 return false;
4483}
4484
4485bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4486 // Check the obvious case.
4487 if (A == B) return true;
4488
4489 // For for negative and positive zero.
4490 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4491 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4492 if (CA->isZero() && CB->isZero()) return true;
4493
4494 // Otherwise they may not be equal.
4495 return false;
4496}
4497
4498// FIXME: unify with llvm::haveNoCommonBitsSet.
4499// FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4500bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4501 assert(A.getValueType() == B.getValueType() &&(static_cast <bool> (A.getValueType() == B.getValueType
() && "Values must have the same type") ? void (0) : __assert_fail
("A.getValueType() == B.getValueType() && \"Values must have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4502, __extension__ __PRETTY_FUNCTION__))
4502 "Values must have the same type")(static_cast <bool> (A.getValueType() == B.getValueType
() && "Values must have the same type") ? void (0) : __assert_fail
("A.getValueType() == B.getValueType() && \"Values must have the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4502, __extension__ __PRETTY_FUNCTION__))
;
4503 return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),
4504 computeKnownBits(B));
4505}
4506
4507static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
4508 SelectionDAG &DAG) {
4509 if (cast<ConstantSDNode>(Step)->isNullValue())
4510 return DAG.getConstant(0, DL, VT);
4511
4512 return SDValue();
4513}
4514
4515static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4516 ArrayRef<SDValue> Ops,
4517 SelectionDAG &DAG) {
4518 int NumOps = Ops.size();
4519 assert(NumOps != 0 && "Can't build an empty vector!")(static_cast <bool> (NumOps != 0 && "Can't build an empty vector!"
) ? void (0) : __assert_fail ("NumOps != 0 && \"Can't build an empty vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4519, __extension__ __PRETTY_FUNCTION__))
;
4520 assert(!VT.isScalableVector() &&(static_cast <bool> (!VT.isScalableVector() && "BUILD_VECTOR cannot be used with scalable types"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"BUILD_VECTOR cannot be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4521, __extension__ __PRETTY_FUNCTION__))
4521 "BUILD_VECTOR cannot be used with scalable types")(static_cast <bool> (!VT.isScalableVector() && "BUILD_VECTOR cannot be used with scalable types"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"BUILD_VECTOR cannot be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4521, __extension__ __PRETTY_FUNCTION__))
;
4522 assert(VT.getVectorNumElements() == (unsigned)NumOps &&(static_cast <bool> (VT.getVectorNumElements() == (unsigned
)NumOps && "Incorrect element count in BUILD_VECTOR!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)NumOps && \"Incorrect element count in BUILD_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4523, __extension__ __PRETTY_FUNCTION__))
4523 "Incorrect element count in BUILD_VECTOR!")(static_cast <bool> (VT.getVectorNumElements() == (unsigned
)NumOps && "Incorrect element count in BUILD_VECTOR!"
) ? void (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)NumOps && \"Incorrect element count in BUILD_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4523, __extension__ __PRETTY_FUNCTION__))
;
4524
4525 // BUILD_VECTOR of UNDEFs is UNDEF.
4526 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4527 return DAG.getUNDEF(VT);
4528
4529 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4530 SDValue IdentitySrc;
4531 bool IsIdentity = true;
4532 for (int i = 0; i != NumOps; ++i) {
4533 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4534 Ops[i].getOperand(0).getValueType() != VT ||
4535 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
4536 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
4537 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) {
4538 IsIdentity = false;
4539 break;
4540 }
4541 IdentitySrc = Ops[i].getOperand(0);
4542 }
4543 if (IsIdentity)
4544 return IdentitySrc;
4545
4546 return SDValue();
4547}
4548
4549/// Try to simplify vector concatenation to an input value, undef, or build
4550/// vector.
4551static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4552 ArrayRef<SDValue> Ops,
4553 SelectionDAG &DAG) {
4554 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!")(static_cast <bool> (!Ops.empty() && "Can't concatenate an empty list of vectors!"
) ? void (0) : __assert_fail ("!Ops.empty() && \"Can't concatenate an empty list of vectors!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4554, __extension__ __PRETTY_FUNCTION__))
;
4555 assert(llvm::all_of(Ops,(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4559, __extension__ __PRETTY_FUNCTION__))
4556 [Ops](SDValue Op) {(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4559, __extension__ __PRETTY_FUNCTION__))
4557 return Ops[0].getValueType() == Op.getValueType();(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4559, __extension__ __PRETTY_FUNCTION__))
4558 }) &&(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4559, __extension__ __PRETTY_FUNCTION__))
4559 "Concatenation of vectors with inconsistent value types!")(static_cast <bool> (llvm::all_of(Ops, [Ops](SDValue Op
) { return Ops[0].getValueType() == Op.getValueType(); }) &&
"Concatenation of vectors with inconsistent value types!") ?
void (0) : __assert_fail ("llvm::all_of(Ops, [Ops](SDValue Op) { return Ops[0].getValueType() == Op.getValueType(); }) && \"Concatenation of vectors with inconsistent value types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4559, __extension__ __PRETTY_FUNCTION__))
;
4560 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==(static_cast <bool> ((Ops[0].getValueType().getVectorElementCount
() * Ops.size()) == VT.getVectorElementCount() && "Incorrect element count in vector concatenation!"
) ? void (0) : __assert_fail ("(Ops[0].getValueType().getVectorElementCount() * Ops.size()) == VT.getVectorElementCount() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4562, __extension__ __PRETTY_FUNCTION__))
4561 VT.getVectorElementCount() &&(static_cast <bool> ((Ops[0].getValueType().getVectorElementCount
() * Ops.size()) == VT.getVectorElementCount() && "Incorrect element count in vector concatenation!"
) ? void (0) : __assert_fail ("(Ops[0].getValueType().getVectorElementCount() * Ops.size()) == VT.getVectorElementCount() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4562, __extension__ __PRETTY_FUNCTION__))
4562 "Incorrect element count in vector concatenation!")(static_cast <bool> ((Ops[0].getValueType().getVectorElementCount
() * Ops.size()) == VT.getVectorElementCount() && "Incorrect element count in vector concatenation!"
) ? void (0) : __assert_fail ("(Ops[0].getValueType().getVectorElementCount() * Ops.size()) == VT.getVectorElementCount() && \"Incorrect element count in vector concatenation!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4562, __extension__ __PRETTY_FUNCTION__))
;
4563
4564 if (Ops.size() == 1)
4565 return Ops[0];
4566
4567 // Concat of UNDEFs is UNDEF.
4568 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4569 return DAG.getUNDEF(VT);
4570
4571 // Scan the operands and look for extract operations from a single source
4572 // that correspond to insertion at the same location via this concatenation:
4573 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4574 SDValue IdentitySrc;
4575 bool IsIdentity = true;
4576 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
4577 SDValue Op = Ops[i];
4578 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
4579 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4580 Op.getOperand(0).getValueType() != VT ||
4581 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
4582 Op.getConstantOperandVal(1) != IdentityIndex) {
4583 IsIdentity = false;
4584 break;
4585 }
4586 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&(static_cast <bool> ((!IdentitySrc || IdentitySrc == Op
.getOperand(0)) && "Unexpected identity source vector for concat of extracts"
) ? void (0) : __assert_fail ("(!IdentitySrc || IdentitySrc == Op.getOperand(0)) && \"Unexpected identity source vector for concat of extracts\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4587, __extension__ __PRETTY_FUNCTION__))
4587 "Unexpected identity source vector for concat of extracts")(static_cast <bool> ((!IdentitySrc || IdentitySrc == Op
.getOperand(0)) && "Unexpected identity source vector for concat of extracts"
) ? void (0) : __assert_fail ("(!IdentitySrc || IdentitySrc == Op.getOperand(0)) && \"Unexpected identity source vector for concat of extracts\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4587, __extension__ __PRETTY_FUNCTION__))
;
4588 IdentitySrc = Op.getOperand(0);
4589 }
4590 if (IsIdentity) {
4591 assert(IdentitySrc && "Failed to set source vector of extracts")(static_cast <bool> (IdentitySrc && "Failed to set source vector of extracts"
) ? void (0) : __assert_fail ("IdentitySrc && \"Failed to set source vector of extracts\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4591, __extension__ __PRETTY_FUNCTION__))
;
4592 return IdentitySrc;
4593 }
4594
4595 // The code below this point is only designed to work for fixed width
4596 // vectors, so we bail out for now.
4597 if (VT.isScalableVector())
4598 return SDValue();
4599
4600 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4601 // simplified to one big BUILD_VECTOR.
4602 // FIXME: Add support for SCALAR_TO_VECTOR as well.
4603 EVT SVT = VT.getScalarType();
4604 SmallVector<SDValue, 16> Elts;
4605 for (SDValue Op : Ops) {
4606 EVT OpVT = Op.getValueType();
4607 if (Op.isUndef())
4608 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4609 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4610 Elts.append(Op->op_begin(), Op->op_end());
4611 else
4612 return SDValue();
4613 }
4614
4615 // BUILD_VECTOR requires all inputs to be of the same type, find the
4616 // maximum type and extend them all.
4617 for (SDValue Op : Elts)
4618 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
4619
4620 if (SVT.bitsGT(VT.getScalarType())) {
4621 for (SDValue &Op : Elts) {
4622 if (Op.isUndef())
4623 Op = DAG.getUNDEF(SVT);
4624 else
4625 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4626 ? DAG.getZExtOrTrunc(Op, DL, SVT)
4627 : DAG.getSExtOrTrunc(Op, DL, SVT);
4628 }
4629 }
4630
4631 SDValue V = DAG.getBuildVector(VT, DL, Elts);
4632 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4633 return V;
4634}
4635
4636/// Gets or creates the specified node.
4637SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4638 FoldingSetNodeID ID;
4639 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4640 void *IP = nullptr;
4641 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4642 return SDValue(E, 0);
4643
4644 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4645 getVTList(VT));
4646 CSEMap.InsertNode(N, IP);
4647
4648 InsertNode(N);
4649 SDValue V = SDValue(N, 0);
4650 NewSDValueDbgMsg(V, "Creating new node: ", this);
4651 return V;
4652}
4653
4654SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4655 SDValue Operand) {
4656 SDNodeFlags Flags;
4657 if (Inserter)
4658 Flags = Inserter->getFlags();
4659 return getNode(Opcode, DL, VT, Operand, Flags);
4660}
4661
4662SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4663 SDValue Operand, const SDNodeFlags Flags) {
4664 assert(Operand.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (Operand.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Operand.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4665, __extension__ __PRETTY_FUNCTION__))
4665 "Operand is DELETED_NODE!")(static_cast <bool> (Operand.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Operand.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4665, __extension__ __PRETTY_FUNCTION__))
;
4666 // Constant fold unary operations with an integer constant operand. Even
4667 // opaque constant will be folded, because the folding of unary operations
4668 // doesn't create new constants with different values. Nevertheless, the
4669 // opaque flag is preserved during folding to prevent future folding with
4670 // other constants.
4671 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4672 const APInt &Val = C->getAPIntValue();
4673 switch (Opcode) {
4674 default: break;
4675 case ISD::SIGN_EXTEND:
4676 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4677 C->isTargetOpcode(), C->isOpaque());
4678 case ISD::TRUNCATE:
4679 if (C->isOpaque())
4680 break;
4681 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4682 case ISD::ZERO_EXTEND:
4683 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4684 C->isTargetOpcode(), C->isOpaque());
4685 case ISD::ANY_EXTEND:
4686 // Some targets like RISCV prefer to sign extend some types.
4687 if (TLI->isSExtCheaperThanZExt(Operand.getValueType(), VT))
4688 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4689 C->isTargetOpcode(), C->isOpaque());
4690 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4691 C->isTargetOpcode(), C->isOpaque());
4692 case ISD::UINT_TO_FP:
4693 case ISD::SINT_TO_FP: {
4694 APFloat apf(EVTToAPFloatSemantics(VT),
4695 APInt::getNullValue(VT.getSizeInBits()));
4696 (void)apf.convertFromAPInt(Val,
4697 Opcode==ISD::SINT_TO_FP,
4698 APFloat::rmNearestTiesToEven);
4699 return getConstantFP(apf, DL, VT);
4700 }
4701 case ISD::BITCAST:
4702 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
4703 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4704 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
4705 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4706 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
4707 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4708 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
4709 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4710 break;
4711 case ISD::ABS:
4712 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4713 C->isOpaque());
4714 case ISD::BITREVERSE:
4715 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4716 C->isOpaque());
4717 case ISD::BSWAP:
4718 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4719 C->isOpaque());
4720 case ISD::CTPOP:
4721 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4722 C->isOpaque());
4723 case ISD::CTLZ:
4724 case ISD::CTLZ_ZERO_UNDEF:
4725 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4726 C->isOpaque());
4727 case ISD::CTTZ:
4728 case ISD::CTTZ_ZERO_UNDEF:
4729 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4730 C->isOpaque());
4731 case ISD::FP16_TO_FP: {
4732 bool Ignored;
4733 APFloat FPV(APFloat::IEEEhalf(),
4734 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
4735
4736 // This can return overflow, underflow, or inexact; we don't care.
4737 // FIXME need to be more flexible about rounding mode.
4738 (void)FPV.convert(EVTToAPFloatSemantics(VT),
4739 APFloat::rmNearestTiesToEven, &Ignored);
4740 return getConstantFP(FPV, DL, VT);
4741 }
4742 case ISD::STEP_VECTOR: {
4743 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Operand, *this))
4744 return V;
4745 break;
4746 }
4747 }
4748 }
4749
4750 // Constant fold unary operations with a floating point constant operand.
4751 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4752 APFloat V = C->getValueAPF(); // make copy
4753 switch (Opcode) {
4754 case ISD::FNEG:
4755 V.changeSign();
4756 return getConstantFP(V, DL, VT);
4757 case ISD::FABS:
4758 V.clearSign();
4759 return getConstantFP(V, DL, VT);
4760 case ISD::FCEIL: {
4761 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4762 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4763 return getConstantFP(V, DL, VT);
4764 break;
4765 }
4766 case ISD::FTRUNC: {
4767 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4768 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4769 return getConstantFP(V, DL, VT);
4770 break;
4771 }
4772 case ISD::FFLOOR: {
4773 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4774 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4775 return getConstantFP(V, DL, VT);
4776 break;
4777 }
4778 case ISD::FP_EXTEND: {
4779 bool ignored;
4780 // This can return overflow, underflow, or inexact; we don't care.
4781 // FIXME need to be more flexible about rounding mode.
4782 (void)V.convert(EVTToAPFloatSemantics(VT),
4783 APFloat::rmNearestTiesToEven, &ignored);
4784 return getConstantFP(V, DL, VT);
4785 }
4786 case ISD::FP_TO_SINT:
4787 case ISD::FP_TO_UINT: {
4788 bool ignored;
4789 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4790 // FIXME need to be more flexible about rounding mode.
4791 APFloat::opStatus s =
4792 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4793 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4794 break;
4795 return getConstant(IntVal, DL, VT);
4796 }
4797 case ISD::BITCAST:
4798 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
4799 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4800 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
4801 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4802 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
4803 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4804 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
4805 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4806 break;
4807 case ISD::FP_TO_FP16: {
4808 bool Ignored;
4809 // This can return overflow, underflow, or inexact; we don't care.
4810 // FIXME need to be more flexible about rounding mode.
4811 (void)V.convert(APFloat::IEEEhalf(),
4812 APFloat::rmNearestTiesToEven, &Ignored);
4813 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4814 }
4815 }
4816 }
4817
4818 // Constant fold unary operations with a vector integer or float operand.
4819 switch (Opcode) {
4820 default:
4821 // FIXME: Entirely reasonable to perform folding of other unary
4822 // operations here as the need arises.
4823 break;
4824 case ISD::FNEG:
4825 case ISD::FABS:
4826 case ISD::FCEIL:
4827 case ISD::FTRUNC:
4828 case ISD::FFLOOR:
4829 case ISD::FP_EXTEND:
4830 case ISD::FP_TO_SINT:
4831 case ISD::FP_TO_UINT:
4832 case ISD::TRUNCATE:
4833 case ISD::ANY_EXTEND:
4834 case ISD::ZERO_EXTEND:
4835 case ISD::SIGN_EXTEND:
4836 case ISD::UINT_TO_FP:
4837 case ISD::SINT_TO_FP:
4838 case ISD::ABS:
4839 case ISD::BITREVERSE:
4840 case ISD::BSWAP:
4841 case ISD::CTLZ:
4842 case ISD::CTLZ_ZERO_UNDEF:
4843 case ISD::CTTZ:
4844 case ISD::CTTZ_ZERO_UNDEF:
4845 case ISD::CTPOP: {
4846 SDValue Ops = {Operand};
4847 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4848 return Fold;
4849 }
4850 }
4851
4852 unsigned OpOpcode = Operand.getNode()->getOpcode();
4853 switch (Opcode) {
4854 case ISD::STEP_VECTOR:
4855 assert(VT.isScalableVector() &&(static_cast <bool> (VT.isScalableVector() && "STEP_VECTOR can only be used with scalable types"
) ? void (0) : __assert_fail ("VT.isScalableVector() && \"STEP_VECTOR can only be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4856, __extension__ __PRETTY_FUNCTION__))
4856 "STEP_VECTOR can only be used with scalable types")(static_cast <bool> (VT.isScalableVector() && "STEP_VECTOR can only be used with scalable types"
) ? void (0) : __assert_fail ("VT.isScalableVector() && \"STEP_VECTOR can only be used with scalable types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4856, __extension__ __PRETTY_FUNCTION__))
;
4857 assert(OpOpcode == ISD::TargetConstant &&(static_cast <bool> (OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand") ? void (0) : __assert_fail ("OpOpcode == ISD::TargetConstant && VT.getVectorElementType() == Operand.getValueType() && \"Unexpected step operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4859, __extension__ __PRETTY_FUNCTION__))
4858 VT.getVectorElementType() == Operand.getValueType() &&(static_cast <bool> (OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand") ? void (0) : __assert_fail ("OpOpcode == ISD::TargetConstant && VT.getVectorElementType() == Operand.getValueType() && \"Unexpected step operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4859, __extension__ __PRETTY_FUNCTION__))
4859 "Unexpected step operand")(static_cast <bool> (OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand") ? void (0) : __assert_fail ("OpOpcode == ISD::TargetConstant && VT.getVectorElementType() == Operand.getValueType() && \"Unexpected step operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4859, __extension__ __PRETTY_FUNCTION__))
;
4860 break;
4861 case ISD::FREEZE:
4862 assert(VT == Operand.getValueType() && "Unexpected VT!")(static_cast <bool> (VT == Operand.getValueType() &&
"Unexpected VT!") ? void (0) : __assert_fail ("VT == Operand.getValueType() && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4862, __extension__ __PRETTY_FUNCTION__))
;
4863 break;
4864 case ISD::TokenFactor:
4865 case ISD::MERGE_VALUES:
4866 case ISD::CONCAT_VECTORS:
4867 return Operand; // Factor, merge or concat of one node? No need.
4868 case ISD::BUILD_VECTOR: {
4869 // Attempt to simplify BUILD_VECTOR.
4870 SDValue Ops[] = {Operand};
4871 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4872 return V;
4873 break;
4874 }
4875 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node")::llvm::llvm_unreachable_internal("Invalid method to make FP_ROUND node"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4875)
;
4876 case ISD::FP_EXTEND:
4877 assert(VT.isFloatingPoint() &&(static_cast <bool> (VT.isFloatingPoint() && Operand
.getValueType().isFloatingPoint() && "Invalid FP cast!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4878, __extension__ __PRETTY_FUNCTION__))
4878 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!")(static_cast <bool> (VT.isFloatingPoint() && Operand
.getValueType().isFloatingPoint() && "Invalid FP cast!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4878, __extension__ __PRETTY_FUNCTION__))
;
4879 if (Operand.getValueType() == VT) return Operand; // noop conversion.
4880 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4883, __extension__ __PRETTY_FUNCTION__))
4881 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4883, __extension__ __PRETTY_FUNCTION__))
4882 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4883, __extension__ __PRETTY_FUNCTION__))
4883 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4883, __extension__ __PRETTY_FUNCTION__))
;
4884 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid fpext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid fpext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4885, __extension__ __PRETTY_FUNCTION__))
4885 "Invalid fpext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid fpext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid fpext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4885, __extension__ __PRETTY_FUNCTION__))
;
4886 if (Operand.isUndef())
4887 return getUNDEF(VT);
4888 break;
4889 case ISD::FP_TO_SINT:
4890 case ISD::FP_TO_UINT:
4891 if (Operand.isUndef())
4892 return getUNDEF(VT);
4893 break;
4894 case ISD::SINT_TO_FP:
4895 case ISD::UINT_TO_FP:
4896 // [us]itofp(undef) = 0, because the result value is bounded.
4897 if (Operand.isUndef())
4898 return getConstantFP(0.0, DL, VT);
4899 break;
4900 case ISD::SIGN_EXTEND:
4901 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid SIGN_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid SIGN_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4902, __extension__ __PRETTY_FUNCTION__))
4902 "Invalid SIGN_EXTEND!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid SIGN_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid SIGN_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4902, __extension__ __PRETTY_FUNCTION__))
;
4903 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4905, __extension__ __PRETTY_FUNCTION__))
4904 "SIGN_EXTEND result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4905, __extension__ __PRETTY_FUNCTION__))
4905 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "SIGN_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"SIGN_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4905, __extension__ __PRETTY_FUNCTION__))
;
4906 if (Operand.getValueType() == VT) return Operand; // noop extension
4907 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4910, __extension__ __PRETTY_FUNCTION__))
4908 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4910, __extension__ __PRETTY_FUNCTION__))
4909 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4910, __extension__ __PRETTY_FUNCTION__))
4910 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4910, __extension__ __PRETTY_FUNCTION__))
;
4911 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid sext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid sext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4912, __extension__ __PRETTY_FUNCTION__))
4912 "Invalid sext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid sext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid sext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4912, __extension__ __PRETTY_FUNCTION__))
;
4913 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4914 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4915 if (OpOpcode == ISD::UNDEF)
4916 // sext(undef) = 0, because the top bits will all be the same.
4917 return getConstant(0, DL, VT);
4918 break;
4919 case ISD::ZERO_EXTEND:
4920 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ZERO_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ZERO_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4921, __extension__ __PRETTY_FUNCTION__))
4921 "Invalid ZERO_EXTEND!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ZERO_EXTEND!")
? void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ZERO_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4921, __extension__ __PRETTY_FUNCTION__))
;
4922 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4924, __extension__ __PRETTY_FUNCTION__))
4923 "ZERO_EXTEND result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4924, __extension__ __PRETTY_FUNCTION__))
4924 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ZERO_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ZERO_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4924, __extension__ __PRETTY_FUNCTION__))
;
4925 if (Operand.getValueType() == VT) return Operand; // noop extension
4926 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4929, __extension__ __PRETTY_FUNCTION__))
4927 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4929, __extension__ __PRETTY_FUNCTION__))
4928 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4929, __extension__ __PRETTY_FUNCTION__))
4929 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4929, __extension__ __PRETTY_FUNCTION__))
;
4930 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid zext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid zext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4931, __extension__ __PRETTY_FUNCTION__))
4931 "Invalid zext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid zext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid zext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4931, __extension__ __PRETTY_FUNCTION__))
;
4932 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
4933 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4934 if (OpOpcode == ISD::UNDEF)
4935 // zext(undef) = 0, because the top bits will be zero.
4936 return getConstant(0, DL, VT);
4937 break;
4938 case ISD::ANY_EXTEND:
4939 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ANY_EXTEND!") ?
void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ANY_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4940, __extension__ __PRETTY_FUNCTION__))
4940 "Invalid ANY_EXTEND!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid ANY_EXTEND!") ?
void (0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid ANY_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4940, __extension__ __PRETTY_FUNCTION__))
;
4941 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4943, __extension__ __PRETTY_FUNCTION__))
4942 "ANY_EXTEND result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4943, __extension__ __PRETTY_FUNCTION__))
4943 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "ANY_EXTEND result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"ANY_EXTEND result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4943, __extension__ __PRETTY_FUNCTION__))
;
4944 if (Operand.getValueType() == VT) return Operand; // noop extension
4945 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4948, __extension__ __PRETTY_FUNCTION__))
4946 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4948, __extension__ __PRETTY_FUNCTION__))
4947 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4948, __extension__ __PRETTY_FUNCTION__))
4948 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4948, __extension__ __PRETTY_FUNCTION__))
;
4949 assert(Operand.getValueType().bitsLT(VT) &&(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid anyext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid anyext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4950, __extension__ __PRETTY_FUNCTION__))
4950 "Invalid anyext node, dst < src!")(static_cast <bool> (Operand.getValueType().bitsLT(VT) &&
"Invalid anyext node, dst < src!") ? void (0) : __assert_fail
("Operand.getValueType().bitsLT(VT) && \"Invalid anyext node, dst < src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4950, __extension__ __PRETTY_FUNCTION__))
;
4951
4952 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4953 OpOpcode == ISD::ANY_EXTEND)
4954 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
4955 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4956 if (OpOpcode == ISD::UNDEF)
4957 return getUNDEF(VT);
4958
4959 // (ext (trunc x)) -> x
4960 if (OpOpcode == ISD::TRUNCATE) {
4961 SDValue OpOp = Operand.getOperand(0);
4962 if (OpOp.getValueType() == VT) {
4963 transferDbgValues(Operand, OpOp);
4964 return OpOp;
4965 }
4966 }
4967 break;
4968 case ISD::TRUNCATE:
4969 assert(VT.isInteger() && Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid TRUNCATE!") ? void
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid TRUNCATE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__))
4970 "Invalid TRUNCATE!")(static_cast <bool> (VT.isInteger() && Operand.
getValueType().isInteger() && "Invalid TRUNCATE!") ? void
(0) : __assert_fail ("VT.isInteger() && Operand.getValueType().isInteger() && \"Invalid TRUNCATE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__))
;
4971 assert(VT.isVector() == Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4973, __extension__ __PRETTY_FUNCTION__))
4972 "TRUNCATE result type type should be vector iff the operand "(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4973, __extension__ __PRETTY_FUNCTION__))
4973 "type is vector!")(static_cast <bool> (VT.isVector() == Operand.getValueType
().isVector() && "TRUNCATE result type type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VT.isVector() == Operand.getValueType().isVector() && \"TRUNCATE result type type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4973, __extension__ __PRETTY_FUNCTION__))
;
4974 if (Operand.getValueType() == VT) return Operand; // noop truncate
4975 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4978, __extension__ __PRETTY_FUNCTION__))
4976 VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4978, __extension__ __PRETTY_FUNCTION__))
4977 Operand.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4978, __extension__ __PRETTY_FUNCTION__))
4978 "Vector element count mismatch!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == Operand.getValueType().getVectorElementCount()) &&
"Vector element count mismatch!") ? void (0) : __assert_fail
("(!VT.isVector() || VT.getVectorElementCount() == Operand.getValueType().getVectorElementCount()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4978, __extension__ __PRETTY_FUNCTION__))
;
4979 assert(Operand.getValueType().bitsGT(VT) &&(static_cast <bool> (Operand.getValueType().bitsGT(VT) &&
"Invalid truncate node, src < dst!") ? void (0) : __assert_fail
("Operand.getValueType().bitsGT(VT) && \"Invalid truncate node, src < dst!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__))
4980 "Invalid truncate node, src < dst!")(static_cast <bool> (Operand.getValueType().bitsGT(VT) &&
"Invalid truncate node, src < dst!") ? void (0) : __assert_fail
("Operand.getValueType().bitsGT(VT) && \"Invalid truncate node, src < dst!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__))
;
4981 if (OpOpcode == ISD::TRUNCATE)
4982 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4983 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4984 OpOpcode == ISD::ANY_EXTEND) {
4985 // If the source is smaller than the dest, we still need an extend.
4986 if (Operand.getOperand(0).getValueType().getScalarType()
4987 .bitsLT(VT.getScalarType()))
4988 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4989 if (Operand.getOperand(0).getValueType().bitsGT(VT))
4990 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4991 return Operand.getOperand(0);
4992 }
4993 if (OpOpcode == ISD::UNDEF)
4994 return getUNDEF(VT);
4995 break;
4996 case ISD::ANY_EXTEND_VECTOR_INREG:
4997 case ISD::ZERO_EXTEND_VECTOR_INREG:
4998 case ISD::SIGN_EXTEND_VECTOR_INREG:
4999 assert(VT.isVector() && "This DAG node is restricted to vector types.")(static_cast <bool> (VT.isVector() && "This DAG node is restricted to vector types."
) ? void (0) : __assert_fail ("VT.isVector() && \"This DAG node is restricted to vector types.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 4999, __extension__ __PRETTY_FUNCTION__))
;
5000 assert(Operand.getValueType().bitsLE(VT) &&(static_cast <bool> (Operand.getValueType().bitsLE(VT) &&
"The input must be the same size or smaller than the result."
) ? void (0) : __assert_fail ("Operand.getValueType().bitsLE(VT) && \"The input must be the same size or smaller than the result.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5001, __extension__ __PRETTY_FUNCTION__))
5001 "The input must be the same size or smaller than the result.")(static_cast <bool> (Operand.getValueType().bitsLE(VT) &&
"The input must be the same size or smaller than the result."
) ? void (0) : __assert_fail ("Operand.getValueType().bitsLE(VT) && \"The input must be the same size or smaller than the result.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5001, __extension__ __PRETTY_FUNCTION__))
;
5002 assert(VT.getVectorMinNumElements() <(static_cast <bool> (VT.getVectorMinNumElements() < Operand
.getValueType().getVectorMinNumElements() && "The destination vector type must have fewer lanes than the input."
) ? void (0) : __assert_fail ("VT.getVectorMinNumElements() < Operand.getValueType().getVectorMinNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5004, __extension__ __PRETTY_FUNCTION__))
5003 Operand.getValueType().getVectorMinNumElements() &&(static_cast <bool> (VT.getVectorMinNumElements() < Operand
.getValueType().getVectorMinNumElements() && "The destination vector type must have fewer lanes than the input."
) ? void (0) : __assert_fail ("VT.getVectorMinNumElements() < Operand.getValueType().getVectorMinNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5004, __extension__ __PRETTY_FUNCTION__))
5004 "The destination vector type must have fewer lanes than the input.")(static_cast <bool> (VT.getVectorMinNumElements() < Operand
.getValueType().getVectorMinNumElements() && "The destination vector type must have fewer lanes than the input."
) ? void (0) : __assert_fail ("VT.getVectorMinNumElements() < Operand.getValueType().getVectorMinNumElements() && \"The destination vector type must have fewer lanes than the input.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5004, __extension__ __PRETTY_FUNCTION__))
;
5005 break;
5006 case ISD::ABS:
5007 assert(VT.isInteger() && VT == Operand.getValueType() &&(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid ABS!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid ABS!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5008, __extension__ __PRETTY_FUNCTION__))
5008 "Invalid ABS!")(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid ABS!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid ABS!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5008, __extension__ __PRETTY_FUNCTION__))
;
5009 if (OpOpcode == ISD::UNDEF)
5010 return getUNDEF(VT);
5011 break;
5012 case ISD::BSWAP:
5013 assert(VT.isInteger() && VT == Operand.getValueType() &&(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BSWAP!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BSWAP!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5014, __extension__ __PRETTY_FUNCTION__))
5014 "Invalid BSWAP!")(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BSWAP!") ? void (0) : __assert_fail
("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BSWAP!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5014, __extension__ __PRETTY_FUNCTION__))
;
5015 assert((VT.getScalarSizeInBits() % 16 == 0) &&(static_cast <bool> ((VT.getScalarSizeInBits() % 16 == 0
) && "BSWAP types must be a multiple of 16 bits!") ? void
(0) : __assert_fail ("(VT.getScalarSizeInBits() % 16 == 0) && \"BSWAP types must be a multiple of 16 bits!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5016, __extension__ __PRETTY_FUNCTION__))
5016 "BSWAP types must be a multiple of 16 bits!")(static_cast <bool> ((VT.getScalarSizeInBits() % 16 == 0
) && "BSWAP types must be a multiple of 16 bits!") ? void
(0) : __assert_fail ("(VT.getScalarSizeInBits() % 16 == 0) && \"BSWAP types must be a multiple of 16 bits!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5016, __extension__ __PRETTY_FUNCTION__))
;
5017 if (OpOpcode == ISD::UNDEF)
5018 return getUNDEF(VT);
5019 break;
5020 case ISD::BITREVERSE:
5021 assert(VT.isInteger() && VT == Operand.getValueType() &&(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BITREVERSE!") ? void (0) :
__assert_fail ("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BITREVERSE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5022, __extension__ __PRETTY_FUNCTION__))
5022 "Invalid BITREVERSE!")(static_cast <bool> (VT.isInteger() && VT == Operand
.getValueType() && "Invalid BITREVERSE!") ? void (0) :
__assert_fail ("VT.isInteger() && VT == Operand.getValueType() && \"Invalid BITREVERSE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5022, __extension__ __PRETTY_FUNCTION__))
;
5023 if (OpOpcode == ISD::UNDEF)
5024 return getUNDEF(VT);
5025 break;
5026 case ISD::BITCAST:
5027 // Basic sanity checking.
5028 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&(static_cast <bool> (VT.getSizeInBits() == Operand.getValueSizeInBits
() && "Cannot BITCAST between types of different sizes!"
) ? void (0) : __assert_fail ("VT.getSizeInBits() == Operand.getValueSizeInBits() && \"Cannot BITCAST between types of different sizes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5029, __extension__ __PRETTY_FUNCTION__))
5029 "Cannot BITCAST between types of different sizes!")(static_cast <bool> (VT.getSizeInBits() == Operand.getValueSizeInBits
() && "Cannot BITCAST between types of different sizes!"
) ? void (0) : __assert_fail ("VT.getSizeInBits() == Operand.getValueSizeInBits() && \"Cannot BITCAST between types of different sizes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5029, __extension__ __PRETTY_FUNCTION__))
;
5030 if (VT == Operand.getValueType()) return Operand; // noop conversion.
5031 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5032 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
5033 if (OpOpcode == ISD::UNDEF)
5034 return getUNDEF(VT);
5035 break;
5036 case ISD::SCALAR_TO_VECTOR:
5037 assert(VT.isVector() && !Operand.getValueType().isVector() &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
5038 (VT.getVectorElementType() == Operand.getValueType() ||(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
5039 (VT.getVectorElementType().isInteger() &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
5040 Operand.getValueType().isInteger() &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
5041 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
5042 "Illegal SCALAR_TO_VECTOR node!")(static_cast <bool> (VT.isVector() && !Operand.
getValueType().isVector() && (VT.getVectorElementType
() == Operand.getValueType() || (VT.getVectorElementType().isInteger
() && Operand.getValueType().isInteger() && VT
.getVectorElementType().bitsLE(Operand.getValueType()))) &&
"Illegal SCALAR_TO_VECTOR node!") ? void (0) : __assert_fail
("VT.isVector() && !Operand.getValueType().isVector() && (VT.getVectorElementType() == Operand.getValueType() || (VT.getVectorElementType().isInteger() && Operand.getValueType().isInteger() && VT.getVectorElementType().bitsLE(Operand.getValueType()))) && \"Illegal SCALAR_TO_VECTOR node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5042, __extension__ __PRETTY_FUNCTION__))
;
5043 if (OpOpcode == ISD::UNDEF)
5044 return getUNDEF(VT);
5045 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5046 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5047 isa<ConstantSDNode>(Operand.getOperand(1)) &&
5048 Operand.getConstantOperandVal(1) == 0 &&
5049 Operand.getOperand(0).getValueType() == VT)
5050 return Operand.getOperand(0);
5051 break;
5052 case ISD::FNEG:
5053 // Negation of an unknown bag of bits is still completely undefined.
5054 if (OpOpcode == ISD::UNDEF)
5055 return getUNDEF(VT);
5056
5057 if (OpOpcode == ISD::FNEG) // --X -> X
5058 return Operand.getOperand(0);
5059 break;
5060 case ISD::FABS:
5061 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5062 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
5063 break;
5064 case ISD::VSCALE:
5065 assert(VT == Operand.getValueType() && "Unexpected VT!")(static_cast <bool> (VT == Operand.getValueType() &&
"Unexpected VT!") ? void (0) : __assert_fail ("VT == Operand.getValueType() && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))
;
5066 break;
5067 case ISD::CTPOP:
5068 if (Operand.getValueType().getScalarType() == MVT::i1)
5069 return Operand;
5070 break;
5071 case ISD::CTLZ:
5072 case ISD::CTTZ:
5073 if (Operand.getValueType().getScalarType() == MVT::i1)
5074 return getNOT(DL, Operand, Operand.getValueType());
5075 break;
5076 case ISD::VECREDUCE_SMIN:
5077 case ISD::VECREDUCE_UMAX:
5078 if (Operand.getValueType().getScalarType() == MVT::i1)
5079 return getNode(ISD::VECREDUCE_OR, DL, VT, Operand);
5080 break;
5081 case ISD::VECREDUCE_SMAX:
5082 case ISD::VECREDUCE_UMIN:
5083 if (Operand.getValueType().getScalarType() == MVT::i1)
5084 return getNode(ISD::VECREDUCE_AND, DL, VT, Operand);
5085 break;
5086 }
5087
5088 SDNode *N;
5089 SDVTList VTs = getVTList(VT);
5090 SDValue Ops[] = {Operand};
5091 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
5092 FoldingSetNodeID ID;
5093 AddNodeIDNode(ID, Opcode, VTs, Ops);
5094 void *IP = nullptr;
5095 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5096 E->intersectFlagsWith(Flags);
5097 return SDValue(E, 0);
5098 }
5099
5100 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5101 N->setFlags(Flags);
5102 createOperands(N, Ops);
5103 CSEMap.InsertNode(N, IP);
5104 } else {
5105 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5106 createOperands(N, Ops);
5107 }
5108
5109 InsertNode(N);
5110 SDValue V = SDValue(N, 0);
5111 NewSDValueDbgMsg(V, "Creating new node: ", this);
5112 return V;
5113}
5114
5115static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5116 const APInt &C2) {
5117 switch (Opcode) {
5118 case ISD::ADD: return C1 + C2;
5119 case ISD::SUB: return C1 - C2;
5120 case ISD::MUL: return C1 * C2;
5121 case ISD::AND: return C1 & C2;
5122 case ISD::OR: return C1 | C2;
5123 case ISD::XOR: return C1 ^ C2;
5124 case ISD::SHL: return C1 << C2;
5125 case ISD::SRL: return C1.lshr(C2);
5126 case ISD::SRA: return C1.ashr(C2);
5127 case ISD::ROTL: return C1.rotl(C2);
5128 case ISD::ROTR: return C1.rotr(C2);
5129 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
5130 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
5131 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
5132 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
5133 case ISD::SADDSAT: return C1.sadd_sat(C2);
5134 case ISD::UADDSAT: return C1.uadd_sat(C2);
5135 case ISD::SSUBSAT: return C1.ssub_sat(C2);
5136 case ISD::USUBSAT: return C1.usub_sat(C2);
5137 case ISD::UDIV:
5138 if (!C2.getBoolValue())
5139 break;
5140 return C1.udiv(C2);
5141 case ISD::UREM:
5142 if (!C2.getBoolValue())
5143 break;
5144 return C1.urem(C2);
5145 case ISD::SDIV:
5146 if (!C2.getBoolValue())
5147 break;
5148 return C1.sdiv(C2);
5149 case ISD::SREM:
5150 if (!C2.getBoolValue())
5151 break;
5152 return C1.srem(C2);
5153 case ISD::MULHS: {
5154 unsigned FullWidth = C1.getBitWidth() * 2;
5155 APInt C1Ext = C1.sext(FullWidth);
5156 APInt C2Ext = C2.sext(FullWidth);
5157 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5158 }
5159 case ISD::MULHU: {
5160 unsigned FullWidth = C1.getBitWidth() * 2;
5161 APInt C1Ext = C1.zext(FullWidth);
5162 APInt C2Ext = C2.zext(FullWidth);
5163 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5164 }
5165 }
5166 return llvm::None;
5167}
5168
5169SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
5170 const GlobalAddressSDNode *GA,
5171 const SDNode *N2) {
5172 if (GA->getOpcode() != ISD::GlobalAddress)
5173 return SDValue();
5174 if (!TLI->isOffsetFoldingLegal(GA))
5175 return SDValue();
5176 auto *C2 = dyn_cast<ConstantSDNode>(N2);
5177 if (!C2)
5178 return SDValue();
5179 int64_t Offset = C2->getSExtValue();
5180 switch (Opcode) {
5181 case ISD::ADD: break;
5182 case ISD::SUB: Offset = -uint64_t(Offset); break;
5183 default: return SDValue();
5184 }
5185 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
5186 GA->getOffset() + uint64_t(Offset));
5187}
5188
5189bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
5190 switch (Opcode) {
5191 case ISD::SDIV:
5192 case ISD::UDIV:
5193 case ISD::SREM:
5194 case ISD::UREM: {
5195 // If a divisor is zero/undef or any element of a divisor vector is
5196 // zero/undef, the whole op is undef.
5197 assert(Ops.size() == 2 && "Div/rem should have 2 operands")(static_cast <bool> (Ops.size() == 2 && "Div/rem should have 2 operands"
) ? void (0) : __assert_fail ("Ops.size() == 2 && \"Div/rem should have 2 operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5197, __extension__ __PRETTY_FUNCTION__))
;
5198 SDValue Divisor = Ops[1];
5199 if (Divisor.isUndef() || isNullConstant(Divisor))
5200 return true;
5201
5202 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
5203 llvm::any_of(Divisor->op_values(),
5204 [](SDValue V) { return V.isUndef() ||
5205 isNullConstant(V); });
5206 // TODO: Handle signed overflow.
5207 }
5208 // TODO: Handle oversized shifts.
5209 default:
5210 return false;
5211 }
5212}
5213
5214SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
5215 EVT VT, ArrayRef<SDValue> Ops) {
5216 // If the opcode is a target-specific ISD node, there's nothing we can
5217 // do here and the operand rules may not line up with the below, so
5218 // bail early.
5219 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
5220 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
5221 // foldCONCAT_VECTORS in getNode before this is called.
5222 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
5223 return SDValue();
5224
5225 // For now, the array Ops should only contain two values.
5226 // This enforcement will be removed once this function is merged with
5227 // FoldConstantVectorArithmetic
5228 if (Ops.size() != 2)
5229 return SDValue();
5230
5231 if (isUndef(Opcode, Ops))
5232 return getUNDEF(VT);
5233
5234 SDNode *N1 = Ops[0].getNode();
5235 SDNode *N2 = Ops[1].getNode();
5236
5237 // Handle the case of two scalars.
5238 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
5239 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
5240 if (C1->isOpaque() || C2->isOpaque())
5241 return SDValue();
5242
5243 Optional<APInt> FoldAttempt =
5244 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
5245 if (!FoldAttempt)
5246 return SDValue();
5247
5248 SDValue Folded = getConstant(FoldAttempt.getValue(), DL, VT);
5249 assert((!Folded || !VT.isVector()) &&(static_cast <bool> ((!Folded || !VT.isVector()) &&
"Can't fold vectors ops with scalar operands") ? void (0) : __assert_fail
("(!Folded || !VT.isVector()) && \"Can't fold vectors ops with scalar operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5250, __extension__ __PRETTY_FUNCTION__))
5250 "Can't fold vectors ops with scalar operands")(static_cast <bool> ((!Folded || !VT.isVector()) &&
"Can't fold vectors ops with scalar operands") ? void (0) : __assert_fail
("(!Folded || !VT.isVector()) && \"Can't fold vectors ops with scalar operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5250, __extension__ __PRETTY_FUNCTION__))
;
5251 return Folded;
5252 }
5253 }
5254
5255 // fold (add Sym, c) -> Sym+c
5256 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
5257 return FoldSymbolOffset(Opcode, VT, GA, N2);
5258 if (TLI->isCommutativeBinOp(Opcode))
5259 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
5260 return FoldSymbolOffset(Opcode, VT, GA, N1);
5261
5262 // For fixed width vectors, extract each constant element and fold them
5263 // individually. Either input may be an undef value.
5264 bool IsBVOrSV1 = N1->getOpcode() == ISD::BUILD_VECTOR ||
5265 N1->getOpcode() == ISD::SPLAT_VECTOR;
5266 if (!IsBVOrSV1 && !N1->isUndef())
5267 return SDValue();
5268 bool IsBVOrSV2 = N2->getOpcode() == ISD::BUILD_VECTOR ||
5269 N2->getOpcode() == ISD::SPLAT_VECTOR;
5270 if (!IsBVOrSV2 && !N2->isUndef())
5271 return SDValue();
5272 // If both operands are undef, that's handled the same way as scalars.
5273 if (!IsBVOrSV1 && !IsBVOrSV2)
5274 return SDValue();
5275
5276 EVT SVT = VT.getScalarType();
5277 EVT LegalSVT = SVT;
5278 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5279 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5280 if (LegalSVT.bitsLT(SVT))
5281 return SDValue();
5282 }
5283
5284 SmallVector<SDValue, 4> Outputs;
5285 unsigned NumOps = 0;
5286 if (IsBVOrSV1)
5287 NumOps = std::max(NumOps, N1->getNumOperands());
5288 if (IsBVOrSV2)
5289 NumOps = std::max(NumOps, N2->getNumOperands());
5290 assert(NumOps != 0 && "Expected non-zero operands")(static_cast <bool> (NumOps != 0 && "Expected non-zero operands"
) ? void (0) : __assert_fail ("NumOps != 0 && \"Expected non-zero operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5290, __extension__ __PRETTY_FUNCTION__))
;
5291 // Scalable vectors should only be SPLAT_VECTOR or UNDEF here. We only need
5292 // one iteration for that.
5293 assert((!VT.isScalableVector() || NumOps == 1) &&(static_cast <bool> ((!VT.isScalableVector() || NumOps ==
1) && "Scalable vector should only have one scalar")
? void (0) : __assert_fail ("(!VT.isScalableVector() || NumOps == 1) && \"Scalable vector should only have one scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5294, __extension__ __PRETTY_FUNCTION__))
5294 "Scalable vector should only have one scalar")(static_cast <bool> ((!VT.isScalableVector() || NumOps ==
1) && "Scalable vector should only have one scalar")
? void (0) : __assert_fail ("(!VT.isScalableVector() || NumOps == 1) && \"Scalable vector should only have one scalar\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5294, __extension__ __PRETTY_FUNCTION__))
;
5295
5296 for (unsigned I = 0; I != NumOps; ++I) {
5297 // We can have a fixed length SPLAT_VECTOR and a BUILD_VECTOR so we need
5298 // to use operand 0 of the SPLAT_VECTOR for each fixed element.
5299 SDValue V1;
5300 if (N1->getOpcode() == ISD::BUILD_VECTOR)
5301 V1 = N1->getOperand(I);
5302 else if (N1->getOpcode() == ISD::SPLAT_VECTOR)
5303 V1 = N1->getOperand(0);
5304 else
5305 V1 = getUNDEF(SVT);
5306
5307 SDValue V2;
5308 if (N2->getOpcode() == ISD::BUILD_VECTOR)
5309 V2 = N2->getOperand(I);
5310 else if (N2->getOpcode() == ISD::SPLAT_VECTOR)
5311 V2 = N2->getOperand(0);
5312 else
5313 V2 = getUNDEF(SVT);
5314
5315 if (SVT.isInteger()) {
5316 if (V1.getValueType().bitsGT(SVT))
5317 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
5318 if (V2.getValueType().bitsGT(SVT))
5319 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
5320 }
5321
5322 if (V1.getValueType() != SVT || V2.getValueType() != SVT)
5323 return SDValue();
5324
5325 // Fold one vector element.
5326 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
5327 if (LegalSVT != SVT)
5328 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5329
5330 // Scalar folding only succeeded if the result is a constant or UNDEF.
5331 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5332 ScalarResult.getOpcode() != ISD::ConstantFP)
5333 return SDValue();
5334 Outputs.push_back(ScalarResult);
5335 }
5336
5337 if (N1->getOpcode() == ISD::BUILD_VECTOR ||
5338 N2->getOpcode() == ISD::BUILD_VECTOR) {
5339 assert(VT.getVectorNumElements() == Outputs.size() &&(static_cast <bool> (VT.getVectorNumElements() == Outputs
.size() && "Vector size mismatch!") ? void (0) : __assert_fail
("VT.getVectorNumElements() == Outputs.size() && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))
5340 "Vector size mismatch!")(static_cast <bool> (VT.getVectorNumElements() == Outputs
.size() && "Vector size mismatch!") ? void (0) : __assert_fail
("VT.getVectorNumElements() == Outputs.size() && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))
;
5341
5342 // Build a big vector out of the scalar elements we generated.
5343 return getBuildVector(VT, SDLoc(), Outputs);
5344 }
5345
5346 assert((N1->getOpcode() == ISD::SPLAT_VECTOR ||(static_cast <bool> ((N1->getOpcode() == ISD::SPLAT_VECTOR
|| N2->getOpcode() == ISD::SPLAT_VECTOR) && "One operand should be a splat vector"
) ? void (0) : __assert_fail ("(N1->getOpcode() == ISD::SPLAT_VECTOR || N2->getOpcode() == ISD::SPLAT_VECTOR) && \"One operand should be a splat vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5348, __extension__ __PRETTY_FUNCTION__))
5347 N2->getOpcode() == ISD::SPLAT_VECTOR) &&(static_cast <bool> ((N1->getOpcode() == ISD::SPLAT_VECTOR
|| N2->getOpcode() == ISD::SPLAT_VECTOR) && "One operand should be a splat vector"
) ? void (0) : __assert_fail ("(N1->getOpcode() == ISD::SPLAT_VECTOR || N2->getOpcode() == ISD::SPLAT_VECTOR) && \"One operand should be a splat vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5348, __extension__ __PRETTY_FUNCTION__))
5348 "One operand should be a splat vector")(static_cast <bool> ((N1->getOpcode() == ISD::SPLAT_VECTOR
|| N2->getOpcode() == ISD::SPLAT_VECTOR) && "One operand should be a splat vector"
) ? void (0) : __assert_fail ("(N1->getOpcode() == ISD::SPLAT_VECTOR || N2->getOpcode() == ISD::SPLAT_VECTOR) && \"One operand should be a splat vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5348, __extension__ __PRETTY_FUNCTION__))
;
5349
5350 assert(Outputs.size() == 1 && "Vector size mismatch!")(static_cast <bool> (Outputs.size() == 1 && "Vector size mismatch!"
) ? void (0) : __assert_fail ("Outputs.size() == 1 && \"Vector size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5350, __extension__ __PRETTY_FUNCTION__))
;
5351 return getSplatVector(VT, SDLoc(), Outputs[0]);
5352}
5353
5354// TODO: Merge with FoldConstantArithmetic
5355SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
5356 const SDLoc &DL, EVT VT,
5357 ArrayRef<SDValue> Ops,
5358 const SDNodeFlags Flags) {
5359 // If the opcode is a target-specific ISD node, there's nothing we can
5360 // do here and the operand rules may not line up with the below, so
5361 // bail early.
5362 if (Opcode >= ISD::BUILTIN_OP_END)
5363 return SDValue();
5364
5365 if (isUndef(Opcode, Ops))
5366 return getUNDEF(VT);
5367
5368 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
5369 if (!VT.isVector())
5370 return SDValue();
5371
5372 ElementCount NumElts = VT.getVectorElementCount();
5373
5374 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
5375 return !Op.getValueType().isVector() ||
5376 Op.getValueType().getVectorElementCount() == NumElts;
5377 };
5378
5379 auto IsConstantBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
5380 APInt SplatVal;
5381 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
5382 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
5383 (BV && BV->isConstant()) ||
5384 (Op.getOpcode() == ISD::SPLAT_VECTOR &&
5385 ISD::isConstantSplatVector(Op.getNode(), SplatVal));
5386 };
5387
5388 // All operands must be vector types with the same number of elements as
5389 // the result type and must be either UNDEF or a build vector of constant
5390 // or UNDEF scalars.
5391 if (!llvm::all_of(Ops, IsConstantBuildVectorSplatVectorOrUndef) ||
5392 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
5393 return SDValue();
5394
5395 // If we are comparing vectors, then the result needs to be a i1 boolean
5396 // that is then sign-extended back to the legal result type.
5397 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
5398
5399 // Find legal integer scalar type for constant promotion and
5400 // ensure that its scalar size is at least as large as source.
5401 EVT LegalSVT = VT.getScalarType();
5402 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5403 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5404 if (LegalSVT.bitsLT(VT.getScalarType()))
5405 return SDValue();
5406 }
5407
5408 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
5409 // only have one operand to check. For fixed-length vector types we may have
5410 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
5411 unsigned NumOperands = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
5412
5413 // Constant fold each scalar lane separately.
5414 SmallVector<SDValue, 4> ScalarResults;
5415 for (unsigned I = 0; I != NumOperands; I++) {
5416 SmallVector<SDValue, 4> ScalarOps;
5417 for (SDValue Op : Ops) {
5418 EVT InSVT = Op.getValueType().getScalarType();
5419 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
5420 Op.getOpcode() != ISD::SPLAT_VECTOR) {
5421 // We've checked that this is UNDEF or a constant of some kind.
5422 if (Op.isUndef())
5423 ScalarOps.push_back(getUNDEF(InSVT));
5424 else
5425 ScalarOps.push_back(Op);
5426 continue;
5427 }
5428
5429 SDValue ScalarOp =
5430 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
5431 EVT ScalarVT = ScalarOp.getValueType();
5432
5433 // Build vector (integer) scalar operands may need implicit
5434 // truncation - do this before constant folding.
5435 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
5436 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
5437
5438 ScalarOps.push_back(ScalarOp);
5439 }
5440
5441 // Constant fold the scalar operands.
5442 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
5443
5444 // Legalize the (integer) scalar constant if necessary.
5445 if (LegalSVT != SVT)
5446 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5447
5448 // Scalar folding only succeeded if the result is a constant or UNDEF.
5449 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5450 ScalarResult.getOpcode() != ISD::ConstantFP)
5451 return SDValue();
5452 ScalarResults.push_back(ScalarResult);
5453 }
5454
5455 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
5456 : getBuildVector(VT, DL, ScalarResults);
5457 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
5458 return V;
5459}
5460
5461SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
5462 EVT VT, SDValue N1, SDValue N2) {
5463 // TODO: We don't do any constant folding for strict FP opcodes here, but we
5464 // should. That will require dealing with a potentially non-default
5465 // rounding mode, checking the "opStatus" return value from the APFloat
5466 // math calculations, and possibly other variations.
5467 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
5468 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
5469 if (N1CFP && N2CFP) {
5470 APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
5471 switch (Opcode) {
5472 case ISD::FADD:
5473 C1.add(C2, APFloat::rmNearestTiesToEven);
5474 return getConstantFP(C1, DL, VT);
5475 case ISD::FSUB:
5476 C1.subtract(C2, APFloat::rmNearestTiesToEven);
5477 return getConstantFP(C1, DL, VT);
5478 case ISD::FMUL:
5479 C1.multiply(C2, APFloat::rmNearestTiesToEven);
5480 return getConstantFP(C1, DL, VT);
5481 case ISD::FDIV:
5482 C1.divide(C2, APFloat::rmNearestTiesToEven);
5483 return getConstantFP(C1, DL, VT);
5484 case ISD::FREM:
5485 C1.mod(C2);
5486 return getConstantFP(C1, DL, VT);
5487 case ISD::FCOPYSIGN:
5488 C1.copySign(C2);
5489 return getConstantFP(C1, DL, VT);
5490 default: break;
5491 }
5492 }
5493 if (N1CFP && Opcode == ISD::FP_ROUND) {
5494 APFloat C1 = N1CFP->getValueAPF(); // make copy
5495 bool Unused;
5496 // This can return overflow, underflow, or inexact; we don't care.
5497 // FIXME need to be more flexible about rounding mode.
5498 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
5499 &Unused);
5500 return getConstantFP(C1, DL, VT);
5501 }
5502
5503 switch (Opcode) {
5504 case ISD::FSUB:
5505 // -0.0 - undef --> undef (consistent with "fneg undef")
5506 if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef())
5507 return getUNDEF(VT);
5508 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5509
5510 case ISD::FADD:
5511 case ISD::FMUL:
5512 case ISD::FDIV:
5513 case ISD::FREM:
5514 // If both operands are undef, the result is undef. If 1 operand is undef,
5515 // the result is NaN. This should match the behavior of the IR optimizer.
5516 if (N1.isUndef() && N2.isUndef())
5517 return getUNDEF(VT);
5518 if (N1.isUndef() || N2.isUndef())
5519 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
5520 }
5521 return SDValue();
5522}
5523
5524SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
5525 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!")(static_cast <bool> (Val.getValueType().isInteger() &&
"Invalid AssertAlign!") ? void (0) : __assert_fail ("Val.getValueType().isInteger() && \"Invalid AssertAlign!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5525, __extension__ __PRETTY_FUNCTION__))
;
5526
5527 // There's no need to assert on a byte-aligned pointer. All pointers are at
5528 // least byte aligned.
5529 if (A == Align(1))
5530 return Val;
5531
5532 FoldingSetNodeID ID;
5533 AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val});
5534 ID.AddInteger(A.value());
5535
5536 void *IP = nullptr;
5537 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5538 return SDValue(E, 0);
5539
5540 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
5541 Val.getValueType(), A);
5542 createOperands(N, {Val});
5543
5544 CSEMap.InsertNode(N, IP);
5545 InsertNode(N);
5546
5547 SDValue V(N, 0);
5548 NewSDValueDbgMsg(V, "Creating new node: ", this);
5549 return V;
5550}
5551
5552SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5553 SDValue N1, SDValue N2) {
5554 SDNodeFlags Flags;
5555 if (Inserter)
5556 Flags = Inserter->getFlags();
5557 return getNode(Opcode, DL, VT, N1, N2, Flags);
5558}
5559
5560SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5561 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5562 assert(N1.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5564, __extension__ __PRETTY_FUNCTION__))
5563 N2.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5564, __extension__ __PRETTY_FUNCTION__))
5564 "Operand is DELETED_NODE!")(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5564, __extension__ __PRETTY_FUNCTION__))
;
5565 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5566 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
5567 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5568 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5569
5570 // Canonicalize constant to RHS if commutative.
5571 if (TLI->isCommutativeBinOp(Opcode)) {
5572 if (N1C && !N2C) {
5573 std::swap(N1C, N2C);
5574 std::swap(N1, N2);
5575 } else if (N1CFP && !N2CFP) {
5576 std::swap(N1CFP, N2CFP);
5577 std::swap(N1, N2);
5578 }
5579 }
5580
5581 switch (Opcode) {
5582 default: break;
5583 case ISD::TokenFactor:
5584 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&(static_cast <bool> (VT == MVT::Other && N1.getValueType
() == MVT::Other && N2.getValueType() == MVT::Other &&
"Invalid token factor!") ? void (0) : __assert_fail ("VT == MVT::Other && N1.getValueType() == MVT::Other && N2.getValueType() == MVT::Other && \"Invalid token factor!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5585, __extension__ __PRETTY_FUNCTION__))
5585 N2.getValueType() == MVT::Other && "Invalid token factor!")(static_cast <bool> (VT == MVT::Other && N1.getValueType
() == MVT::Other && N2.getValueType() == MVT::Other &&
"Invalid token factor!") ? void (0) : __assert_fail ("VT == MVT::Other && N1.getValueType() == MVT::Other && N2.getValueType() == MVT::Other && \"Invalid token factor!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5585, __extension__ __PRETTY_FUNCTION__))
;
5586 // Fold trivial token factors.
5587 if (N1.getOpcode() == ISD::EntryToken) return N2;
5588 if (N2.getOpcode() == ISD::EntryToken) return N1;
5589 if (N1 == N2) return N1;
5590 break;
5591 case ISD::BUILD_VECTOR: {
5592 // Attempt to simplify BUILD_VECTOR.
5593 SDValue Ops[] = {N1, N2};
5594 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5595 return V;
5596 break;
5597 }
5598 case ISD::CONCAT_VECTORS: {
5599 SDValue Ops[] = {N1, N2};
5600 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5601 return V;
5602 break;
5603 }
5604 case ISD::AND:
5605 assert(VT.isInteger() && "This operator does not apply to FP types!")(static_cast <bool> (VT.isInteger() && "This operator does not apply to FP types!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5605, __extension__ __PRETTY_FUNCTION__))
;
5606 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5607, __extension__ __PRETTY_FUNCTION__))
5607 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5607, __extension__ __PRETTY_FUNCTION__))
;
5608 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
5609 // worth handling here.
5610 if (N2C && N2C->isNullValue())
5611 return N2;
5612 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
5613 return N1;
5614 break;
5615 case ISD::OR:
5616 case ISD::XOR:
5617 case ISD::ADD:
5618 case ISD::SUB:
5619 assert(VT.isInteger() && "This operator does not apply to FP types!")(static_cast <bool> (VT.isInteger() && "This operator does not apply to FP types!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5619, __extension__ __PRETTY_FUNCTION__))
;
5620 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5621, __extension__ __PRETTY_FUNCTION__))
5621 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5621, __extension__ __PRETTY_FUNCTION__))
;
5622 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
5623 // it's worth handling here.
5624 if (N2C && N2C->isNullValue())
5625 return N1;
5626 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
5627 VT.getVectorElementType() == MVT::i1)
5628 return getNode(ISD::XOR, DL, VT, N1, N2);
5629 break;
5630 case ISD::MUL:
5631 assert(VT.isInteger() && "This operator does not apply to FP types!")(static_cast <bool> (VT.isInteger() && "This operator does not apply to FP types!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5631, __extension__ __PRETTY_FUNCTION__))
;
5632 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5633, __extension__ __PRETTY_FUNCTION__))
5633 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5633, __extension__ __PRETTY_FUNCTION__))
;
5634 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5635 return getNode(ISD::AND, DL, VT, N1, N2);
5636 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5637 const APInt &MulImm = N1->getConstantOperandAPInt(0);
5638 const APInt &N2CImm = N2C->getAPIntValue();
5639 return getVScale(DL, VT, MulImm * N2CImm);
5640 }
5641 break;
5642 case ISD::UDIV:
5643 case ISD::UREM:
5644 case ISD::MULHU:
5645 case ISD::MULHS:
5646 case ISD::SDIV:
5647 case ISD::SREM:
5648 case ISD::SADDSAT:
5649 case ISD::SSUBSAT:
5650 case ISD::UADDSAT:
5651 case ISD::USUBSAT:
5652 assert(VT.isInteger() && "This operator does not apply to FP types!")(static_cast <bool> (VT.isInteger() && "This operator does not apply to FP types!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5652, __extension__ __PRETTY_FUNCTION__))
;
5653 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5654, __extension__ __PRETTY_FUNCTION__))
5654 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5654, __extension__ __PRETTY_FUNCTION__))
;
5655 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
5656 // fold (add_sat x, y) -> (or x, y) for bool types.
5657 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
5658 return getNode(ISD::OR, DL, VT, N1, N2);
5659 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
5660 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
5661 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
5662 }
5663 break;
5664 case ISD::SMIN:
5665 case ISD::UMAX:
5666 assert(VT.isInteger() && "This operator does not apply to FP types!")(static_cast <bool> (VT.isInteger() && "This operator does not apply to FP types!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5666, __extension__ __PRETTY_FUNCTION__))
;
5667 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5668, __extension__ __PRETTY_FUNCTION__))
5668 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5668, __extension__ __PRETTY_FUNCTION__))
;
5669 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5670 return getNode(ISD::OR, DL, VT, N1, N2);
5671 break;
5672 case ISD::SMAX:
5673 case ISD::UMIN:
5674 assert(VT.isInteger() && "This operator does not apply to FP types!")(static_cast <bool> (VT.isInteger() && "This operator does not apply to FP types!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"This operator does not apply to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5674, __extension__ __PRETTY_FUNCTION__))
;
5675 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5676, __extension__ __PRETTY_FUNCTION__))
5676 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5676, __extension__ __PRETTY_FUNCTION__))
;
5677 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5678 return getNode(ISD::AND, DL, VT, N1, N2);
5679 break;
5680 case ISD::FADD:
5681 case ISD::FSUB:
5682 case ISD::FMUL:
5683 case ISD::FDIV:
5684 case ISD::FREM:
5685 assert(VT.isFloatingPoint() && "This operator only applies to FP types!")(static_cast <bool> (VT.isFloatingPoint() && "This operator only applies to FP types!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"This operator only applies to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5685, __extension__ __PRETTY_FUNCTION__))
;
5686 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5687, __extension__ __PRETTY_FUNCTION__))
5687 N1.getValueType() == VT && "Binary operator types must match!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && N1.getValueType() == VT && "Binary operator types must match!"
) ? void (0) : __assert_fail ("N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && \"Binary operator types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5687, __extension__ __PRETTY_FUNCTION__))
;
5688 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
5689 return V;
5690 break;
5691 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
5692 assert(N1.getValueType() == VT &&(static_cast <bool> (N1.getValueType() == VT &&
N1.getValueType().isFloatingPoint() && N2.getValueType
().isFloatingPoint() && "Invalid FCOPYSIGN!") ? void (
0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5695, __extension__ __PRETTY_FUNCTION__))
5693 N1.getValueType().isFloatingPoint() &&(static_cast <bool> (N1.getValueType() == VT &&
N1.getValueType().isFloatingPoint() && N2.getValueType
().isFloatingPoint() && "Invalid FCOPYSIGN!") ? void (
0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5695, __extension__ __PRETTY_FUNCTION__))
5694 N2.getValueType().isFloatingPoint() &&(static_cast <bool> (N1.getValueType() == VT &&
N1.getValueType().isFloatingPoint() && N2.getValueType
().isFloatingPoint() && "Invalid FCOPYSIGN!") ? void (
0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5695, __extension__ __PRETTY_FUNCTION__))
5695 "Invalid FCOPYSIGN!")(static_cast <bool> (N1.getValueType() == VT &&
N1.getValueType().isFloatingPoint() && N2.getValueType
().isFloatingPoint() && "Invalid FCOPYSIGN!") ? void (
0) : __assert_fail ("N1.getValueType() == VT && N1.getValueType().isFloatingPoint() && N2.getValueType().isFloatingPoint() && \"Invalid FCOPYSIGN!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5695, __extension__ __PRETTY_FUNCTION__))
;
5696 break;
5697 case ISD::SHL:
5698 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5699 const APInt &MulImm = N1->getConstantOperandAPInt(0);
5700 const APInt &ShiftImm = N2C->getAPIntValue();
5701 return getVScale(DL, VT, MulImm << ShiftImm);
5702 }
5703 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5704 case ISD::SRA:
5705 case ISD::SRL:
5706 if (SDValue V = simplifyShift(N1, N2))
5707 return V;
5708 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5709 case ISD::ROTL:
5710 case ISD::ROTR:
5711 assert(VT == N1.getValueType() &&(static_cast <bool> (VT == N1.getValueType() &&
"Shift operators return type must be the same as their first arg"
) ? void (0) : __assert_fail ("VT == N1.getValueType() && \"Shift operators return type must be the same as their first arg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5712, __extension__ __PRETTY_FUNCTION__))
5712 "Shift operators return type must be the same as their first arg")(static_cast <bool> (VT == N1.getValueType() &&
"Shift operators return type must be the same as their first arg"
) ? void (0) : __assert_fail ("VT == N1.getValueType() && \"Shift operators return type must be the same as their first arg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5712, __extension__ __PRETTY_FUNCTION__))
;
5713 assert(VT.isInteger() && N2.getValueType().isInteger() &&(static_cast <bool> (VT.isInteger() && N2.getValueType
().isInteger() && "Shifts only work on integers") ? void
(0) : __assert_fail ("VT.isInteger() && N2.getValueType().isInteger() && \"Shifts only work on integers\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5714, __extension__ __PRETTY_FUNCTION__))
5714 "Shifts only work on integers")(static_cast <bool> (VT.isInteger() && N2.getValueType
().isInteger() && "Shifts only work on integers") ? void
(0) : __assert_fail ("VT.isInteger() && N2.getValueType().isInteger() && \"Shifts only work on integers\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5714, __extension__ __PRETTY_FUNCTION__))
;
5715 assert((!VT.isVector() || VT == N2.getValueType()) &&(static_cast <bool> ((!VT.isVector() || VT == N2.getValueType
()) && "Vector shift amounts must be in the same as their first arg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT == N2.getValueType()) && \"Vector shift amounts must be in the same as their first arg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5716, __extension__ __PRETTY_FUNCTION__))
5716 "Vector shift amounts must be in the same as their first arg")(static_cast <bool> ((!VT.isVector() || VT == N2.getValueType
()) && "Vector shift amounts must be in the same as their first arg"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT == N2.getValueType()) && \"Vector shift amounts must be in the same as their first arg\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5716, __extension__ __PRETTY_FUNCTION__))
;
5717 // Verify that the shift amount VT is big enough to hold valid shift
5718 // amounts. This catches things like trying to shift an i1024 value by an
5719 // i8, which is easy to fall into in generic code that uses
5720 // TLI.getShiftAmount().
5721 assert(N2.getValueType().getScalarSizeInBits() >=(static_cast <bool> (N2.getValueType().getScalarSizeInBits
() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && "Invalid use of small shift amount with oversized value!"
) ? void (0) : __assert_fail ("N2.getValueType().getScalarSizeInBits() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && \"Invalid use of small shift amount with oversized value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5723, __extension__ __PRETTY_FUNCTION__))
5722 Log2_32_Ceil(VT.getScalarSizeInBits()) &&(static_cast <bool> (N2.getValueType().getScalarSizeInBits
() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && "Invalid use of small shift amount with oversized value!"
) ? void (0) : __assert_fail ("N2.getValueType().getScalarSizeInBits() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && \"Invalid use of small shift amount with oversized value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5723, __extension__ __PRETTY_FUNCTION__))
5723 "Invalid use of small shift amount with oversized value!")(static_cast <bool> (N2.getValueType().getScalarSizeInBits
() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && "Invalid use of small shift amount with oversized value!"
) ? void (0) : __assert_fail ("N2.getValueType().getScalarSizeInBits() >= Log2_32_Ceil(VT.getScalarSizeInBits()) && \"Invalid use of small shift amount with oversized value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5723, __extension__ __PRETTY_FUNCTION__))
;
5724
5725 // Always fold shifts of i1 values so the code generator doesn't need to
5726 // handle them. Since we know the size of the shift has to be less than the
5727 // size of the value, the shift/rotate count is guaranteed to be zero.
5728 if (VT == MVT::i1)
5729 return N1;
5730 if (N2C && N2C->isNullValue())
5731 return N1;
5732 break;
5733 case ISD::FP_ROUND:
5734 assert(VT.isFloatingPoint() &&(static_cast <bool> (VT.isFloatingPoint() && N1
.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType
()) && N2C && (N2C->getZExtValue() == 0 ||
N2C->getZExtValue() == 1) && "Invalid FP_ROUND!")
? void (0) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5738, __extension__ __PRETTY_FUNCTION__))
5735 N1.getValueType().isFloatingPoint() &&(static_cast <bool> (VT.isFloatingPoint() && N1
.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType
()) && N2C && (N2C->getZExtValue() == 0 ||
N2C->getZExtValue() == 1) && "Invalid FP_ROUND!")
? void (0) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5738, __extension__ __PRETTY_FUNCTION__))
5736 VT.bitsLE(N1.getValueType()) &&(static_cast <bool> (VT.isFloatingPoint() && N1
.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType
()) && N2C && (N2C->getZExtValue() == 0 ||
N2C->getZExtValue() == 1) && "Invalid FP_ROUND!")
? void (0) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5738, __extension__ __PRETTY_FUNCTION__))
5737 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&(static_cast <bool> (VT.isFloatingPoint() && N1
.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType
()) && N2C && (N2C->getZExtValue() == 0 ||
N2C->getZExtValue() == 1) && "Invalid FP_ROUND!")
? void (0) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5738, __extension__ __PRETTY_FUNCTION__))
5738 "Invalid FP_ROUND!")(static_cast <bool> (VT.isFloatingPoint() && N1
.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType
()) && N2C && (N2C->getZExtValue() == 0 ||
N2C->getZExtValue() == 1) && "Invalid FP_ROUND!")
? void (0) : __assert_fail ("VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && VT.bitsLE(N1.getValueType()) && N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && \"Invalid FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5738, __extension__ __PRETTY_FUNCTION__))
;
5739 if (N1.getValueType() == VT) return N1; // noop conversion.
5740 break;
5741 case ISD::AssertSext:
5742 case ISD::AssertZext: {
5743 EVT EVT = cast<VTSDNode>(N2)->getVT();
5744 assert(VT == N1.getValueType() && "Not an inreg extend!")(static_cast <bool> (VT == N1.getValueType() &&
"Not an inreg extend!") ? void (0) : __assert_fail ("VT == N1.getValueType() && \"Not an inreg extend!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5744, __extension__ __PRETTY_FUNCTION__))
;
5745 assert(VT.isInteger() && EVT.isInteger() &&(static_cast <bool> (VT.isInteger() && EVT.isInteger
() && "Cannot *_EXTEND_INREG FP types") ? void (0) : __assert_fail
("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5746, __extension__ __PRETTY_FUNCTION__))
5746 "Cannot *_EXTEND_INREG FP types")(static_cast <bool> (VT.isInteger() && EVT.isInteger
() && "Cannot *_EXTEND_INREG FP types") ? void (0) : __assert_fail
("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5746, __extension__ __PRETTY_FUNCTION__))
;
5747 assert(!EVT.isVector() &&(static_cast <bool> (!EVT.isVector() && "AssertSExt/AssertZExt type should be the vector element type "
"rather than the vector type!") ? void (0) : __assert_fail (
"!EVT.isVector() && \"AssertSExt/AssertZExt type should be the vector element type \" \"rather than the vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__))
5748 "AssertSExt/AssertZExt type should be the vector element type "(static_cast <bool> (!EVT.isVector() && "AssertSExt/AssertZExt type should be the vector element type "
"rather than the vector type!") ? void (0) : __assert_fail (
"!EVT.isVector() && \"AssertSExt/AssertZExt type should be the vector element type \" \"rather than the vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__))
5749 "rather than the vector type!")(static_cast <bool> (!EVT.isVector() && "AssertSExt/AssertZExt type should be the vector element type "
"rather than the vector type!") ? void (0) : __assert_fail (
"!EVT.isVector() && \"AssertSExt/AssertZExt type should be the vector element type \" \"rather than the vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__))
;
5750 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!")(static_cast <bool> (EVT.bitsLE(VT.getScalarType()) &&
"Not extending!") ? void (0) : __assert_fail ("EVT.bitsLE(VT.getScalarType()) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5750, __extension__ __PRETTY_FUNCTION__))
;
5751 if (VT.getScalarType() == EVT) return N1; // noop assertion.
5752 break;
5753 }
5754 case ISD::SIGN_EXTEND_INREG: {
5755 EVT EVT = cast<VTSDNode>(N2)->getVT();
5756 assert(VT == N1.getValueType() && "Not an inreg extend!")(static_cast <bool> (VT == N1.getValueType() &&
"Not an inreg extend!") ? void (0) : __assert_fail ("VT == N1.getValueType() && \"Not an inreg extend!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5756, __extension__ __PRETTY_FUNCTION__))
;
5757 assert(VT.isInteger() && EVT.isInteger() &&(static_cast <bool> (VT.isInteger() && EVT.isInteger
() && "Cannot *_EXTEND_INREG FP types") ? void (0) : __assert_fail
("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5758, __extension__ __PRETTY_FUNCTION__))
5758 "Cannot *_EXTEND_INREG FP types")(static_cast <bool> (VT.isInteger() && EVT.isInteger
() && "Cannot *_EXTEND_INREG FP types") ? void (0) : __assert_fail
("VT.isInteger() && EVT.isInteger() && \"Cannot *_EXTEND_INREG FP types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5758, __extension__ __PRETTY_FUNCTION__))
;
5759 assert(EVT.isVector() == VT.isVector() &&(static_cast <bool> (EVT.isVector() == VT.isVector() &&
"SIGN_EXTEND_INREG type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("EVT.isVector() == VT.isVector() && \"SIGN_EXTEND_INREG type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5761, __extension__ __PRETTY_FUNCTION__))
5760 "SIGN_EXTEND_INREG type should be vector iff the operand "(static_cast <bool> (EVT.isVector() == VT.isVector() &&
"SIGN_EXTEND_INREG type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("EVT.isVector() == VT.isVector() && \"SIGN_EXTEND_INREG type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5761, __extension__ __PRETTY_FUNCTION__))
5761 "type is vector!")(static_cast <bool> (EVT.isVector() == VT.isVector() &&
"SIGN_EXTEND_INREG type should be vector iff the operand " "type is vector!"
) ? void (0) : __assert_fail ("EVT.isVector() == VT.isVector() && \"SIGN_EXTEND_INREG type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5761, __extension__ __PRETTY_FUNCTION__))
;
5762 assert((!EVT.isVector() ||(static_cast <bool> ((!EVT.isVector() || EVT.getVectorElementCount
() == VT.getVectorElementCount()) && "Vector element counts must match in SIGN_EXTEND_INREG"
) ? void (0) : __assert_fail ("(!EVT.isVector() || EVT.getVectorElementCount() == VT.getVectorElementCount()) && \"Vector element counts must match in SIGN_EXTEND_INREG\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5764, __extension__ __PRETTY_FUNCTION__))
5763 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&(static_cast <bool> ((!EVT.isVector() || EVT.getVectorElementCount
() == VT.getVectorElementCount()) && "Vector element counts must match in SIGN_EXTEND_INREG"
) ? void (0) : __assert_fail ("(!EVT.isVector() || EVT.getVectorElementCount() == VT.getVectorElementCount()) && \"Vector element counts must match in SIGN_EXTEND_INREG\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5764, __extension__ __PRETTY_FUNCTION__))
5764 "Vector element counts must match in SIGN_EXTEND_INREG")(static_cast <bool> ((!EVT.isVector() || EVT.getVectorElementCount
() == VT.getVectorElementCount()) && "Vector element counts must match in SIGN_EXTEND_INREG"
) ? void (0) : __assert_fail ("(!EVT.isVector() || EVT.getVectorElementCount() == VT.getVectorElementCount()) && \"Vector element counts must match in SIGN_EXTEND_INREG\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5764, __extension__ __PRETTY_FUNCTION__))
;
5765 assert(EVT.bitsLE(VT) && "Not extending!")(static_cast <bool> (EVT.bitsLE(VT) && "Not extending!"
) ? void (0) : __assert_fail ("EVT.bitsLE(VT) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5765, __extension__ __PRETTY_FUNCTION__))
;
5766 if (EVT == VT) return N1; // Not actually extending
5767
5768 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
5769 unsigned FromBits = EVT.getScalarSizeInBits();
5770 Val <<= Val.getBitWidth() - FromBits;
5771 Val.ashrInPlace(Val.getBitWidth() - FromBits);
5772 return getConstant(Val, DL, ConstantVT);
5773 };
5774
5775 if (N1C) {
5776 const APInt &Val = N1C->getAPIntValue();
5777 return SignExtendInReg(Val, VT);
5778 }
5779
5780 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
5781 SmallVector<SDValue, 8> Ops;
5782 llvm::EVT OpVT = N1.getOperand(0).getValueType();
5783 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
5784 SDValue Op = N1.getOperand(i);
5785 if (Op.isUndef()) {
5786 Ops.push_back(getUNDEF(OpVT));
5787 continue;
5788 }
5789 ConstantSDNode *C = cast<ConstantSDNode>(Op);
5790 APInt Val = C->getAPIntValue();
5791 Ops.push_back(SignExtendInReg(Val, OpVT));
5792 }
5793 return getBuildVector(VT, DL, Ops);
5794 }
5795 break;
5796 }
5797 case ISD::FP_TO_SINT_SAT:
5798 case ISD::FP_TO_UINT_SAT: {
5799 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&(static_cast <bool> (VT.isInteger() && cast<
VTSDNode>(N2)->getVT().isInteger() && N1.getValueType
().isFloatingPoint() && "Invalid FP_TO_*INT_SAT") ? void
(0) : __assert_fail ("VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() && N1.getValueType().isFloatingPoint() && \"Invalid FP_TO_*INT_SAT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5800, __extension__ __PRETTY_FUNCTION__))
5800 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT")(static_cast <bool> (VT.isInteger() && cast<
VTSDNode>(N2)->getVT().isInteger() && N1.getValueType
().isFloatingPoint() && "Invalid FP_TO_*INT_SAT") ? void
(0) : __assert_fail ("VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() && N1.getValueType().isFloatingPoint() && \"Invalid FP_TO_*INT_SAT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5800, __extension__ __PRETTY_FUNCTION__))
;
5801 assert(N1.getValueType().isVector() == VT.isVector() &&(static_cast <bool> (N1.getValueType().isVector() == VT
.isVector() && "FP_TO_*INT_SAT type should be vector iff the operand type is "
"vector!") ? void (0) : __assert_fail ("N1.getValueType().isVector() == VT.isVector() && \"FP_TO_*INT_SAT type should be vector iff the operand type is \" \"vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5803, __extension__ __PRETTY_FUNCTION__))
5802 "FP_TO_*INT_SAT type should be vector iff the operand type is "(static_cast <bool> (N1.getValueType().isVector() == VT
.isVector() && "FP_TO_*INT_SAT type should be vector iff the operand type is "
"vector!") ? void (0) : __assert_fail ("N1.getValueType().isVector() == VT.isVector() && \"FP_TO_*INT_SAT type should be vector iff the operand type is \" \"vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5803, __extension__ __PRETTY_FUNCTION__))
5803 "vector!")(static_cast <bool> (N1.getValueType().isVector() == VT
.isVector() && "FP_TO_*INT_SAT type should be vector iff the operand type is "
"vector!") ? void (0) : __assert_fail ("N1.getValueType().isVector() == VT.isVector() && \"FP_TO_*INT_SAT type should be vector iff the operand type is \" \"vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5803, __extension__ __PRETTY_FUNCTION__))
;
5804 assert((!VT.isVector() || VT.getVectorNumElements() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorNumElements
() == N1.getValueType().getVectorNumElements()) && "Vector element counts must match in FP_TO_*INT_SAT"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == N1.getValueType().getVectorNumElements()) && \"Vector element counts must match in FP_TO_*INT_SAT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5806, __extension__ __PRETTY_FUNCTION__))
5805 N1.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorNumElements
() == N1.getValueType().getVectorNumElements()) && "Vector element counts must match in FP_TO_*INT_SAT"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == N1.getValueType().getVectorNumElements()) && \"Vector element counts must match in FP_TO_*INT_SAT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5806, __extension__ __PRETTY_FUNCTION__))
5806 "Vector element counts must match in FP_TO_*INT_SAT")(static_cast <bool> ((!VT.isVector() || VT.getVectorNumElements
() == N1.getValueType().getVectorNumElements()) && "Vector element counts must match in FP_TO_*INT_SAT"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorNumElements() == N1.getValueType().getVectorNumElements()) && \"Vector element counts must match in FP_TO_*INT_SAT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5806, __extension__ __PRETTY_FUNCTION__))
;
5807 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&(static_cast <bool> (!cast<VTSDNode>(N2)->getVT
().isVector() && "Type to saturate to must be a scalar."
) ? void (0) : __assert_fail ("!cast<VTSDNode>(N2)->getVT().isVector() && \"Type to saturate to must be a scalar.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5808, __extension__ __PRETTY_FUNCTION__))
5808 "Type to saturate to must be a scalar.")(static_cast <bool> (!cast<VTSDNode>(N2)->getVT
().isVector() && "Type to saturate to must be a scalar."
) ? void (0) : __assert_fail ("!cast<VTSDNode>(N2)->getVT().isVector() && \"Type to saturate to must be a scalar.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5808, __extension__ __PRETTY_FUNCTION__))
;
5809 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&(static_cast <bool> (cast<VTSDNode>(N2)->getVT
().bitsLE(VT.getScalarType()) && "Not extending!") ? void
(0) : __assert_fail ("cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5810, __extension__ __PRETTY_FUNCTION__))
5810 "Not extending!")(static_cast <bool> (cast<VTSDNode>(N2)->getVT
().bitsLE(VT.getScalarType()) && "Not extending!") ? void
(0) : __assert_fail ("cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) && \"Not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5810, __extension__ __PRETTY_FUNCTION__))
;
5811 break;
5812 }
5813 case ISD::EXTRACT_VECTOR_ELT:
5814 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&(static_cast <bool> (VT.getSizeInBits() >= N1.getValueType
().getScalarSizeInBits() && "The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector."
) ? void (0) : __assert_fail ("VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && \"The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5816, __extension__ __PRETTY_FUNCTION__))
5815 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \(static_cast <bool> (VT.getSizeInBits() >= N1.getValueType
().getScalarSizeInBits() && "The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector."
) ? void (0) : __assert_fail ("VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && \"The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5816, __extension__ __PRETTY_FUNCTION__))
5816 element type of the vector.")(static_cast <bool> (VT.getSizeInBits() >= N1.getValueType
().getScalarSizeInBits() && "The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector."
) ? void (0) : __assert_fail ("VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && \"The result of EXTRACT_VECTOR_ELT must be at least as wide as the element type of the vector.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5816, __extension__ __PRETTY_FUNCTION__))
;
5817
5818 // Extract from an undefined value or using an undefined index is undefined.
5819 if (N1.isUndef() || N2.isUndef())
5820 return getUNDEF(VT);
5821
5822 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
5823 // vectors. For scalable vectors we will provide appropriate support for
5824 // dealing with arbitrary indices.
5825 if (N2C && N1.getValueType().isFixedLengthVector() &&
5826 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
5827 return getUNDEF(VT);
5828
5829 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
5830 // expanding copies of large vectors from registers. This only works for
5831 // fixed length vectors, since we need to know the exact number of
5832 // elements.
5833 if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() &&
5834 N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) {
5835 unsigned Factor =
5836 N1.getOperand(0).getValueType().getVectorNumElements();
5837 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
5838 N1.getOperand(N2C->getZExtValue() / Factor),
5839 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
5840 }
5841
5842 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
5843 // lowering is expanding large vector constants.
5844 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
5845 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
5846 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||(static_cast <bool> ((N1.getOpcode() != ISD::BUILD_VECTOR
|| N1.getValueType().isFixedLengthVector()) && "BUILD_VECTOR used for scalable vectors"
) ? void (0) : __assert_fail ("(N1.getOpcode() != ISD::BUILD_VECTOR || N1.getValueType().isFixedLengthVector()) && \"BUILD_VECTOR used for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5848, __extension__ __PRETTY_FUNCTION__))
5847 N1.getValueType().isFixedLengthVector()) &&(static_cast <bool> ((N1.getOpcode() != ISD::BUILD_VECTOR
|| N1.getValueType().isFixedLengthVector()) && "BUILD_VECTOR used for scalable vectors"
) ? void (0) : __assert_fail ("(N1.getOpcode() != ISD::BUILD_VECTOR || N1.getValueType().isFixedLengthVector()) && \"BUILD_VECTOR used for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5848, __extension__ __PRETTY_FUNCTION__))
5848 "BUILD_VECTOR used for scalable vectors")(static_cast <bool> ((N1.getOpcode() != ISD::BUILD_VECTOR
|| N1.getValueType().isFixedLengthVector()) && "BUILD_VECTOR used for scalable vectors"
) ? void (0) : __assert_fail ("(N1.getOpcode() != ISD::BUILD_VECTOR || N1.getValueType().isFixedLengthVector()) && \"BUILD_VECTOR used for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5848, __extension__ __PRETTY_FUNCTION__))
;
5849 unsigned Index =
5850 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
5851 SDValue Elt = N1.getOperand(Index);
5852
5853 if (VT != Elt.getValueType())
5854 // If the vector element type is not legal, the BUILD_VECTOR operands
5855 // are promoted and implicitly truncated, and the result implicitly
5856 // extended. Make that explicit here.
5857 Elt = getAnyExtOrTrunc(Elt, DL, VT);
5858
5859 return Elt;
5860 }
5861
5862 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
5863 // operations are lowered to scalars.
5864 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
5865 // If the indices are the same, return the inserted element else
5866 // if the indices are known different, extract the element from
5867 // the original vector.
5868 SDValue N1Op2 = N1.getOperand(2);
5869 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
5870
5871 if (N1Op2C && N2C) {
5872 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
5873 if (VT == N1.getOperand(1).getValueType())
5874 return N1.getOperand(1);
5875 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
5876 }
5877 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
5878 }
5879 }
5880
5881 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
5882 // when vector types are scalarized and v1iX is legal.
5883 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
5884 // Here we are completely ignoring the extract element index (N2),
5885 // which is fine for fixed width vectors, since any index other than 0
5886 // is undefined anyway. However, this cannot be ignored for scalable
5887 // vectors - in theory we could support this, but we don't want to do this
5888 // without a profitability check.
5889 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5890 N1.getValueType().isFixedLengthVector() &&
5891 N1.getValueType().getVectorNumElements() == 1) {
5892 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
5893 N1.getOperand(1));
5894 }
5895 break;
5896 case ISD::EXTRACT_ELEMENT:
5897 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!")(static_cast <bool> (N2C && (unsigned)N2C->getZExtValue
() < 2 && "Bad EXTRACT_ELEMENT!") ? void (0) : __assert_fail
("N2C && (unsigned)N2C->getZExtValue() < 2 && \"Bad EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5897, __extension__ __PRETTY_FUNCTION__))
;
5898 assert(!N1.getValueType().isVector() && !VT.isVector() &&(static_cast <bool> (!N1.getValueType().isVector() &&
!VT.isVector() && (N1.getValueType().isInteger() == VT
.isInteger()) && N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? void (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5901, __extension__ __PRETTY_FUNCTION__))
5899 (N1.getValueType().isInteger() == VT.isInteger()) &&(static_cast <bool> (!N1.getValueType().isVector() &&
!VT.isVector() && (N1.getValueType().isInteger() == VT
.isInteger()) && N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? void (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5901, __extension__ __PRETTY_FUNCTION__))
5900 N1.getValueType() != VT &&(static_cast <bool> (!N1.getValueType().isVector() &&
!VT.isVector() && (N1.getValueType().isInteger() == VT
.isInteger()) && N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? void (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5901, __extension__ __PRETTY_FUNCTION__))
5901 "Wrong types for EXTRACT_ELEMENT!")(static_cast <bool> (!N1.getValueType().isVector() &&
!VT.isVector() && (N1.getValueType().isInteger() == VT
.isInteger()) && N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"
) ? void (0) : __assert_fail ("!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && N1.getValueType() != VT && \"Wrong types for EXTRACT_ELEMENT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5901, __extension__ __PRETTY_FUNCTION__))
;
5902
5903 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
5904 // 64-bit integers into 32-bit parts. Instead of building the extract of
5905 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
5906 if (N1.getOpcode() == ISD::BUILD_PAIR)
5907 return N1.getOperand(N2C->getZExtValue());
5908
5909 // EXTRACT_ELEMENT of a constant int is also very common.
5910 if (N1C) {
5911 unsigned ElementSize = VT.getSizeInBits();
5912 unsigned Shift = ElementSize * N2C->getZExtValue();
5913 const APInt &Val = N1C->getAPIntValue();
5914 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
5915 }
5916 break;
5917 case ISD::EXTRACT_SUBVECTOR: {
5918 EVT N1VT = N1.getValueType();
5919 assert(VT.isVector() && N1VT.isVector() &&(static_cast <bool> (VT.isVector() && N1VT.isVector
() && "Extract subvector VTs must be vectors!") ? void
(0) : __assert_fail ("VT.isVector() && N1VT.isVector() && \"Extract subvector VTs must be vectors!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5920, __extension__ __PRETTY_FUNCTION__))
5920 "Extract subvector VTs must be vectors!")(static_cast <bool> (VT.isVector() && N1VT.isVector
() && "Extract subvector VTs must be vectors!") ? void
(0) : __assert_fail ("VT.isVector() && N1VT.isVector() && \"Extract subvector VTs must be vectors!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5920, __extension__ __PRETTY_FUNCTION__))
;
5921 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&(static_cast <bool> (VT.getVectorElementType() == N1VT.
getVectorElementType() && "Extract subvector VTs must have the same element type!"
) ? void (0) : __assert_fail ("VT.getVectorElementType() == N1VT.getVectorElementType() && \"Extract subvector VTs must have the same element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5922, __extension__ __PRETTY_FUNCTION__))
5922 "Extract subvector VTs must have the same element type!")(static_cast <bool> (VT.getVectorElementType() == N1VT.
getVectorElementType() && "Extract subvector VTs must have the same element type!"
) ? void (0) : __assert_fail ("VT.getVectorElementType() == N1VT.getVectorElementType() && \"Extract subvector VTs must have the same element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5922, __extension__ __PRETTY_FUNCTION__))
;
5923 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&(static_cast <bool> ((VT.isFixedLengthVector() || N1VT.
isScalableVector()) && "Cannot extract a scalable vector from a fixed length vector!"
) ? void (0) : __assert_fail ("(VT.isFixedLengthVector() || N1VT.isScalableVector()) && \"Cannot extract a scalable vector from a fixed length vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5924, __extension__ __PRETTY_FUNCTION__))
5924 "Cannot extract a scalable vector from a fixed length vector!")(static_cast <bool> ((VT.isFixedLengthVector() || N1VT.
isScalableVector()) && "Cannot extract a scalable vector from a fixed length vector!"
) ? void (0) : __assert_fail ("(VT.isFixedLengthVector() || N1VT.isScalableVector()) && \"Cannot extract a scalable vector from a fixed length vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5924, __extension__ __PRETTY_FUNCTION__))
;
5925 assert((VT.isScalableVector() != N1VT.isScalableVector() ||(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements
()) && "Extract subvector must be from larger vector to smaller vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && \"Extract subvector must be from larger vector to smaller vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5927, __extension__ __PRETTY_FUNCTION__))
5926 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements
()) && "Extract subvector must be from larger vector to smaller vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && \"Extract subvector must be from larger vector to smaller vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5927, __extension__ __PRETTY_FUNCTION__))
5927 "Extract subvector must be from larger vector to smaller vector!")(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements
()) && "Extract subvector must be from larger vector to smaller vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && \"Extract subvector must be from larger vector to smaller vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5927, __extension__ __PRETTY_FUNCTION__))
;
5928 assert(N2C && "Extract subvector index must be a constant")(static_cast <bool> (N2C && "Extract subvector index must be a constant"
) ? void (0) : __assert_fail ("N2C && \"Extract subvector index must be a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5928, __extension__ __PRETTY_FUNCTION__))
;
5929 assert((VT.isScalableVector() != N1VT.isScalableVector() ||(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || (VT.getVectorMinNumElements() + N2C->getZExtValue())
<= N1VT.getVectorMinNumElements()) && "Extract subvector overflow!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= N1VT.getVectorMinNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5932, __extension__ __PRETTY_FUNCTION__))
5930 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || (VT.getVectorMinNumElements() + N2C->getZExtValue())
<= N1VT.getVectorMinNumElements()) && "Extract subvector overflow!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= N1VT.getVectorMinNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5932, __extension__ __PRETTY_FUNCTION__))
5931 N1VT.getVectorMinNumElements()) &&(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || (VT.getVectorMinNumElements() + N2C->getZExtValue())
<= N1VT.getVectorMinNumElements()) && "Extract subvector overflow!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= N1VT.getVectorMinNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5932, __extension__ __PRETTY_FUNCTION__))
5932 "Extract subvector overflow!")(static_cast <bool> ((VT.isScalableVector() != N1VT.isScalableVector
() || (VT.getVectorMinNumElements() + N2C->getZExtValue())
<= N1VT.getVectorMinNumElements()) && "Extract subvector overflow!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N1VT.isScalableVector() || (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= N1VT.getVectorMinNumElements()) && \"Extract subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5932, __extension__ __PRETTY_FUNCTION__))
;
5933 assert(N2C->getAPIntValue().getBitWidth() ==(static_cast <bool> (N2C->getAPIntValue().getBitWidth
() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits
() && "Constant index for EXTRACT_SUBVECTOR has an invalid size"
) ? void (0) : __assert_fail ("N2C->getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && \"Constant index for EXTRACT_SUBVECTOR has an invalid size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5935, __extension__ __PRETTY_FUNCTION__))
5934 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&(static_cast <bool> (N2C->getAPIntValue().getBitWidth
() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits
() && "Constant index for EXTRACT_SUBVECTOR has an invalid size"
) ? void (0) : __assert_fail ("N2C->getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && \"Constant index for EXTRACT_SUBVECTOR has an invalid size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5935, __extension__ __PRETTY_FUNCTION__))
5935 "Constant index for EXTRACT_SUBVECTOR has an invalid size")(static_cast <bool> (N2C->getAPIntValue().getBitWidth
() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits
() && "Constant index for EXTRACT_SUBVECTOR has an invalid size"
) ? void (0) : __assert_fail ("N2C->getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && \"Constant index for EXTRACT_SUBVECTOR has an invalid size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 5935, __extension__ __PRETTY_FUNCTION__))
;
5936
5937 // Trivial extraction.
5938 if (VT == N1VT)
5939 return N1;
5940
5941 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
5942 if (N1.isUndef())
5943 return getUNDEF(VT);
5944
5945 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
5946 // the concat have the same type as the extract.
5947 if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 &&
5948 VT == N1.getOperand(0).getValueType()) {
5949 unsigned Factor = VT.getVectorMinNumElements();
5950 return N1.getOperand(N2C->getZExtValue() / Factor);
5951 }
5952
5953 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
5954 // during shuffle legalization.
5955 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
5956 VT == N1.getOperand(1).getValueType())
5957 return N1.getOperand(1);
5958 break;
5959 }
5960 }
5961
5962 // Perform trivial constant folding.
5963 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
5964 return SV;
5965
5966 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
5967 return V;
5968
5969 // Canonicalize an UNDEF to the RHS, even over a constant.
5970 if (N1.isUndef()) {
5971 if (TLI->isCommutativeBinOp(Opcode)) {
5972 std::swap(N1, N2);
5973 } else {
5974 switch (Opcode) {
5975 case ISD::SIGN_EXTEND_INREG:
5976 case ISD::SUB:
5977 return getUNDEF(VT); // fold op(undef, arg2) -> undef
5978 case ISD::UDIV:
5979 case ISD::SDIV:
5980 case ISD::UREM:
5981 case ISD::SREM:
5982 case ISD::SSUBSAT:
5983 case ISD::USUBSAT:
5984 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
5985 }
5986 }
5987 }
5988
5989 // Fold a bunch of operators when the RHS is undef.
5990 if (N2.isUndef()) {
5991 switch (Opcode) {
5992 case ISD::XOR:
5993 if (N1.isUndef())
5994 // Handle undef ^ undef -> 0 special case. This is a common
5995 // idiom (misuse).
5996 return getConstant(0, DL, VT);
5997 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5998 case ISD::ADD:
5999 case ISD::SUB:
6000 case ISD::UDIV:
6001 case ISD::SDIV:
6002 case ISD::UREM:
6003 case ISD::SREM:
6004 return getUNDEF(VT); // fold op(arg1, undef) -> undef
6005 case ISD::MUL:
6006 case ISD::AND:
6007 case ISD::SSUBSAT:
6008 case ISD::USUBSAT:
6009 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
6010 case ISD::OR:
6011 case ISD::SADDSAT:
6012 case ISD::UADDSAT:
6013 return getAllOnesConstant(DL, VT);
6014 }
6015 }
6016
6017 // Memoize this node if possible.
6018 SDNode *N;
6019 SDVTList VTs = getVTList(VT);
6020 SDValue Ops[] = {N1, N2};
6021 if (VT != MVT::Glue) {
6022 FoldingSetNodeID ID;
6023 AddNodeIDNode(ID, Opcode, VTs, Ops);
6024 void *IP = nullptr;
6025 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
6026 E->intersectFlagsWith(Flags);
6027 return SDValue(E, 0);
6028 }
6029
6030 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6031 N->setFlags(Flags);
6032 createOperands(N, Ops);
6033 CSEMap.InsertNode(N, IP);
6034 } else {
6035 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6036 createOperands(N, Ops);
6037 }
6038
6039 InsertNode(N);
6040 SDValue V = SDValue(N, 0);
6041 NewSDValueDbgMsg(V, "Creating new node: ", this);
6042 return V;
6043}
6044
6045SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6046 SDValue N1, SDValue N2, SDValue N3) {
6047 SDNodeFlags Flags;
6048 if (Inserter)
6049 Flags = Inserter->getFlags();
6050 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
6051}
6052
6053SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6054 SDValue N1, SDValue N2, SDValue N3,
6055 const SDNodeFlags Flags) {
6056 assert(N1.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && N3
.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && N3.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6059, __extension__ __PRETTY_FUNCTION__))
6057 N2.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && N3
.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && N3.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6059, __extension__ __PRETTY_FUNCTION__))
6058 N3.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && N3
.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && N3.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6059, __extension__ __PRETTY_FUNCTION__))
6059 "Operand is DELETED_NODE!")(static_cast <bool> (N1.getOpcode() != ISD::DELETED_NODE
&& N2.getOpcode() != ISD::DELETED_NODE && N3
.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"
) ? void (0) : __assert_fail ("N1.getOpcode() != ISD::DELETED_NODE && N2.getOpcode() != ISD::DELETED_NODE && N3.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6059, __extension__ __PRETTY_FUNCTION__))
;
6060 // Perform various simplifications.
6061 switch (Opcode) {
6062 case ISD::FMA: {
6063 assert(VT.isFloatingPoint() && "This operator only applies to FP types!")(static_cast <bool> (VT.isFloatingPoint() && "This operator only applies to FP types!"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"This operator only applies to FP types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6063, __extension__ __PRETTY_FUNCTION__))
;
6064 assert(N1.getValueType() == VT && N2.getValueType() == VT &&(static_cast <bool> (N1.getValueType() == VT &&
N2.getValueType() == VT && N3.getValueType() == VT &&
"FMA types must match!") ? void (0) : __assert_fail ("N1.getValueType() == VT && N2.getValueType() == VT && N3.getValueType() == VT && \"FMA types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6065, __extension__ __PRETTY_FUNCTION__))
6065 N3.getValueType() == VT && "FMA types must match!")(static_cast <bool> (N1.getValueType() == VT &&
N2.getValueType() == VT && N3.getValueType() == VT &&
"FMA types must match!") ? void (0) : __assert_fail ("N1.getValueType() == VT && N2.getValueType() == VT && N3.getValueType() == VT && \"FMA types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6065, __extension__ __PRETTY_FUNCTION__))
;
6066 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
6067 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
6068 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
6069 if (N1CFP && N2CFP && N3CFP) {
6070 APFloat V1 = N1CFP->getValueAPF();
6071 const APFloat &V2 = N2CFP->getValueAPF();
6072 const APFloat &V3 = N3CFP->getValueAPF();
6073 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
6074 return getConstantFP(V1, DL, VT);
6075 }
6076 break;
6077 }
6078 case ISD::BUILD_VECTOR: {
6079 // Attempt to simplify BUILD_VECTOR.
6080 SDValue Ops[] = {N1, N2, N3};
6081 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6082 return V;
6083 break;
6084 }
6085 case ISD::CONCAT_VECTORS: {
6086 SDValue Ops[] = {N1, N2, N3};
6087 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
6088 return V;
6089 break;
6090 }
6091 case ISD::SETCC: {
6092 assert(VT.isInteger() && "SETCC result type must be an integer!")(static_cast <bool> (VT.isInteger() && "SETCC result type must be an integer!"
) ? void (0) : __assert_fail ("VT.isInteger() && \"SETCC result type must be an integer!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6092, __extension__ __PRETTY_FUNCTION__))
;
6093 assert(N1.getValueType() == N2.getValueType() &&(static_cast <bool> (N1.getValueType() == N2.getValueType
() && "SETCC operands must have the same type!") ? void
(0) : __assert_fail ("N1.getValueType() == N2.getValueType() && \"SETCC operands must have the same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6094, __extension__ __PRETTY_FUNCTION__))
6094 "SETCC operands must have the same type!")(static_cast <bool> (N1.getValueType() == N2.getValueType
() && "SETCC operands must have the same type!") ? void
(0) : __assert_fail ("N1.getValueType() == N2.getValueType() && \"SETCC operands must have the same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6094, __extension__ __PRETTY_FUNCTION__))
;
6095 assert(VT.isVector() == N1.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == N1.getValueType()
.isVector() && "SETCC type should be vector iff the operand type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == N1.getValueType().isVector() && \"SETCC type should be vector iff the operand type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6096, __extension__ __PRETTY_FUNCTION__))
6096 "SETCC type should be vector iff the operand type is vector!")(static_cast <bool> (VT.isVector() == N1.getValueType()
.isVector() && "SETCC type should be vector iff the operand type is vector!"
) ? void (0) : __assert_fail ("VT.isVector() == N1.getValueType().isVector() && \"SETCC type should be vector iff the operand type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6096, __extension__ __PRETTY_FUNCTION__))
;
6097 assert((!VT.isVector() || VT.getVectorElementCount() ==(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == N1.getValueType().getVectorElementCount()) && "SETCC vector element counts must match!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == N1.getValueType().getVectorElementCount()) && \"SETCC vector element counts must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6099, __extension__ __PRETTY_FUNCTION__))
6098 N1.getValueType().getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == N1.getValueType().getVectorElementCount()) && "SETCC vector element counts must match!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == N1.getValueType().getVectorElementCount()) && \"SETCC vector element counts must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6099, __extension__ __PRETTY_FUNCTION__))
6099 "SETCC vector element counts must match!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == N1.getValueType().getVectorElementCount()) && "SETCC vector element counts must match!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == N1.getValueType().getVectorElementCount()) && \"SETCC vector element counts must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6099, __extension__ __PRETTY_FUNCTION__))
;
6100 // Use FoldSetCC to simplify SETCC's.
6101 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
6102 return V;
6103 // Vector constant folding.
6104 SDValue Ops[] = {N1, N2, N3};
6105 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
6106 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
6107 return V;
6108 }
6109 break;
6110 }
6111 case ISD::SELECT:
6112 case ISD::VSELECT:
6113 if (SDValue V = simplifySelect(N1, N2, N3))
6114 return V;
6115 break;
6116 case ISD::VECTOR_SHUFFLE:
6117 llvm_unreachable("should use getVectorShuffle constructor!")::llvm::llvm_unreachable_internal("should use getVectorShuffle constructor!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6117)
;
6118 case ISD::INSERT_VECTOR_ELT: {
6119 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
6120 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
6121 // for scalable vectors where we will generate appropriate code to
6122 // deal with out-of-bounds cases correctly.
6123 if (N3C && N1.getValueType().isFixedLengthVector() &&
6124 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
6125 return getUNDEF(VT);
6126
6127 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
6128 if (N3.isUndef())
6129 return getUNDEF(VT);
6130
6131 // If the inserted element is an UNDEF, just use the input vector.
6132 if (N2.isUndef())
6133 return N1;
6134
6135 break;
6136 }
6137 case ISD::INSERT_SUBVECTOR: {
6138 // Inserting undef into undef is still undef.
6139 if (N1.isUndef() && N2.isUndef())
6140 return getUNDEF(VT);
6141
6142 EVT N2VT = N2.getValueType();
6143 assert(VT == N1.getValueType() &&(static_cast <bool> (VT == N1.getValueType() &&
"Dest and insert subvector source types must match!") ? void
(0) : __assert_fail ("VT == N1.getValueType() && \"Dest and insert subvector source types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6144, __extension__ __PRETTY_FUNCTION__))
6144 "Dest and insert subvector source types must match!")(static_cast <bool> (VT == N1.getValueType() &&
"Dest and insert subvector source types must match!") ? void
(0) : __assert_fail ("VT == N1.getValueType() && \"Dest and insert subvector source types must match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6144, __extension__ __PRETTY_FUNCTION__))
;
6145 assert(VT.isVector() && N2VT.isVector() &&(static_cast <bool> (VT.isVector() && N2VT.isVector
() && "Insert subvector VTs must be vectors!") ? void
(0) : __assert_fail ("VT.isVector() && N2VT.isVector() && \"Insert subvector VTs must be vectors!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6146, __extension__ __PRETTY_FUNCTION__))
6146 "Insert subvector VTs must be vectors!")(static_cast <bool> (VT.isVector() && N2VT.isVector
() && "Insert subvector VTs must be vectors!") ? void
(0) : __assert_fail ("VT.isVector() && N2VT.isVector() && \"Insert subvector VTs must be vectors!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6146, __extension__ __PRETTY_FUNCTION__))
;
6147 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&(static_cast <bool> ((VT.isScalableVector() || N2VT.isFixedLengthVector
()) && "Cannot insert a scalable vector into a fixed length vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() || N2VT.isFixedLengthVector()) && \"Cannot insert a scalable vector into a fixed length vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6148, __extension__ __PRETTY_FUNCTION__))
6148 "Cannot insert a scalable vector into a fixed length vector!")(static_cast <bool> ((VT.isScalableVector() || N2VT.isFixedLengthVector
()) && "Cannot insert a scalable vector into a fixed length vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() || N2VT.isFixedLengthVector()) && \"Cannot insert a scalable vector into a fixed length vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6148, __extension__ __PRETTY_FUNCTION__))
;
6149 assert((VT.isScalableVector() != N2VT.isScalableVector() ||(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements
()) && "Insert subvector must be from smaller vector to larger vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N2VT.isScalableVector() || VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && \"Insert subvector must be from smaller vector to larger vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6151, __extension__ __PRETTY_FUNCTION__))
6150 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements
()) && "Insert subvector must be from smaller vector to larger vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N2VT.isScalableVector() || VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && \"Insert subvector must be from smaller vector to larger vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6151, __extension__ __PRETTY_FUNCTION__))
6151 "Insert subvector must be from smaller vector to larger vector!")(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements
()) && "Insert subvector must be from smaller vector to larger vector!"
) ? void (0) : __assert_fail ("(VT.isScalableVector() != N2VT.isScalableVector() || VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && \"Insert subvector must be from smaller vector to larger vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6151, __extension__ __PRETTY_FUNCTION__))
;
6152 assert(isa<ConstantSDNode>(N3) &&(static_cast <bool> (isa<ConstantSDNode>(N3) &&
"Insert subvector index must be constant") ? void (0) : __assert_fail
("isa<ConstantSDNode>(N3) && \"Insert subvector index must be constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6153, __extension__ __PRETTY_FUNCTION__))
6153 "Insert subvector index must be constant")(static_cast <bool> (isa<ConstantSDNode>(N3) &&
"Insert subvector index must be constant") ? void (0) : __assert_fail
("isa<ConstantSDNode>(N3) && \"Insert subvector index must be constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6153, __extension__ __PRETTY_FUNCTION__))
;
6154 assert((VT.isScalableVector() != N2VT.isScalableVector() ||(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode
>(N3)->getZExtValue()) <= VT.getVectorMinNumElements
()) && "Insert subvector overflow!") ? void (0) : __assert_fail
("(VT.isScalableVector() != N2VT.isScalableVector() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode>(N3)->getZExtValue()) <= VT.getVectorMinNumElements()) && \"Insert subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6158, __extension__ __PRETTY_FUNCTION__))
6155 (N2VT.getVectorMinNumElements() +(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode
>(N3)->getZExtValue()) <= VT.getVectorMinNumElements
()) && "Insert subvector overflow!") ? void (0) : __assert_fail
("(VT.isScalableVector() != N2VT.isScalableVector() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode>(N3)->getZExtValue()) <= VT.getVectorMinNumElements()) && \"Insert subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6158, __extension__ __PRETTY_FUNCTION__))
6156 cast<ConstantSDNode>(N3)->getZExtValue()) <=(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode
>(N3)->getZExtValue()) <= VT.getVectorMinNumElements
()) && "Insert subvector overflow!") ? void (0) : __assert_fail
("(VT.isScalableVector() != N2VT.isScalableVector() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode>(N3)->getZExtValue()) <= VT.getVectorMinNumElements()) && \"Insert subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6158, __extension__ __PRETTY_FUNCTION__))
6157 VT.getVectorMinNumElements()) &&(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode
>(N3)->getZExtValue()) <= VT.getVectorMinNumElements
()) && "Insert subvector overflow!") ? void (0) : __assert_fail
("(VT.isScalableVector() != N2VT.isScalableVector() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode>(N3)->getZExtValue()) <= VT.getVectorMinNumElements()) && \"Insert subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6158, __extension__ __PRETTY_FUNCTION__))
6158 "Insert subvector overflow!")(static_cast <bool> ((VT.isScalableVector() != N2VT.isScalableVector
() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode
>(N3)->getZExtValue()) <= VT.getVectorMinNumElements
()) && "Insert subvector overflow!") ? void (0) : __assert_fail
("(VT.isScalableVector() != N2VT.isScalableVector() || (N2VT.getVectorMinNumElements() + cast<ConstantSDNode>(N3)->getZExtValue()) <= VT.getVectorMinNumElements()) && \"Insert subvector overflow!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6158, __extension__ __PRETTY_FUNCTION__))
;
6159 assert(cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() ==(static_cast <bool> (cast<ConstantSDNode>(N3)->
getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout
()).getFixedSizeInBits() && "Constant index for INSERT_SUBVECTOR has an invalid size"
) ? void (0) : __assert_fail ("cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && \"Constant index for INSERT_SUBVECTOR has an invalid size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6161, __extension__ __PRETTY_FUNCTION__))
6160 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&(static_cast <bool> (cast<ConstantSDNode>(N3)->
getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout
()).getFixedSizeInBits() && "Constant index for INSERT_SUBVECTOR has an invalid size"
) ? void (0) : __assert_fail ("cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && \"Constant index for INSERT_SUBVECTOR has an invalid size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6161, __extension__ __PRETTY_FUNCTION__))
6161 "Constant index for INSERT_SUBVECTOR has an invalid size")(static_cast <bool> (cast<ConstantSDNode>(N3)->
getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout
()).getFixedSizeInBits() && "Constant index for INSERT_SUBVECTOR has an invalid size"
) ? void (0) : __assert_fail ("cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() == TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && \"Constant index for INSERT_SUBVECTOR has an invalid size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6161, __extension__ __PRETTY_FUNCTION__))
;
6162
6163 // Trivial insertion.
6164 if (VT == N2VT)
6165 return N2;
6166
6167 // If this is an insert of an extracted vector into an undef vector, we
6168 // can just use the input to the extract.
6169 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6170 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
6171 return N2.getOperand(0);
6172 break;
6173 }
6174 case ISD::BITCAST:
6175 // Fold bit_convert nodes from a type to themselves.
6176 if (N1.getValueType() == VT)
6177 return N1;
6178 break;
6179 }
6180
6181 // Memoize node if it doesn't produce a flag.
6182 SDNode *N;
6183 SDVTList VTs = getVTList(VT);
6184 SDValue Ops[] = {N1, N2, N3};
6185 if (VT != MVT::Glue) {
6186 FoldingSetNodeID ID;
6187 AddNodeIDNode(ID, Opcode, VTs, Ops);
6188 void *IP = nullptr;
6189 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
6190 E->intersectFlagsWith(Flags);
6191 return SDValue(E, 0);
6192 }
6193
6194 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6195 N->setFlags(Flags);
6196 createOperands(N, Ops);
6197 CSEMap.InsertNode(N, IP);
6198 } else {
6199 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6200 createOperands(N, Ops);
6201 }
6202
6203 InsertNode(N);
6204 SDValue V = SDValue(N, 0);
6205 NewSDValueDbgMsg(V, "Creating new node: ", this);
6206 return V;
6207}
6208
6209SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6210 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
6211 SDValue Ops[] = { N1, N2, N3, N4 };
6212 return getNode(Opcode, DL, VT, Ops);
6213}
6214
6215SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6216 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
6217 SDValue N5) {
6218 SDValue Ops[] = { N1, N2, N3, N4, N5 };
6219 return getNode(Opcode, DL, VT, Ops);
6220}
6221
6222/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
6223/// the incoming stack arguments to be loaded from the stack.
6224SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
6225 SmallVector<SDValue, 8> ArgChains;
6226
6227 // Include the original chain at the beginning of the list. When this is
6228 // used by target LowerCall hooks, this helps legalize find the
6229 // CALLSEQ_BEGIN node.
6230 ArgChains.push_back(Chain);
6231
6232 // Add a chain value for each stack argument.
6233 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
6234 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
6235 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
6236 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
6237 if (FI->getIndex() < 0)
6238 ArgChains.push_back(SDValue(L, 1));
6239
6240 // Build a tokenfactor for all the chains.
6241 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
6242}
6243
6244/// getMemsetValue - Vectorized representation of the memset value
6245/// operand.
6246static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
6247 const SDLoc &dl) {
6248 assert(!Value.isUndef())(static_cast <bool> (!Value.isUndef()) ? void (0) : __assert_fail
("!Value.isUndef()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6248, __extension__ __PRETTY_FUNCTION__))
;
6249
6250 unsigned NumBits = VT.getScalarSizeInBits();
6251 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
6252 assert(C->getAPIntValue().getBitWidth() == 8)(static_cast <bool> (C->getAPIntValue().getBitWidth(
) == 8) ? void (0) : __assert_fail ("C->getAPIntValue().getBitWidth() == 8"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6252, __extension__ __PRETTY_FUNCTION__))
;
6253 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
6254 if (VT.isInteger()) {
6255 bool IsOpaque = VT.getSizeInBits() > 64 ||
6256 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
6257 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
6258 }
6259 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
6260 VT);
6261 }
6262
6263 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?")(static_cast <bool> (Value.getValueType() == MVT::i8 &&
"memset with non-byte fill value?") ? void (0) : __assert_fail
("Value.getValueType() == MVT::i8 && \"memset with non-byte fill value?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6263, __extension__ __PRETTY_FUNCTION__))
;
6264 EVT IntVT = VT.getScalarType();
6265 if (!IntVT.isInteger())
6266 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
6267
6268 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
6269 if (NumBits > 8) {
6270 // Use a multiplication with 0x010101... to extend the input to the
6271 // required length.
6272 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
6273 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
6274 DAG.getConstant(Magic, dl, IntVT));
6275 }
6276
6277 if (VT != Value.getValueType() && !VT.isInteger())
6278 Value = DAG.getBitcast(VT.getScalarType(), Value);
6279 if (VT != Value.getValueType())
6280 Value = DAG.getSplatBuildVector(VT, dl, Value);
6281
6282 return Value;
6283}
6284
6285/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
6286/// used when a memcpy is turned into a memset when the source is a constant
6287/// string ptr.
6288static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
6289 const TargetLowering &TLI,
6290 const ConstantDataArraySlice &Slice) {
6291 // Handle vector with all elements zero.
6292 if (Slice.Array == nullptr) {
6293 if (VT.isInteger())
6294 return DAG.getConstant(0, dl, VT);
6295 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
6296 return DAG.getConstantFP(0.0, dl, VT);
6297 if (VT.isVector()) {
6298 unsigned NumElts = VT.getVectorNumElements();
6299 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
6300 return DAG.getNode(ISD::BITCAST, dl, VT,
6301 DAG.getConstant(0, dl,
6302 EVT::getVectorVT(*DAG.getContext(),
6303 EltVT, NumElts)));
6304 }
6305 llvm_unreachable("Expected type!")::llvm::llvm_unreachable_internal("Expected type!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6305)
;
6306 }
6307
6308 assert(!VT.isVector() && "Can't handle vector type here!")(static_cast <bool> (!VT.isVector() && "Can't handle vector type here!"
) ? void (0) : __assert_fail ("!VT.isVector() && \"Can't handle vector type here!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6308, __extension__ __PRETTY_FUNCTION__))
;
6309 unsigned NumVTBits = VT.getSizeInBits();
6310 unsigned NumVTBytes = NumVTBits / 8;
6311 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
6312
6313 APInt Val(NumVTBits, 0);
6314 if (DAG.getDataLayout().isLittleEndian()) {
6315 for (unsigned i = 0; i != NumBytes; ++i)
6316 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
6317 } else {
6318 for (unsigned i = 0; i != NumBytes; ++i)
6319 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
6320 }
6321
6322 // If the "cost" of materializing the integer immediate is less than the cost
6323 // of a load, then it is cost effective to turn the load into the immediate.
6324 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
6325 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
6326 return DAG.getConstant(Val, dl, VT);
6327 return SDValue(nullptr, 0);
6328}
6329
6330SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
6331 const SDLoc &DL,
6332 const SDNodeFlags Flags) {
6333 EVT VT = Base.getValueType();
6334 SDValue Index;
6335
6336 if (Offset.isScalable())
6337 Index = getVScale(DL, Base.getValueType(),
6338 APInt(Base.getValueSizeInBits().getFixedSize(),
6339 Offset.getKnownMinSize()));
6340 else
6341 Index = getConstant(Offset.getFixedSize(), DL, VT);
6342
6343 return getMemBasePlusOffset(Base, Index, DL, Flags);
6344}
6345
6346SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
6347 const SDLoc &DL,
6348 const SDNodeFlags Flags) {
6349 assert(Offset.getValueType().isInteger())(static_cast <bool> (Offset.getValueType().isInteger())
? void (0) : __assert_fail ("Offset.getValueType().isInteger()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6349, __extension__ __PRETTY_FUNCTION__))
;
6350 EVT BasePtrVT = Ptr.getValueType();
6351 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
6352}
6353
6354/// Returns true if memcpy source is constant data.
6355static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
6356 uint64_t SrcDelta = 0;
6357 GlobalAddressSDNode *G = nullptr;
6358 if (Src.getOpcode() == ISD::GlobalAddress)
6359 G = cast<GlobalAddressSDNode>(Src);
6360 else if (Src.getOpcode() == ISD::ADD &&
6361 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
6362 Src.getOperand(1).getOpcode() == ISD::Constant) {
6363 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
6364 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
6365 }
6366 if (!G)
6367 return false;
6368
6369 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
6370 SrcDelta + G->getOffset());
6371}
6372
6373static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
6374 SelectionDAG &DAG) {
6375 // On Darwin, -Os means optimize for size without hurting performance, so
6376 // only really optimize for size when -Oz (MinSize) is used.
6377 if (MF.getTarget().getTargetTriple().isOSDarwin())
6378 return MF.getFunction().hasMinSize();
6379 return DAG.shouldOptForSize();
6380}
6381
6382static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
6383 SmallVector<SDValue, 32> &OutChains, unsigned From,
6384 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
6385 SmallVector<SDValue, 16> &OutStoreChains) {
6386 assert(OutLoadChains.size() && "Missing loads in memcpy inlining")(static_cast <bool> (OutLoadChains.size() && "Missing loads in memcpy inlining"
) ? void (0) : __assert_fail ("OutLoadChains.size() && \"Missing loads in memcpy inlining\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6386, __extension__ __PRETTY_FUNCTION__))
;
6387 assert(OutStoreChains.size() && "Missing stores in memcpy inlining")(static_cast <bool> (OutStoreChains.size() && "Missing stores in memcpy inlining"
) ? void (0) : __assert_fail ("OutStoreChains.size() && \"Missing stores in memcpy inlining\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6387, __extension__ __PRETTY_FUNCTION__))
;
6388 SmallVector<SDValue, 16> GluedLoadChains;
6389 for (unsigned i = From; i < To; ++i) {
6390 OutChains.push_back(OutLoadChains[i]);
6391 GluedLoadChains.push_back(OutLoadChains[i]);
6392 }
6393
6394 // Chain for all loads.
6395 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
6396 GluedLoadChains);
6397
6398 for (unsigned i = From; i < To; ++i) {
6399 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
6400 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
6401 ST->getBasePtr(), ST->getMemoryVT(),
6402 ST->getMemOperand());
6403 OutChains.push_back(NewStore);
6404 }
6405}
6406
6407static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
6408 SDValue Chain, SDValue Dst, SDValue Src,
6409 uint64_t Size, Align Alignment,
6410 bool isVol, bool AlwaysInline,
6411 MachinePointerInfo DstPtrInfo,
6412 MachinePointerInfo SrcPtrInfo,
6413 const AAMDNodes &AAInfo) {
6414 // Turn a memcpy of undef to nop.
6415 // FIXME: We need to honor volatile even is Src is undef.
6416 if (Src.isUndef())
6417 return Chain;
6418
6419 // Expand memcpy to a series of load and store ops if the size operand falls
6420 // below a certain threshold.
6421 // TODO: In the AlwaysInline case, if the size is big then generate a loop
6422 // rather than maybe a humongous number of loads and stores.
6423 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6424 const DataLayout &DL = DAG.getDataLayout();
6425 LLVMContext &C = *DAG.getContext();
6426 std::vector<EVT> MemOps;
6427 bool DstAlignCanChange = false;
6428 MachineFunction &MF = DAG.getMachineFunction();
6429 MachineFrameInfo &MFI = MF.getFrameInfo();
6430 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6431 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6432 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6433 DstAlignCanChange = true;
6434 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
6435 if (!SrcAlign || Alignment > *SrcAlign)
6436 SrcAlign = Alignment;
6437 assert(SrcAlign && "SrcAlign must be set")(static_cast <bool> (SrcAlign && "SrcAlign must be set"
) ? void (0) : __assert_fail ("SrcAlign && \"SrcAlign must be set\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6437, __extension__ __PRETTY_FUNCTION__))
;
6438 ConstantDataArraySlice Slice;
6439 // If marked as volatile, perform a copy even when marked as constant.
6440 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
6441 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
6442 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
6443 const MemOp Op = isZeroConstant
6444 ? MemOp::Set(Size, DstAlignCanChange, Alignment,
6445 /*IsZeroMemset*/ true, isVol)
6446 : MemOp::Copy(Size, DstAlignCanChange, Alignment,
6447 *SrcAlign, isVol, CopyFromConstant);
6448 if (!TLI.findOptimalMemOpLowering(
6449 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
6450 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
6451 return SDValue();
6452
6453 if (DstAlignCanChange) {
6454 Type *Ty = MemOps[0].getTypeForEVT(C);
6455 Align NewAlign = DL.getABITypeAlign(Ty);
6456
6457 // Don't promote to an alignment that would require dynamic stack
6458 // realignment.
6459 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6460 if (!TRI->hasStackRealignment(MF))
6461 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
6462 NewAlign = NewAlign / 2;
6463
6464 if (NewAlign > Alignment) {
6465 // Give the stack frame object a larger alignment if needed.
6466 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6467 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6468 Alignment = NewAlign;
6469 }
6470 }
6471
6472 // Prepare AAInfo for loads/stores after lowering this memcpy.
6473 AAMDNodes NewAAInfo = AAInfo;
6474 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
6475
6476 MachineMemOperand::Flags MMOFlags =
6477 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6478 SmallVector<SDValue, 16> OutLoadChains;
6479 SmallVector<SDValue, 16> OutStoreChains;
6480 SmallVector<SDValue, 32> OutChains;
6481 unsigned NumMemOps = MemOps.size();
6482 uint64_t SrcOff = 0, DstOff = 0;
6483 for (unsigned i = 0; i != NumMemOps; ++i) {
6484 EVT VT = MemOps[i];
6485 unsigned VTSize = VT.getSizeInBits() / 8;
6486 SDValue Value, Store;
6487
6488 if (VTSize > Size) {
6489 // Issuing an unaligned load / store pair that overlaps with the previous
6490 // pair. Adjust the offset accordingly.
6491 assert(i == NumMemOps-1 && i != 0)(static_cast <bool> (i == NumMemOps-1 && i != 0
) ? void (0) : __assert_fail ("i == NumMemOps-1 && i != 0"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6491, __extension__ __PRETTY_FUNCTION__))
;
6492 SrcOff -= VTSize - Size;
6493 DstOff -= VTSize - Size;
6494 }
6495
6496 if (CopyFromConstant &&
6497 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
6498 // It's unlikely a store of a vector immediate can be done in a single
6499 // instruction. It would require a load from a constantpool first.
6500 // We only handle zero vectors here.
6501 // FIXME: Handle other cases where store of vector immediate is done in
6502 // a single instruction.
6503 ConstantDataArraySlice SubSlice;
6504 if (SrcOff < Slice.Length) {
6505 SubSlice = Slice;
6506 SubSlice.move(SrcOff);
6507 } else {
6508 // This is an out-of-bounds access and hence UB. Pretend we read zero.
6509 SubSlice.Array = nullptr;
6510 SubSlice.Offset = 0;
6511 SubSlice.Length = VTSize;
6512 }
6513 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
6514 if (Value.getNode()) {
6515 Store = DAG.getStore(
6516 Chain, dl, Value,
6517 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6518 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
6519 OutChains.push_back(Store);
6520 }
6521 }
6522
6523 if (!Store.getNode()) {
6524 // The type might not be legal for the target. This should only happen
6525 // if the type is smaller than a legal type, as on PPC, so the right
6526 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
6527 // to Load/Store if NVT==VT.
6528 // FIXME does the case above also need this?
6529 EVT NVT = TLI.getTypeToTransformTo(C, VT);
6530 assert(NVT.bitsGE(VT))(static_cast <bool> (NVT.bitsGE(VT)) ? void (0) : __assert_fail
("NVT.bitsGE(VT)", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6530, __extension__ __PRETTY_FUNCTION__))
;
6531
6532 bool isDereferenceable =
6533 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6534 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6535 if (isDereferenceable)
6536 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6537
6538 Value = DAG.getExtLoad(
6539 ISD::EXTLOAD, dl, NVT, Chain,
6540 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
6541 SrcPtrInfo.getWithOffset(SrcOff), VT,
6542 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
6543 OutLoadChains.push_back(Value.getValue(1));
6544
6545 Store = DAG.getTruncStore(
6546 Chain, dl, Value,
6547 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6548 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
6549 OutStoreChains.push_back(Store);
6550 }
6551 SrcOff += VTSize;
6552 DstOff += VTSize;
6553 Size -= VTSize;
6554 }
6555
6556 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
6557 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
6558 unsigned NumLdStInMemcpy = OutStoreChains.size();
6559
6560 if (NumLdStInMemcpy) {
6561 // It may be that memcpy might be converted to memset if it's memcpy
6562 // of constants. In such a case, we won't have loads and stores, but
6563 // just stores. In the absence of loads, there is nothing to gang up.
6564 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
6565 // If target does not care, just leave as it.
6566 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
6567 OutChains.push_back(OutLoadChains[i]);
6568 OutChains.push_back(OutStoreChains[i]);
6569 }
6570 } else {
6571 // Ld/St less than/equal limit set by target.
6572 if (NumLdStInMemcpy <= GluedLdStLimit) {
6573 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6574 NumLdStInMemcpy, OutLoadChains,
6575 OutStoreChains);
6576 } else {
6577 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
6578 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
6579 unsigned GlueIter = 0;
6580
6581 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
6582 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
6583 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
6584
6585 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
6586 OutLoadChains, OutStoreChains);
6587 GlueIter += GluedLdStLimit;
6588 }
6589
6590 // Residual ld/st.
6591 if (RemainingLdStInMemcpy) {
6592 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6593 RemainingLdStInMemcpy, OutLoadChains,
6594 OutStoreChains);
6595 }
6596 }
6597 }
6598 }
6599 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6600}
6601
6602static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
6603 SDValue Chain, SDValue Dst, SDValue Src,
6604 uint64_t Size, Align Alignment,
6605 bool isVol, bool AlwaysInline,
6606 MachinePointerInfo DstPtrInfo,
6607 MachinePointerInfo SrcPtrInfo,
6608 const AAMDNodes &AAInfo) {
6609 // Turn a memmove of undef to nop.
6610 // FIXME: We need to honor volatile even is Src is undef.
6611 if (Src.isUndef())
6612 return Chain;
6613
6614 // Expand memmove to a series of load and store ops if the size operand falls
6615 // below a certain threshold.
6616 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6617 const DataLayout &DL = DAG.getDataLayout();
6618 LLVMContext &C = *DAG.getContext();
6619 std::vector<EVT> MemOps;
6620 bool DstAlignCanChange = false;
6621 MachineFunction &MF = DAG.getMachineFunction();
6622 MachineFrameInfo &MFI = MF.getFrameInfo();
6623 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6624 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6625 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6626 DstAlignCanChange = true;
6627 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
6628 if (!SrcAlign || Alignment > *SrcAlign)
6629 SrcAlign = Alignment;
6630 assert(SrcAlign && "SrcAlign must be set")(static_cast <bool> (SrcAlign && "SrcAlign must be set"
) ? void (0) : __assert_fail ("SrcAlign && \"SrcAlign must be set\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6630, __extension__ __PRETTY_FUNCTION__))
;
6631 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
6632 if (!TLI.findOptimalMemOpLowering(
6633 MemOps, Limit,
6634 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
6635 /*IsVolatile*/ true),
6636 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
6637 MF.getFunction().getAttributes()))
6638 return SDValue();
6639
6640 if (DstAlignCanChange) {
6641 Type *Ty = MemOps[0].getTypeForEVT(C);
6642 Align NewAlign = DL.getABITypeAlign(Ty);
6643 if (NewAlign > Alignment) {
6644 // Give the stack frame object a larger alignment if needed.
6645 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6646 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6647 Alignment = NewAlign;
6648 }
6649 }
6650
6651 // Prepare AAInfo for loads/stores after lowering this memmove.
6652 AAMDNodes NewAAInfo = AAInfo;
6653 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
6654
6655 MachineMemOperand::Flags MMOFlags =
6656 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6657 uint64_t SrcOff = 0, DstOff = 0;
6658 SmallVector<SDValue, 8> LoadValues;
6659 SmallVector<SDValue, 8> LoadChains;
6660 SmallVector<SDValue, 8> OutChains;
6661 unsigned NumMemOps = MemOps.size();
6662 for (unsigned i = 0; i < NumMemOps; i++) {
6663 EVT VT = MemOps[i];
6664 unsigned VTSize = VT.getSizeInBits() / 8;
6665 SDValue Value;
6666
6667 bool isDereferenceable =
6668 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6669 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6670 if (isDereferenceable)
6671 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6672
6673 Value = DAG.getLoad(
6674 VT, dl, Chain,
6675 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
6676 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
6677 LoadValues.push_back(Value);
6678 LoadChains.push_back(Value.getValue(1));
6679 SrcOff += VTSize;
6680 }
6681 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
6682 OutChains.clear();
6683 for (unsigned i = 0; i < NumMemOps; i++) {
6684 EVT VT = MemOps[i];
6685 unsigned VTSize = VT.getSizeInBits() / 8;
6686 SDValue Store;
6687
6688 Store = DAG.getStore(
6689 Chain, dl, LoadValues[i],
6690 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6691 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
6692 OutChains.push_back(Store);
6693 DstOff += VTSize;
6694 }
6695
6696 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6697}
6698
6699/// Lower the call to 'memset' intrinsic function into a series of store
6700/// operations.
6701///
6702/// \param DAG Selection DAG where lowered code is placed.
6703/// \param dl Link to corresponding IR location.
6704/// \param Chain Control flow dependency.
6705/// \param Dst Pointer to destination memory location.
6706/// \param Src Value of byte to write into the memory.
6707/// \param Size Number of bytes to write.
6708/// \param Alignment Alignment of the destination in bytes.
6709/// \param isVol True if destination is volatile.
6710/// \param DstPtrInfo IR information on the memory pointer.
6711/// \returns New head in the control flow, if lowering was successful, empty
6712/// SDValue otherwise.
6713///
6714/// The function tries to replace 'llvm.memset' intrinsic with several store
6715/// operations and value calculation code. This is usually profitable for small
6716/// memory size.
6717static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
6718 SDValue Chain, SDValue Dst, SDValue Src,
6719 uint64_t Size, Align Alignment, bool isVol,
6720 MachinePointerInfo DstPtrInfo,
6721 const AAMDNodes &AAInfo) {
6722 // Turn a memset of undef to nop.
6723 // FIXME: We need to honor volatile even is Src is undef.
6724 if (Src.isUndef())
6725 return Chain;
6726
6727 // Expand memset to a series of load/store ops if the size operand
6728 // falls below a certain threshold.
6729 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6730 std::vector<EVT> MemOps;
6731 bool DstAlignCanChange = false;
6732 MachineFunction &MF = DAG.getMachineFunction();
6733 MachineFrameInfo &MFI = MF.getFrameInfo();
6734 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6735 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6736 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6737 DstAlignCanChange = true;
6738 bool IsZeroVal =
6739 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
6740 if (!TLI.findOptimalMemOpLowering(
6741 MemOps, TLI.getMaxStoresPerMemset(OptSize),
6742 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
6743 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
6744 return SDValue();
6745
6746 if (DstAlignCanChange) {
6747 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
6748 Align NewAlign = DAG.getDataLayout().getABITypeAlign(Ty);
6749 if (NewAlign > Alignment) {
6750 // Give the stack frame object a larger alignment if needed.
6751 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6752 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6753 Alignment = NewAlign;
6754 }
6755 }
6756
6757 SmallVector<SDValue, 8> OutChains;
6758 uint64_t DstOff = 0;
6759 unsigned NumMemOps = MemOps.size();
6760
6761 // Find the largest store and generate the bit pattern for it.
6762 EVT LargestVT = MemOps[0];
6763 for (unsigned i = 1; i < NumMemOps; i++)
6764 if (MemOps[i].bitsGT(LargestVT))
6765 LargestVT = MemOps[i];
6766 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
6767
6768 // Prepare AAInfo for loads/stores after lowering this memset.
6769 AAMDNodes NewAAInfo = AAInfo;
6770 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
6771
6772 for (unsigned i = 0; i < NumMemOps; i++) {
6773 EVT VT = MemOps[i];
6774 unsigned VTSize = VT.getSizeInBits() / 8;
6775 if (VTSize > Size) {
6776 // Issuing an unaligned load / store pair that overlaps with the previous
6777 // pair. Adjust the offset accordingly.
6778 assert(i == NumMemOps-1 && i != 0)(static_cast <bool> (i == NumMemOps-1 && i != 0
) ? void (0) : __assert_fail ("i == NumMemOps-1 && i != 0"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6778, __extension__ __PRETTY_FUNCTION__))
;
6779 DstOff -= VTSize - Size;
6780 }
6781
6782 // If this store is smaller than the largest store see whether we can get
6783 // the smaller value for free with a truncate.
6784 SDValue Value = MemSetValue;
6785 if (VT.bitsLT(LargestVT)) {
6786 if (!LargestVT.isVector() && !VT.isVector() &&
6787 TLI.isTruncateFree(LargestVT, VT))
6788 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
6789 else
6790 Value = getMemsetValue(Src, VT, DAG, dl);
6791 }
6792 assert(Value.getValueType() == VT && "Value with wrong type.")(static_cast <bool> (Value.getValueType() == VT &&
"Value with wrong type.") ? void (0) : __assert_fail ("Value.getValueType() == VT && \"Value with wrong type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6792, __extension__ __PRETTY_FUNCTION__))
;
6793 SDValue Store = DAG.getStore(
6794 Chain, dl, Value,
6795 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6796 DstPtrInfo.getWithOffset(DstOff), Alignment,
6797 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
6798 NewAAInfo);
6799 OutChains.push_back(Store);
6800 DstOff += VT.getSizeInBits() / 8;
6801 Size -= VTSize;
6802 }
6803
6804 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6805}
6806
6807static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
6808 unsigned AS) {
6809 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
6810 // pointer operands can be losslessly bitcasted to pointers of address space 0
6811 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
6812 report_fatal_error("cannot lower memory intrinsic in address space " +
6813 Twine(AS));
6814 }
6815}
6816
6817SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6818 SDValue Src, SDValue Size, Align Alignment,
6819 bool isVol, bool AlwaysInline, bool isTailCall,
6820 MachinePointerInfo DstPtrInfo,
6821 MachinePointerInfo SrcPtrInfo,
6822 const AAMDNodes &AAInfo) {
6823 // Check to see if we should lower the memcpy to loads and stores first.
6824 // For cases within the target-specified limits, this is the best choice.
6825 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6826 if (ConstantSize) {
6827 // Memcpy with size zero? Just return the original chain.
6828 if (ConstantSize->isNullValue())
6829 return Chain;
6830
6831 SDValue Result = getMemcpyLoadsAndStores(
6832 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
6833 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
6834 if (Result.getNode())
6835 return Result;
6836 }
6837
6838 // Then check to see if we should lower the memcpy with target-specific
6839 // code. If the target chooses to do this, this is the next best.
6840 if (TSI) {
6841 SDValue Result = TSI->EmitTargetCodeForMemcpy(
6842 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
6843 DstPtrInfo, SrcPtrInfo);
6844 if (Result.getNode())
6845 return Result;
6846 }
6847
6848 // If we really need inline code and the target declined to provide it,
6849 // use a (potentially long) sequence of loads and stores.
6850 if (AlwaysInline) {
6851 assert(ConstantSize && "AlwaysInline requires a constant size!")(static_cast <bool> (ConstantSize && "AlwaysInline requires a constant size!"
) ? void (0) : __assert_fail ("ConstantSize && \"AlwaysInline requires a constant size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 6851, __extension__ __PRETTY_FUNCTION__))
;
6852 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
6853 ConstantSize->getZExtValue(), Alignment,
6854 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo);
6855 }
6856
6857 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6858 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6859
6860 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
6861 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
6862 // respect volatile, so they may do things like read or write memory
6863 // beyond the given memory regions. But fixing this isn't easy, and most
6864 // people don't care.
6865
6866 // Emit a library call.
6867 TargetLowering::ArgListTy Args;
6868 TargetLowering::ArgListEntry Entry;
6869 Entry.Ty = Type::getInt8PtrTy(*getContext());
6870 Entry.Node = Dst; Args.push_back(Entry);
6871 Entry.Node = Src; Args.push_back(Entry);
6872
6873 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6874 Entry.Node = Size; Args.push_back(Entry);
6875 // FIXME: pass in SDLoc
6876 TargetLowering::CallLoweringInfo CLI(*this);
6877 CLI.setDebugLoc(dl)
6878 .setChain(Chain)
6879 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
6880 Dst.getValueType().getTypeForEVT(*getContext()),
6881 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
6882 TLI->getPointerTy(getDataLayout())),
6883 std::move(Args))
6884 .setDiscardResult()
6885 .setTailCall(isTailCall);
6886
6887 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6888 return CallResult.second;
6889}
6890
6891SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
6892 SDValue Dst, unsigned DstAlign,
6893 SDValue Src, unsigned SrcAlign,
6894 SDValue Size, Type *SizeTy,
6895 unsigned ElemSz, bool isTailCall,
6896 MachinePointerInfo DstPtrInfo,
6897 MachinePointerInfo SrcPtrInfo) {
6898 // Emit a library call.
6899 TargetLowering::ArgListTy Args;
6900 TargetLowering::ArgListEntry Entry;
6901 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6902 Entry.Node = Dst;
6903 Args.push_back(Entry);
6904
6905 Entry.Node = Src;
6906 Args.push_back(Entry);
6907
6908 Entry.Ty = SizeTy;
6909 Entry.Node = Size;
6910 Args.push_back(Entry);
6911
6912 RTLIB::Libcall LibraryCall =
6913 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6914 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6915 report_fatal_error("Unsupported element size");
6916
6917 TargetLowering::CallLoweringInfo CLI(*this);
6918 CLI.setDebugLoc(dl)
6919 .setChain(Chain)
6920 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6921 Type::getVoidTy(*getContext()),
6922 getExternalSymbol(TLI->getLibcallName(LibraryCall),
6923 TLI->getPointerTy(getDataLayout())),
6924 std::move(Args))
6925 .setDiscardResult()
6926 .setTailCall(isTailCall);
6927
6928 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6929 return CallResult.second;
6930}
6931
6932SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6933 SDValue Src, SDValue Size, Align Alignment,
6934 bool isVol, bool isTailCall,
6935 MachinePointerInfo DstPtrInfo,
6936 MachinePointerInfo SrcPtrInfo,
6937 const AAMDNodes &AAInfo) {
6938 // Check to see if we should lower the memmove to loads and stores first.
6939 // For cases within the target-specified limits, this is the best choice.
6940 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6941 if (ConstantSize) {
6942 // Memmove with size zero? Just return the original chain.
6943 if (ConstantSize->isNullValue())
6944 return Chain;
6945
6946 SDValue Result = getMemmoveLoadsAndStores(
6947 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
6948 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
6949 if (Result.getNode())
6950 return Result;
6951 }
6952
6953 // Then check to see if we should lower the memmove with target-specific
6954 // code. If the target chooses to do this, this is the next best.
6955 if (TSI) {
6956 SDValue Result =
6957 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
6958 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
6959 if (Result.getNode())
6960 return Result;
6961 }
6962
6963 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6964 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6965
6966 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
6967 // not be safe. See memcpy above for more details.
6968
6969 // Emit a library call.
6970 TargetLowering::ArgListTy Args;
6971 TargetLowering::ArgListEntry Entry;
6972 Entry.Ty = Type::getInt8PtrTy(*getContext());
6973 Entry.Node = Dst; Args.push_back(Entry);
6974 Entry.Node = Src; Args.push_back(Entry);
6975
6976 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6977 Entry.Node = Size; Args.push_back(Entry);
6978 // FIXME: pass in SDLoc
6979 TargetLowering::CallLoweringInfo CLI(*this);
6980 CLI.setDebugLoc(dl)
6981 .setChain(Chain)
6982 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
6983 Dst.getValueType().getTypeForEVT(*getContext()),
6984 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
6985 TLI->getPointerTy(getDataLayout())),
6986 std::move(Args))
6987 .setDiscardResult()
6988 .setTailCall(isTailCall);
6989
6990 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6991 return CallResult.second;
6992}
6993
6994SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
6995 SDValue Dst, unsigned DstAlign,
6996 SDValue Src, unsigned SrcAlign,
6997 SDValue Size, Type *SizeTy,
6998 unsigned ElemSz, bool isTailCall,
6999 MachinePointerInfo DstPtrInfo,
7000 MachinePointerInfo SrcPtrInfo) {
7001 // Emit a library call.
7002 TargetLowering::ArgListTy Args;
7003 TargetLowering::ArgListEntry Entry;
7004 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7005 Entry.Node = Dst;
7006 Args.push_back(Entry);
7007
7008 Entry.Node = Src;
7009 Args.push_back(Entry);
7010
7011 Entry.Ty = SizeTy;
7012 Entry.Node = Size;
7013 Args.push_back(Entry);
7014
7015 RTLIB::Libcall LibraryCall =
7016 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
7017 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
7018 report_fatal_error("Unsupported element size");
7019
7020 TargetLowering::CallLoweringInfo CLI(*this);
7021 CLI.setDebugLoc(dl)
7022 .setChain(Chain)
7023 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
7024 Type::getVoidTy(*getContext()),
7025 getExternalSymbol(TLI->getLibcallName(LibraryCall),
7026 TLI->getPointerTy(getDataLayout())),
7027 std::move(Args))
7028 .setDiscardResult()
7029 .setTailCall(isTailCall);
7030
7031 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
7032 return CallResult.second;
7033}
7034
7035SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
7036 SDValue Src, SDValue Size, Align Alignment,
7037 bool isVol, bool isTailCall,
7038 MachinePointerInfo DstPtrInfo,
7039 const AAMDNodes &AAInfo) {
7040 // Check to see if we should lower the memset to stores first.
7041 // For cases within the target-specified limits, this is the best choice.
7042 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
7043 if (ConstantSize) {
7044 // Memset with size zero? Just return the original chain.
7045 if (ConstantSize->isNullValue())
7046 return Chain;
7047
7048 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
7049 ConstantSize->getZExtValue(), Alignment,
7050 isVol, DstPtrInfo, AAInfo);
7051
7052 if (Result.getNode())
7053 return Result;
7054 }
7055
7056 // Then check to see if we should lower the memset with target-specific
7057 // code. If the target chooses to do this, this is the next best.
7058 if (TSI) {
7059 SDValue Result = TSI->EmitTargetCodeForMemset(
7060 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, DstPtrInfo);
7061 if (Result.getNode())
7062 return Result;
7063 }
7064
7065 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
7066
7067 // Emit a library call.
7068 TargetLowering::ArgListTy Args;
7069 TargetLowering::ArgListEntry Entry;
7070 Entry.Node = Dst; Entry.Ty = Type::getInt8PtrTy(*getContext());
7071 Args.push_back(Entry);
7072 Entry.Node = Src;
7073 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
7074 Args.push_back(Entry);
7075 Entry.Node = Size;
7076 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7077 Args.push_back(Entry);
7078
7079 // FIXME: pass in SDLoc
7080 TargetLowering::CallLoweringInfo CLI(*this);
7081 CLI.setDebugLoc(dl)
7082 .setChain(Chain)
7083 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
7084 Dst.getValueType().getTypeForEVT(*getContext()),
7085 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
7086 TLI->getPointerTy(getDataLayout())),
7087 std::move(Args))
7088 .setDiscardResult()
7089 .setTailCall(isTailCall);
7090
7091 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
7092 return CallResult.second;
7093}
7094
7095SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
7096 SDValue Dst, unsigned DstAlign,
7097 SDValue Value, SDValue Size, Type *SizeTy,
7098 unsigned ElemSz, bool isTailCall,
7099 MachinePointerInfo DstPtrInfo) {
7100 // Emit a library call.
7101 TargetLowering::ArgListTy Args;
7102 TargetLowering::ArgListEntry Entry;
7103 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7104 Entry.Node = Dst;
7105 Args.push_back(Entry);
7106
7107 Entry.Ty = Type::getInt8Ty(*getContext());
7108 Entry.Node = Value;
7109 Args.push_back(Entry);
7110
7111 Entry.Ty = SizeTy;
7112 Entry.Node = Size;
7113 Args.push_back(Entry);
7114
7115 RTLIB::Libcall LibraryCall =
7116 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
7117 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
7118 report_fatal_error("Unsupported element size");
7119
7120 TargetLowering::CallLoweringInfo CLI(*this);
7121 CLI.setDebugLoc(dl)
7122 .setChain(Chain)
7123 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
7124 Type::getVoidTy(*getContext()),
7125 getExternalSymbol(TLI->getLibcallName(LibraryCall),
7126 TLI->getPointerTy(getDataLayout())),
7127 std::move(Args))
7128 .setDiscardResult()
7129 .setTailCall(isTailCall);
7130
7131 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
7132 return CallResult.second;
7133}
7134
7135SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
7136 SDVTList VTList, ArrayRef<SDValue> Ops,
7137 MachineMemOperand *MMO) {
7138 FoldingSetNodeID ID;
7139 ID.AddInteger(MemVT.getRawBits());
7140 AddNodeIDNode(ID, Opcode, VTList, Ops);
7141 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7142 void* IP = nullptr;
7143 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7144 cast<AtomicSDNode>(E)->refineAlignment(MMO);
7145 return SDValue(E, 0);
7146 }
7147
7148 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
7149 VTList, MemVT, MMO);
7150 createOperands(N, Ops);
7151
7152 CSEMap.InsertNode(N, IP);
7153 InsertNode(N);
7154 return SDValue(N, 0);
7155}
7156
7157SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
7158 EVT MemVT, SDVTList VTs, SDValue Chain,
7159 SDValue Ptr, SDValue Cmp, SDValue Swp,
7160 MachineMemOperand *MMO) {
7161 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||(static_cast <bool> (Opcode == ISD::ATOMIC_CMP_SWAP || Opcode
== ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS) ? void (0) : __assert_fail
("Opcode == ISD::ATOMIC_CMP_SWAP || Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7162, __extension__ __PRETTY_FUNCTION__))
7162 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS)(static_cast <bool> (Opcode == ISD::ATOMIC_CMP_SWAP || Opcode
== ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS) ? void (0) : __assert_fail
("Opcode == ISD::ATOMIC_CMP_SWAP || Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7162, __extension__ __PRETTY_FUNCTION__))
;
7163 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types")(static_cast <bool> (Cmp.getValueType() == Swp.getValueType
() && "Invalid Atomic Op Types") ? void (0) : __assert_fail
("Cmp.getValueType() == Swp.getValueType() && \"Invalid Atomic Op Types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7163, __extension__ __PRETTY_FUNCTION__))
;
7164
7165 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
7166 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
7167}
7168
7169SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
7170 SDValue Chain, SDValue Ptr, SDValue Val,
7171 MachineMemOperand *MMO) {
7172 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7173 Opcode == ISD::ATOMIC_LOAD_SUB ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7174 Opcode == ISD::ATOMIC_LOAD_AND ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7175 Opcode == ISD::ATOMIC_LOAD_CLR ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7176 Opcode == ISD::ATOMIC_LOAD_OR ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7177 Opcode == ISD::ATOMIC_LOAD_XOR ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7178 Opcode == ISD::ATOMIC_LOAD_NAND ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7179 Opcode == ISD::ATOMIC_LOAD_MIN ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7180 Opcode == ISD::ATOMIC_LOAD_MAX ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7181 Opcode == ISD::ATOMIC_LOAD_UMIN ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7182 Opcode == ISD::ATOMIC_LOAD_UMAX ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7183 Opcode == ISD::ATOMIC_LOAD_FADD ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7184 Opcode == ISD::ATOMIC_LOAD_FSUB ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7185 Opcode == ISD::ATOMIC_SWAP ||(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7186 Opcode == ISD::ATOMIC_STORE) &&(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
7187 "Invalid Atomic Op")(static_cast <bool> ((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND
|| Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR
|| Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND
|| Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
|| Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX
|| Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB
|| Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE
) && "Invalid Atomic Op") ? void (0) : __assert_fail (
"(Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR || Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD || Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7187, __extension__ __PRETTY_FUNCTION__))
;
7188
7189 EVT VT = Val.getValueType();
7190
7191 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
7192 getVTList(VT, MVT::Other);
7193 SDValue Ops[] = {Chain, Ptr, Val};
7194 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
7195}
7196
7197SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
7198 EVT VT, SDValue Chain, SDValue Ptr,
7199 MachineMemOperand *MMO) {
7200 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op")(static_cast <bool> (Opcode == ISD::ATOMIC_LOAD &&
"Invalid Atomic Op") ? void (0) : __assert_fail ("Opcode == ISD::ATOMIC_LOAD && \"Invalid Atomic Op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7200, __extension__ __PRETTY_FUNCTION__))
;
7201
7202 SDVTList VTs = getVTList(VT, MVT::Other);
7203 SDValue Ops[] = {Chain, Ptr};
7204 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
7205}
7206
7207/// getMergeValues - Create a MERGE_VALUES node from the given operands.
7208SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
7209 if (Ops.size() == 1)
7210 return Ops[0];
7211
7212 SmallVector<EVT, 4> VTs;
7213 VTs.reserve(Ops.size());
7214 for (const SDValue &Op : Ops)
7215 VTs.push_back(Op.getValueType());
7216 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
7217}
7218
7219SDValue SelectionDAG::getMemIntrinsicNode(
7220 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
7221 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
7222 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
7223 if (!Size && MemVT.isScalableVector())
7224 Size = MemoryLocation::UnknownSize;
7225 else if (!Size)
7226 Size = MemVT.getStoreSize();
7227
7228 MachineFunction &MF = getMachineFunction();
7229 MachineMemOperand *MMO =
7230 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
7231
7232 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
7233}
7234
7235SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
7236 SDVTList VTList,
7237 ArrayRef<SDValue> Ops, EVT MemVT,
7238 MachineMemOperand *MMO) {
7239 assert((Opcode == ISD::INTRINSIC_VOID ||(static_cast <bool> ((Opcode == ISD::INTRINSIC_VOID || Opcode
== ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int
)Opcode <= std::numeric_limits<int>::max() &&
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
"Opcode is not a memory-accessing opcode!") ? void (0) : __assert_fail
("(Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && \"Opcode is not a memory-accessing opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7244, __extension__ __PRETTY_FUNCTION__))
7240 Opcode == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Opcode == ISD::INTRINSIC_VOID || Opcode
== ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int
)Opcode <= std::numeric_limits<int>::max() &&
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
"Opcode is not a memory-accessing opcode!") ? void (0) : __assert_fail
("(Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && \"Opcode is not a memory-accessing opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7244, __extension__ __PRETTY_FUNCTION__))
7241 Opcode == ISD::PREFETCH ||(static_cast <bool> ((Opcode == ISD::INTRINSIC_VOID || Opcode
== ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int
)Opcode <= std::numeric_limits<int>::max() &&
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
"Opcode is not a memory-accessing opcode!") ? void (0) : __assert_fail
("(Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && \"Opcode is not a memory-accessing opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7244, __extension__ __PRETTY_FUNCTION__))
7242 ((int)Opcode <= std::numeric_limits<int>::max() &&(static_cast <bool> ((Opcode == ISD::INTRINSIC_VOID || Opcode
== ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int
)Opcode <= std::numeric_limits<int>::max() &&
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
"Opcode is not a memory-accessing opcode!") ? void (0) : __assert_fail
("(Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && \"Opcode is not a memory-accessing opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7244, __extension__ __PRETTY_FUNCTION__))
7243 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&(static_cast <bool> ((Opcode == ISD::INTRINSIC_VOID || Opcode
== ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int
)Opcode <= std::numeric_limits<int>::max() &&
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
"Opcode is not a memory-accessing opcode!") ? void (0) : __assert_fail
("(Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && \"Opcode is not a memory-accessing opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7244, __extension__ __PRETTY_FUNCTION__))
7244 "Opcode is not a memory-accessing opcode!")(static_cast <bool> ((Opcode == ISD::INTRINSIC_VOID || Opcode
== ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int
)Opcode <= std::numeric_limits<int>::max() &&
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
"Opcode is not a memory-accessing opcode!") ? void (0) : __assert_fail
("(Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && \"Opcode is not a memory-accessing opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7244, __extension__ __PRETTY_FUNCTION__))
;
7245
7246 // Memoize the node unless it returns a flag.
7247 MemIntrinsicSDNode *N;
7248 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
7249 FoldingSetNodeID ID;
7250 AddNodeIDNode(ID, Opcode, VTList, Ops);
7251 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
7252 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
7253 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7254 void *IP = nullptr;
7255 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7256 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
7257 return SDValue(E, 0);
7258 }
7259
7260 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
7261 VTList, MemVT, MMO);
7262 createOperands(N, Ops);
7263
7264 CSEMap.InsertNode(N, IP);
7265 } else {
7266 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
7267 VTList, MemVT, MMO);
7268 createOperands(N, Ops);
7269 }
7270 InsertNode(N);
7271 SDValue V(N, 0);
7272 NewSDValueDbgMsg(V, "Creating new node: ", this);
7273 return V;
7274}
7275
7276SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
7277 SDValue Chain, int FrameIndex,
7278 int64_t Size, int64_t Offset) {
7279 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
7280 const auto VTs = getVTList(MVT::Other);
7281 SDValue Ops[2] = {
7282 Chain,
7283 getFrameIndex(FrameIndex,
7284 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
7285 true)};
7286
7287 FoldingSetNodeID ID;
7288 AddNodeIDNode(ID, Opcode, VTs, Ops);
7289 ID.AddInteger(FrameIndex);
7290 ID.AddInteger(Size);
7291 ID.AddInteger(Offset);
7292 void *IP = nullptr;
7293 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
7294 return SDValue(E, 0);
7295
7296 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
7297 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
7298 createOperands(N, Ops);
7299 CSEMap.InsertNode(N, IP);
7300 InsertNode(N);
7301 SDValue V(N, 0);
7302 NewSDValueDbgMsg(V, "Creating new node: ", this);
7303 return V;
7304}
7305
7306SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
7307 uint64_t Guid, uint64_t Index,
7308 uint32_t Attr) {
7309 const unsigned Opcode = ISD::PSEUDO_PROBE;
7310 const auto VTs = getVTList(MVT::Other);
7311 SDValue Ops[] = {Chain};
7312 FoldingSetNodeID ID;
7313 AddNodeIDNode(ID, Opcode, VTs, Ops);
7314 ID.AddInteger(Guid);
7315 ID.AddInteger(Index);
7316 void *IP = nullptr;
7317 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
7318 return SDValue(E, 0);
7319
7320 auto *N = newSDNode<PseudoProbeSDNode>(
7321 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
7322 createOperands(N, Ops);
7323 CSEMap.InsertNode(N, IP);
7324 InsertNode(N);
7325 SDValue V(N, 0);
7326 NewSDValueDbgMsg(V, "Creating new node: ", this);
7327 return V;
7328}
7329
7330/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
7331/// MachinePointerInfo record from it. This is particularly useful because the
7332/// code generator has many cases where it doesn't bother passing in a
7333/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
7334static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
7335 SelectionDAG &DAG, SDValue Ptr,
7336 int64_t Offset = 0) {
7337 // If this is FI+Offset, we can model it.
7338 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
7339 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
7340 FI->getIndex(), Offset);
7341
7342 // If this is (FI+Offset1)+Offset2, we can model it.
7343 if (Ptr.getOpcode() != ISD::ADD ||
7344 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
7345 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
7346 return Info;
7347
7348 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
7349 return MachinePointerInfo::getFixedStack(
7350 DAG.getMachineFunction(), FI,
7351 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
7352}
7353
7354/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
7355/// MachinePointerInfo record from it. This is particularly useful because the
7356/// code generator has many cases where it doesn't bother passing in a
7357/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
7358static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
7359 SelectionDAG &DAG, SDValue Ptr,
7360 SDValue OffsetOp) {
7361 // If the 'Offset' value isn't a constant, we can't handle this.
7362 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
9
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
24
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
25
Assuming 'OffsetNode' is null
26
Taking false branch
7363 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
7364 if (OffsetOp.isUndef())
27
Calling 'SDValue::isUndef'
7365 return InferPointerInfo(Info, DAG, Ptr);
7366 return Info;
7367}
7368
7369SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
7370 EVT VT, const SDLoc &dl, SDValue Chain,
7371 SDValue Ptr, SDValue Offset,
7372 MachinePointerInfo PtrInfo, EVT MemVT,
7373 Align Alignment,
7374 MachineMemOperand::Flags MMOFlags,
7375 const AAMDNodes &AAInfo, const MDNode *Ranges) {
7376 assert(Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7377, __extension__ __PRETTY_FUNCTION__))
4
'?' condition is true
7377 "Invalid chain type")(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7377, __extension__ __PRETTY_FUNCTION__))
;
7378
7379 MMOFlags |= MachineMemOperand::MOLoad;
7380 assert((MMOFlags & MachineMemOperand::MOStore) == 0)(static_cast <bool> ((MMOFlags & MachineMemOperand::
MOStore) == 0) ? void (0) : __assert_fail ("(MMOFlags & MachineMemOperand::MOStore) == 0"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7380, __extension__ __PRETTY_FUNCTION__))
;
5
'?' condition is true
7381 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
7382 // clients.
7383 if (PtrInfo.V.isNull())
6
Taking true branch
7384 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
7
Value assigned to 'OffsetOp.Node'
8
Calling 'InferPointerInfo'
7385
7386 uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
7387 MachineFunction &MF = getMachineFunction();
7388 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
7389 Alignment, AAInfo, Ranges);
7390 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
7391}
7392
7393SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
7394 EVT VT, const SDLoc &dl, SDValue Chain,
7395 SDValue Ptr, SDValue Offset, EVT MemVT,
7396 MachineMemOperand *MMO) {
7397 if (VT == MemVT) {
7398 ExtType = ISD::NON_EXTLOAD;
7399 } else if (ExtType == ISD::NON_EXTLOAD) {
7400 assert(VT == MemVT && "Non-extending load from different memory type!")(static_cast <bool> (VT == MemVT && "Non-extending load from different memory type!"
) ? void (0) : __assert_fail ("VT == MemVT && \"Non-extending load from different memory type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7400, __extension__ __PRETTY_FUNCTION__))
;
7401 } else {
7402 // Extending load.
7403 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&(static_cast <bool> (MemVT.getScalarType().bitsLT(VT.getScalarType
()) && "Should only be an extending load, not truncating!"
) ? void (0) : __assert_fail ("MemVT.getScalarType().bitsLT(VT.getScalarType()) && \"Should only be an extending load, not truncating!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7404, __extension__ __PRETTY_FUNCTION__))
7404 "Should only be an extending load, not truncating!")(static_cast <bool> (MemVT.getScalarType().bitsLT(VT.getScalarType
()) && "Should only be an extending load, not truncating!"
) ? void (0) : __assert_fail ("MemVT.getScalarType().bitsLT(VT.getScalarType()) && \"Should only be an extending load, not truncating!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7404, __extension__ __PRETTY_FUNCTION__))
;
7405 assert(VT.isInteger() == MemVT.isInteger() &&(static_cast <bool> (VT.isInteger() == MemVT.isInteger(
) && "Cannot convert from FP to Int or Int -> FP!"
) ? void (0) : __assert_fail ("VT.isInteger() == MemVT.isInteger() && \"Cannot convert from FP to Int or Int -> FP!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7406, __extension__ __PRETTY_FUNCTION__))
7406 "Cannot convert from FP to Int or Int -> FP!")(static_cast <bool> (VT.isInteger() == MemVT.isInteger(
) && "Cannot convert from FP to Int or Int -> FP!"
) ? void (0) : __assert_fail ("VT.isInteger() == MemVT.isInteger() && \"Cannot convert from FP to Int or Int -> FP!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7406, __extension__ __PRETTY_FUNCTION__))
;
7407 assert(VT.isVector() == MemVT.isVector() &&(static_cast <bool> (VT.isVector() == MemVT.isVector() &&
"Cannot use an ext load to convert to or from a vector!") ? void
(0) : __assert_fail ("VT.isVector() == MemVT.isVector() && \"Cannot use an ext load to convert to or from a vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7408, __extension__ __PRETTY_FUNCTION__))
7408 "Cannot use an ext load to convert to or from a vector!")(static_cast <bool> (VT.isVector() == MemVT.isVector() &&
"Cannot use an ext load to convert to or from a vector!") ? void
(0) : __assert_fail ("VT.isVector() == MemVT.isVector() && \"Cannot use an ext load to convert to or from a vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7408, __extension__ __PRETTY_FUNCTION__))
;
7409 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == MemVT.getVectorElementCount()) && "Cannot use an ext load to change the number of vector elements!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == MemVT.getVectorElementCount()) && \"Cannot use an ext load to change the number of vector elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7411, __extension__ __PRETTY_FUNCTION__))
7410 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == MemVT.getVectorElementCount()) && "Cannot use an ext load to change the number of vector elements!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == MemVT.getVectorElementCount()) && \"Cannot use an ext load to change the number of vector elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7411, __extension__ __PRETTY_FUNCTION__))
7411 "Cannot use an ext load to change the number of vector elements!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == MemVT.getVectorElementCount()) && "Cannot use an ext load to change the number of vector elements!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == MemVT.getVectorElementCount()) && \"Cannot use an ext load to change the number of vector elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7411, __extension__ __PRETTY_FUNCTION__))
;
7412 }
7413
7414 bool Indexed = AM != ISD::UNINDEXED;
7415 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!")(static_cast <bool> ((Indexed || Offset.isUndef()) &&
"Unindexed load with an offset!") ? void (0) : __assert_fail
("(Indexed || Offset.isUndef()) && \"Unindexed load with an offset!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7415, __extension__ __PRETTY_FUNCTION__))
;
7416
7417 SDVTList VTs = Indexed ?
7418 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
7419 SDValue Ops[] = { Chain, Ptr, Offset };
7420 FoldingSetNodeID ID;
7421 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
7422 ID.AddInteger(MemVT.getRawBits());
7423 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
7424 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
7425 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7426 void *IP = nullptr;
7427 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7428 cast<LoadSDNode>(E)->refineAlignment(MMO);
7429 return SDValue(E, 0);
7430 }
7431 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7432 ExtType, MemVT, MMO);
7433 createOperands(N, Ops);
7434
7435 CSEMap.InsertNode(N, IP);
7436 InsertNode(N);
7437 SDValue V(N, 0);
7438 NewSDValueDbgMsg(V, "Creating new node: ", this);
7439 return V;
7440}
7441
7442SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7443 SDValue Ptr, MachinePointerInfo PtrInfo,
7444 MaybeAlign Alignment,
7445 MachineMemOperand::Flags MMOFlags,
7446 const AAMDNodes &AAInfo, const MDNode *Ranges) {
7447 SDValue Undef = getUNDEF(Ptr.getValueType());
7448 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
2
Calling 'SelectionDAG::getLoad'
7449 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
7450}
7451
7452SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7453 SDValue Ptr, MachineMemOperand *MMO) {
7454 SDValue Undef = getUNDEF(Ptr.getValueType());
7455 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
7456 VT, MMO);
7457}
7458
7459SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
7460 EVT VT, SDValue Chain, SDValue Ptr,
7461 MachinePointerInfo PtrInfo, EVT MemVT,
7462 MaybeAlign Alignment,
7463 MachineMemOperand::Flags MMOFlags,
7464 const AAMDNodes &AAInfo) {
7465 SDValue Undef = getUNDEF(Ptr.getValueType());
7466 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
7467 MemVT, Alignment, MMOFlags, AAInfo);
7468}
7469
7470SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
7471 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
7472 MachineMemOperand *MMO) {
7473 SDValue Undef = getUNDEF(Ptr.getValueType());
7474 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
7475 MemVT, MMO);
7476}
7477
7478SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
7479 SDValue Base, SDValue Offset,
7480 ISD::MemIndexedMode AM) {
7481 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
7482 assert(LD->getOffset().isUndef() && "Load is already a indexed load!")(static_cast <bool> (LD->getOffset().isUndef() &&
"Load is already a indexed load!") ? void (0) : __assert_fail
("LD->getOffset().isUndef() && \"Load is already a indexed load!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7482, __extension__ __PRETTY_FUNCTION__))
;
7483 // Don't propagate the invariant or dereferenceable flags.
7484 auto MMOFlags =
7485 LD->getMemOperand()->getFlags() &
7486 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
7487 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
7488 LD->getChain(), Base, Offset, LD->getPointerInfo(),
7489 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
7490}
7491
7492SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7493 SDValue Ptr, MachinePointerInfo PtrInfo,
7494 Align Alignment,
7495 MachineMemOperand::Flags MMOFlags,
7496 const AAMDNodes &AAInfo) {
7497 assert(Chain.getValueType() == MVT::Other && "Invalid chain type")(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7497, __extension__ __PRETTY_FUNCTION__))
;
7498
7499 MMOFlags |= MachineMemOperand::MOStore;
7500 assert((MMOFlags & MachineMemOperand::MOLoad) == 0)(static_cast <bool> ((MMOFlags & MachineMemOperand::
MOLoad) == 0) ? void (0) : __assert_fail ("(MMOFlags & MachineMemOperand::MOLoad) == 0"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7500, __extension__ __PRETTY_FUNCTION__))
;
7501
7502 if (PtrInfo.V.isNull())
7503 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7504
7505 MachineFunction &MF = getMachineFunction();
7506 uint64_t Size =
7507 MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
7508 MachineMemOperand *MMO =
7509 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
7510 return getStore(Chain, dl, Val, Ptr, MMO);
7511}
7512
7513SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7514 SDValue Ptr, MachineMemOperand *MMO) {
7515 assert(Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7516, __extension__ __PRETTY_FUNCTION__))
7516 "Invalid chain type")(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7516, __extension__ __PRETTY_FUNCTION__))
;
7517 EVT VT = Val.getValueType();
7518 SDVTList VTs = getVTList(MVT::Other);
7519 SDValue Undef = getUNDEF(Ptr.getValueType());
7520 SDValue Ops[] = { Chain, Val, Ptr, Undef };
7521 FoldingSetNodeID ID;
7522 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7523 ID.AddInteger(VT.getRawBits());
7524 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7525 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
7526 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7527 void *IP = nullptr;
7528 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7529 cast<StoreSDNode>(E)->refineAlignment(MMO);
7530 return SDValue(E, 0);
7531 }
7532 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7533 ISD::UNINDEXED, false, VT, MMO);
7534 createOperands(N, Ops);
7535
7536 CSEMap.InsertNode(N, IP);
7537 InsertNode(N);
7538 SDValue V(N, 0);
7539 NewSDValueDbgMsg(V, "Creating new node: ", this);
7540 return V;
7541}
7542
7543SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7544 SDValue Ptr, MachinePointerInfo PtrInfo,
7545 EVT SVT, Align Alignment,
7546 MachineMemOperand::Flags MMOFlags,
7547 const AAMDNodes &AAInfo) {
7548 assert(Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7549, __extension__ __PRETTY_FUNCTION__))
7549 "Invalid chain type")(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7549, __extension__ __PRETTY_FUNCTION__))
;
7550
7551 MMOFlags |= MachineMemOperand::MOStore;
7552 assert((MMOFlags & MachineMemOperand::MOLoad) == 0)(static_cast <bool> ((MMOFlags & MachineMemOperand::
MOLoad) == 0) ? void (0) : __assert_fail ("(MMOFlags & MachineMemOperand::MOLoad) == 0"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7552, __extension__ __PRETTY_FUNCTION__))
;
7553
7554 if (PtrInfo.V.isNull())
7555 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7556
7557 MachineFunction &MF = getMachineFunction();
7558 MachineMemOperand *MMO = MF.getMachineMemOperand(
7559 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
7560 Alignment, AAInfo);
7561 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
7562}
7563
7564SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7565 SDValue Ptr, EVT SVT,
7566 MachineMemOperand *MMO) {
7567 EVT VT = Val.getValueType();
7568
7569 assert(Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7570, __extension__ __PRETTY_FUNCTION__))
7570 "Invalid chain type")(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7570, __extension__ __PRETTY_FUNCTION__))
;
7571 if (VT == SVT)
7572 return getStore(Chain, dl, Val, Ptr, MMO);
7573
7574 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&(static_cast <bool> (SVT.getScalarType().bitsLT(VT.getScalarType
()) && "Should only be a truncating store, not extending!"
) ? void (0) : __assert_fail ("SVT.getScalarType().bitsLT(VT.getScalarType()) && \"Should only be a truncating store, not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7575, __extension__ __PRETTY_FUNCTION__))
7575 "Should only be a truncating store, not extending!")(static_cast <bool> (SVT.getScalarType().bitsLT(VT.getScalarType
()) && "Should only be a truncating store, not extending!"
) ? void (0) : __assert_fail ("SVT.getScalarType().bitsLT(VT.getScalarType()) && \"Should only be a truncating store, not extending!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7575, __extension__ __PRETTY_FUNCTION__))
;
7576 assert(VT.isInteger() == SVT.isInteger() &&(static_cast <bool> (VT.isInteger() == SVT.isInteger() &&
"Can't do FP-INT conversion!") ? void (0) : __assert_fail ("VT.isInteger() == SVT.isInteger() && \"Can't do FP-INT conversion!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7577, __extension__ __PRETTY_FUNCTION__))
7577 "Can't do FP-INT conversion!")(static_cast <bool> (VT.isInteger() == SVT.isInteger() &&
"Can't do FP-INT conversion!") ? void (0) : __assert_fail ("VT.isInteger() == SVT.isInteger() && \"Can't do FP-INT conversion!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7577, __extension__ __PRETTY_FUNCTION__))
;
7578 assert(VT.isVector() == SVT.isVector() &&(static_cast <bool> (VT.isVector() == SVT.isVector() &&
"Cannot use trunc store to convert to or from a vector!") ? void
(0) : __assert_fail ("VT.isVector() == SVT.isVector() && \"Cannot use trunc store to convert to or from a vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7579, __extension__ __PRETTY_FUNCTION__))
7579 "Cannot use trunc store to convert to or from a vector!")(static_cast <bool> (VT.isVector() == SVT.isVector() &&
"Cannot use trunc store to convert to or from a vector!") ? void
(0) : __assert_fail ("VT.isVector() == SVT.isVector() && \"Cannot use trunc store to convert to or from a vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7579, __extension__ __PRETTY_FUNCTION__))
;
7580 assert((!VT.isVector() ||(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == SVT.getVectorElementCount()) && "Cannot use trunc store to change the number of vector elements!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == SVT.getVectorElementCount()) && \"Cannot use trunc store to change the number of vector elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7582, __extension__ __PRETTY_FUNCTION__))
7581 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == SVT.getVectorElementCount()) && "Cannot use trunc store to change the number of vector elements!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == SVT.getVectorElementCount()) && \"Cannot use trunc store to change the number of vector elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7582, __extension__ __PRETTY_FUNCTION__))
7582 "Cannot use trunc store to change the number of vector elements!")(static_cast <bool> ((!VT.isVector() || VT.getVectorElementCount
() == SVT.getVectorElementCount()) && "Cannot use trunc store to change the number of vector elements!"
) ? void (0) : __assert_fail ("(!VT.isVector() || VT.getVectorElementCount() == SVT.getVectorElementCount()) && \"Cannot use trunc store to change the number of vector elements!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7582, __extension__ __PRETTY_FUNCTION__))
;
7583
7584 SDVTList VTs = getVTList(MVT::Other);
7585 SDValue Undef = getUNDEF(Ptr.getValueType());
7586 SDValue Ops[] = { Chain, Val, Ptr, Undef };
7587 FoldingSetNodeID ID;
7588 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7589 ID.AddInteger(SVT.getRawBits());
7590 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7591 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
7592 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7593 void *IP = nullptr;
7594 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7595 cast<StoreSDNode>(E)->refineAlignment(MMO);
7596 return SDValue(E, 0);
7597 }
7598 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7599 ISD::UNINDEXED, true, SVT, MMO);
7600 createOperands(N, Ops);
7601
7602 CSEMap.InsertNode(N, IP);
7603 InsertNode(N);
7604 SDValue V(N, 0);
7605 NewSDValueDbgMsg(V, "Creating new node: ", this);
7606 return V;
7607}
7608
7609SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
7610 SDValue Base, SDValue Offset,
7611 ISD::MemIndexedMode AM) {
7612 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
7613 assert(ST->getOffset().isUndef() && "Store is already a indexed store!")(static_cast <bool> (ST->getOffset().isUndef() &&
"Store is already a indexed store!") ? void (0) : __assert_fail
("ST->getOffset().isUndef() && \"Store is already a indexed store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7613, __extension__ __PRETTY_FUNCTION__))
;
7614 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
7615 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
7616 FoldingSetNodeID ID;
7617 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7618 ID.AddInteger(ST->getMemoryVT().getRawBits());
7619 ID.AddInteger(ST->getRawSubclassData());
7620 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
7621 void *IP = nullptr;
7622 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
7623 return SDValue(E, 0);
7624
7625 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7626 ST->isTruncatingStore(), ST->getMemoryVT(),
7627 ST->getMemOperand());
7628 createOperands(N, Ops);
7629
7630 CSEMap.InsertNode(N, IP);
7631 InsertNode(N);
7632 SDValue V(N, 0);
7633 NewSDValueDbgMsg(V, "Creating new node: ", this);
7634 return V;
7635}
7636
7637SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7638 SDValue Base, SDValue Offset, SDValue Mask,
7639 SDValue PassThru, EVT MemVT,
7640 MachineMemOperand *MMO,
7641 ISD::MemIndexedMode AM,
7642 ISD::LoadExtType ExtTy, bool isExpanding) {
7643 bool Indexed = AM != ISD::UNINDEXED;
7644 assert((Indexed || Offset.isUndef()) &&(static_cast <bool> ((Indexed || Offset.isUndef()) &&
"Unindexed masked load with an offset!") ? void (0) : __assert_fail
("(Indexed || Offset.isUndef()) && \"Unindexed masked load with an offset!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7645, __extension__ __PRETTY_FUNCTION__))
7645 "Unindexed masked load with an offset!")(static_cast <bool> ((Indexed || Offset.isUndef()) &&
"Unindexed masked load with an offset!") ? void (0) : __assert_fail
("(Indexed || Offset.isUndef()) && \"Unindexed masked load with an offset!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7645, __extension__ __PRETTY_FUNCTION__))
;
7646 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
7647 : getVTList(VT, MVT::Other);
7648 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
7649 FoldingSetNodeID ID;
7650 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
7651 ID.AddInteger(MemVT.getRawBits());
7652 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
7653 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
7654 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7655 void *IP = nullptr;
7656 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7657 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
7658 return SDValue(E, 0);
7659 }
7660 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7661 AM, ExtTy, isExpanding, MemVT, MMO);
7662 createOperands(N, Ops);
7663
7664 CSEMap.InsertNode(N, IP);
7665 InsertNode(N);
7666 SDValue V(N, 0);
7667 NewSDValueDbgMsg(V, "Creating new node: ", this);
7668 return V;
7669}
7670
7671SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
7672 SDValue Base, SDValue Offset,
7673 ISD::MemIndexedMode AM) {
7674 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
7675 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!")(static_cast <bool> (LD->getOffset().isUndef() &&
"Masked load is already a indexed load!") ? void (0) : __assert_fail
("LD->getOffset().isUndef() && \"Masked load is already a indexed load!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7675, __extension__ __PRETTY_FUNCTION__))
;
7676 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
7677 Offset, LD->getMask(), LD->getPassThru(),
7678 LD->getMemoryVT(), LD->getMemOperand(), AM,
7679 LD->getExtensionType(), LD->isExpandingLoad());
7680}
7681
7682SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
7683 SDValue Val, SDValue Base, SDValue Offset,
7684 SDValue Mask, EVT MemVT,
7685 MachineMemOperand *MMO,
7686 ISD::MemIndexedMode AM, bool IsTruncating,
7687 bool IsCompressing) {
7688 assert(Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7689, __extension__ __PRETTY_FUNCTION__))
7689 "Invalid chain type")(static_cast <bool> (Chain.getValueType() == MVT::Other
&& "Invalid chain type") ? void (0) : __assert_fail (
"Chain.getValueType() == MVT::Other && \"Invalid chain type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7689, __extension__ __PRETTY_FUNCTION__))
;
7690 bool Indexed = AM != ISD::UNINDEXED;
7691 assert((Indexed || Offset.isUndef()) &&(static_cast <bool> ((Indexed || Offset.isUndef()) &&
"Unindexed masked store with an offset!") ? void (0) : __assert_fail
("(Indexed || Offset.isUndef()) && \"Unindexed masked store with an offset!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7692, __extension__ __PRETTY_FUNCTION__))
7692 "Unindexed masked store with an offset!")(static_cast <bool> ((Indexed || Offset.isUndef()) &&
"Unindexed masked store with an offset!") ? void (0) : __assert_fail
("(Indexed || Offset.isUndef()) && \"Unindexed masked store with an offset!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7692, __extension__ __PRETTY_FUNCTION__))
;
7693 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
7694 : getVTList(MVT::Other);
7695 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
7696 FoldingSetNodeID ID;
7697 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
7698 ID.AddInteger(MemVT.getRawBits());
7699 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
7700 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
7701 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7702 void *IP = nullptr;
7703 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7704 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
7705 return SDValue(E, 0);
7706 }
7707 auto *N =
7708 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7709 IsTruncating, IsCompressing, MemVT, MMO);
7710 createOperands(N, Ops);
7711
7712 CSEMap.InsertNode(N, IP);
7713 InsertNode(N);
7714 SDValue V(N, 0);
7715 NewSDValueDbgMsg(V, "Creating new node: ", this);
7716 return V;
7717}
7718
7719SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
7720 SDValue Base, SDValue Offset,
7721 ISD::MemIndexedMode AM) {
7722 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
7723 assert(ST->getOffset().isUndef() &&(static_cast <bool> (ST->getOffset().isUndef() &&
"Masked store is already a indexed store!") ? void (0) : __assert_fail
("ST->getOffset().isUndef() && \"Masked store is already a indexed store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7724, __extension__ __PRETTY_FUNCTION__))
7724 "Masked store is already a indexed store!")(static_cast <bool> (ST->getOffset().isUndef() &&
"Masked store is already a indexed store!") ? void (0) : __assert_fail
("ST->getOffset().isUndef() && \"Masked store is already a indexed store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7724, __extension__ __PRETTY_FUNCTION__))
;
7725 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
7726 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
7727 AM, ST->isTruncatingStore(), ST->isCompressingStore());
7728}
7729
7730SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
7731 ArrayRef<SDValue> Ops,
7732 MachineMemOperand *MMO,
7733 ISD::MemIndexType IndexType,
7734 ISD::LoadExtType ExtTy) {
7735 assert(Ops.size() == 6 && "Incompatible number of operands")(static_cast <bool> (Ops.size() == 6 && "Incompatible number of operands"
) ? void (0) : __assert_fail ("Ops.size() == 6 && \"Incompatible number of operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7735, __extension__ __PRETTY_FUNCTION__))
;
7736
7737 FoldingSetNodeID ID;
7738 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
7739 ID.AddInteger(MemVT.getRawBits());
7740 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
7741 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
7742 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7743 void *IP = nullptr;
7744 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7745 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
7746 return SDValue(E, 0);
7747 }
7748
7749 IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]);
7750 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7751 VTs, MemVT, MMO, IndexType, ExtTy);
7752 createOperands(N, Ops);
7753
7754 assert(N->getPassThru().getValueType() == N->getValueType(0) &&(static_cast <bool> (N->getPassThru().getValueType()
== N->getValueType(0) && "Incompatible type of the PassThru value in MaskedGatherSDNode"
) ? void (0) : __assert_fail ("N->getPassThru().getValueType() == N->getValueType(0) && \"Incompatible type of the PassThru value in MaskedGatherSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7755, __extension__ __PRETTY_FUNCTION__))
7755 "Incompatible type of the PassThru value in MaskedGatherSDNode")(static_cast <bool> (N->getPassThru().getValueType()
== N->getValueType(0) && "Incompatible type of the PassThru value in MaskedGatherSDNode"
) ? void (0) : __assert_fail ("N->getPassThru().getValueType() == N->getValueType(0) && \"Incompatible type of the PassThru value in MaskedGatherSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7755, __extension__ __PRETTY_FUNCTION__))
;
7756 assert(N->getMask().getValueType().getVectorElementCount() ==(static_cast <bool> (N->getMask().getValueType().getVectorElementCount
() == N->getValueType(0).getVectorElementCount() &&
"Vector width mismatch between mask and data") ? void (0) : __assert_fail
("N->getMask().getValueType().getVectorElementCount() == N->getValueType(0).getVectorElementCount() && \"Vector width mismatch between mask and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7758, __extension__ __PRETTY_FUNCTION__))
7757 N->getValueType(0).getVectorElementCount() &&(static_cast <bool> (N->getMask().getValueType().getVectorElementCount
() == N->getValueType(0).getVectorElementCount() &&
"Vector width mismatch between mask and data") ? void (0) : __assert_fail
("N->getMask().getValueType().getVectorElementCount() == N->getValueType(0).getVectorElementCount() && \"Vector width mismatch between mask and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7758, __extension__ __PRETTY_FUNCTION__))
7758 "Vector width mismatch between mask and data")(static_cast <bool> (N->getMask().getValueType().getVectorElementCount
() == N->getValueType(0).getVectorElementCount() &&
"Vector width mismatch between mask and data") ? void (0) : __assert_fail
("N->getMask().getValueType().getVectorElementCount() == N->getValueType(0).getVectorElementCount() && \"Vector width mismatch between mask and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7758, __extension__ __PRETTY_FUNCTION__))
;
7759 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==(static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValueType(0).getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValueType(0).getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7761, __extension__ __PRETTY_FUNCTION__))
7760 N->getValueType(0).getVectorElementCount().isScalable() &&(static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValueType(0).getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValueType(0).getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7761, __extension__ __PRETTY_FUNCTION__))
7761 "Scalable flags of index and data do not match")(static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValueType(0).getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValueType(0).getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7761, __extension__ __PRETTY_FUNCTION__))
;
7762 assert(ElementCount::isKnownGE((static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValueType
(0).getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValueType(0).getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7765, __extension__ __PRETTY_FUNCTION__))
7763 N->getIndex().getValueType().getVectorElementCount(),(static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValueType
(0).getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValueType(0).getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7765, __extension__ __PRETTY_FUNCTION__))
7764 N->getValueType(0).getVectorElementCount()) &&(static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValueType
(0).getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValueType(0).getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7765, __extension__ __PRETTY_FUNCTION__))
7765 "Vector width mismatch between index and data")(static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValueType
(0).getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValueType(0).getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7765, __extension__ __PRETTY_FUNCTION__))
;
7766 assert(isa<ConstantSDNode>(N->getScale()) &&(static_cast <bool> (isa<ConstantSDNode>(N->getScale
()) && cast<ConstantSDNode>(N->getScale())->
getAPIntValue().isPowerOf2() && "Scale should be a constant power of 2"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && \"Scale should be a constant power of 2\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7768, __extension__ __PRETTY_FUNCTION__))
7767 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&(static_cast <bool> (isa<ConstantSDNode>(N->getScale
()) && cast<ConstantSDNode>(N->getScale())->
getAPIntValue().isPowerOf2() && "Scale should be a constant power of 2"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && \"Scale should be a constant power of 2\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7768, __extension__ __PRETTY_FUNCTION__))
7768 "Scale should be a constant power of 2")(static_cast <bool> (isa<ConstantSDNode>(N->getScale
()) && cast<ConstantSDNode>(N->getScale())->
getAPIntValue().isPowerOf2() && "Scale should be a constant power of 2"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && \"Scale should be a constant power of 2\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7768, __extension__ __PRETTY_FUNCTION__))
;
7769
7770 CSEMap.InsertNode(N, IP);
7771 InsertNode(N);
7772 SDValue V(N, 0);
7773 NewSDValueDbgMsg(V, "Creating new node: ", this);
7774 return V;
7775}
7776
7777SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
7778 ArrayRef<SDValue> Ops,
7779 MachineMemOperand *MMO,
7780 ISD::MemIndexType IndexType,
7781 bool IsTrunc) {
7782 assert(Ops.size() == 6 && "Incompatible number of operands")(static_cast <bool> (Ops.size() == 6 && "Incompatible number of operands"
) ? void (0) : __assert_fail ("Ops.size() == 6 && \"Incompatible number of operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7782, __extension__ __PRETTY_FUNCTION__))
;
7783
7784 FoldingSetNodeID ID;
7785 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
7786 ID.AddInteger(MemVT.getRawBits());
7787 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
7788 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
7789 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7790 void *IP = nullptr;
7791 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7792 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
7793 return SDValue(E, 0);
7794 }
7795
7796 IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]);
7797 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7798 VTs, MemVT, MMO, IndexType, IsTrunc);
7799 createOperands(N, Ops);
7800
7801 assert(N->getMask().getValueType().getVectorElementCount() ==(static_cast <bool> (N->getMask().getValueType().getVectorElementCount
() == N->getValue().getValueType().getVectorElementCount()
&& "Vector width mismatch between mask and data") ? void
(0) : __assert_fail ("N->getMask().getValueType().getVectorElementCount() == N->getValue().getValueType().getVectorElementCount() && \"Vector width mismatch between mask and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7803, __extension__ __PRETTY_FUNCTION__))
7802 N->getValue().getValueType().getVectorElementCount() &&(static_cast <bool> (N->getMask().getValueType().getVectorElementCount
() == N->getValue().getValueType().getVectorElementCount()
&& "Vector width mismatch between mask and data") ? void
(0) : __assert_fail ("N->getMask().getValueType().getVectorElementCount() == N->getValue().getValueType().getVectorElementCount() && \"Vector width mismatch between mask and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7803, __extension__ __PRETTY_FUNCTION__))
7803 "Vector width mismatch between mask and data")(static_cast <bool> (N->getMask().getValueType().getVectorElementCount
() == N->getValue().getValueType().getVectorElementCount()
&& "Vector width mismatch between mask and data") ? void
(0) : __assert_fail ("N->getMask().getValueType().getVectorElementCount() == N->getValue().getValueType().getVectorElementCount() && \"Vector width mismatch between mask and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7803, __extension__ __PRETTY_FUNCTION__))
;
7804 assert((static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValue().getValueType().getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValue().getValueType().getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7807, __extension__ __PRETTY_FUNCTION__))
7805 N->getIndex().getValueType().getVectorElementCount().isScalable() ==(static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValue().getValueType().getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValue().getValueType().getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7807, __extension__ __PRETTY_FUNCTION__))
7806 N->getValue().getValueType().getVectorElementCount().isScalable() &&(static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValue().getValueType().getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValue().getValueType().getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7807, __extension__ __PRETTY_FUNCTION__))
7807 "Scalable flags of index and data do not match")(static_cast <bool> (N->getIndex().getValueType().getVectorElementCount
().isScalable() == N->getValue().getValueType().getVectorElementCount
().isScalable() && "Scalable flags of index and data do not match"
) ? void (0) : __assert_fail ("N->getIndex().getValueType().getVectorElementCount().isScalable() == N->getValue().getValueType().getVectorElementCount().isScalable() && \"Scalable flags of index and data do not match\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7807, __extension__ __PRETTY_FUNCTION__))
;
7808 assert(ElementCount::isKnownGE((static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValue().getValueType
().getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValue().getValueType().getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7811, __extension__ __PRETTY_FUNCTION__))
7809 N->getIndex().getValueType().getVectorElementCount(),(static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValue().getValueType
().getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValue().getValueType().getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7811, __extension__ __PRETTY_FUNCTION__))
7810 N->getValue().getValueType().getVectorElementCount()) &&(static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValue().getValueType
().getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValue().getValueType().getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7811, __extension__ __PRETTY_FUNCTION__))
7811 "Vector width mismatch between index and data")(static_cast <bool> (ElementCount::isKnownGE( N->getIndex
().getValueType().getVectorElementCount(), N->getValue().getValueType
().getVectorElementCount()) && "Vector width mismatch between index and data"
) ? void (0) : __assert_fail ("ElementCount::isKnownGE( N->getIndex().getValueType().getVectorElementCount(), N->getValue().getValueType().getVectorElementCount()) && \"Vector width mismatch between index and data\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7811, __extension__ __PRETTY_FUNCTION__))
;
7812 assert(isa<ConstantSDNode>(N->getScale()) &&(static_cast <bool> (isa<ConstantSDNode>(N->getScale
()) && cast<ConstantSDNode>(N->getScale())->
getAPIntValue().isPowerOf2() && "Scale should be a constant power of 2"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && \"Scale should be a constant power of 2\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7814, __extension__ __PRETTY_FUNCTION__))
7813 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&(static_cast <bool> (isa<ConstantSDNode>(N->getScale
()) && cast<ConstantSDNode>(N->getScale())->
getAPIntValue().isPowerOf2() && "Scale should be a constant power of 2"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && \"Scale should be a constant power of 2\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7814, __extension__ __PRETTY_FUNCTION__))
7814 "Scale should be a constant power of 2")(static_cast <bool> (isa<ConstantSDNode>(N->getScale
()) && cast<ConstantSDNode>(N->getScale())->
getAPIntValue().isPowerOf2() && "Scale should be a constant power of 2"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && \"Scale should be a constant power of 2\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7814, __extension__ __PRETTY_FUNCTION__))
;
7815
7816 CSEMap.InsertNode(N, IP);
7817 InsertNode(N);
7818 SDValue V(N, 0);
7819 NewSDValueDbgMsg(V, "Creating new node: ", this);
7820 return V;
7821}
7822
7823SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7824 // select undef, T, F --> T (if T is a constant), otherwise F
7825 // select, ?, undef, F --> F
7826 // select, ?, T, undef --> T
7827 if (Cond.isUndef())
7828 return isConstantValueOfAnyType(T) ? T : F;
7829 if (T.isUndef())
7830 return F;
7831 if (F.isUndef())
7832 return T;
7833
7834 // select true, T, F --> T
7835 // select false, T, F --> F
7836 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
7837 return CondC->isNullValue() ? F : T;
7838
7839 // TODO: This should simplify VSELECT with constant condition using something
7840 // like this (but check boolean contents to be complete?):
7841 // if (ISD::isBuildVectorAllOnes(Cond.getNode()))
7842 // return T;
7843 // if (ISD::isBuildVectorAllZeros(Cond.getNode()))
7844 // return F;
7845
7846 // select ?, T, T --> T
7847 if (T == F)
7848 return T;
7849
7850 return SDValue();
7851}
7852
7853SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7854 // shift undef, Y --> 0 (can always assume that the undef value is 0)
7855 if (X.isUndef())
7856 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
7857 // shift X, undef --> undef (because it may shift by the bitwidth)
7858 if (Y.isUndef())
7859 return getUNDEF(X.getValueType());
7860
7861 // shift 0, Y --> 0
7862 // shift X, 0 --> X
7863 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))
7864 return X;
7865
7866 // shift X, C >= bitwidth(X) --> undef
7867 // All vector elements must be too big (or undef) to avoid partial undefs.
7868 auto isShiftTooBig = [X](ConstantSDNode *Val) {
7869 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
7870 };
7871 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
7872 return getUNDEF(X.getValueType());
7873
7874 return SDValue();
7875}
7876
7877SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
7878 SDNodeFlags Flags) {
7879 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
7880 // (an undef operand can be chosen to be Nan/Inf), then the result of this
7881 // operation is poison. That result can be relaxed to undef.
7882 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
7883 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
7884 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
7885 (YC && YC->getValueAPF().isNaN());
7886 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
7887 (YC && YC->getValueAPF().isInfinity());
7888
7889 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
7890 return getUNDEF(X.getValueType());
7891
7892 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
7893 return getUNDEF(X.getValueType());
7894
7895 if (!YC)
7896 return SDValue();
7897
7898 // X + -0.0 --> X
7899 if (Opcode == ISD::FADD)
7900 if (YC->getValueAPF().isNegZero())
7901 return X;
7902
7903 // X - +0.0 --> X
7904 if (Opcode == ISD::FSUB)
7905 if (YC->getValueAPF().isPosZero())
7906 return X;
7907
7908 // X * 1.0 --> X
7909 // X / 1.0 --> X
7910 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
7911 if (YC->getValueAPF().isExactlyValue(1.0))
7912 return X;
7913
7914 // X * 0.0 --> 0.0
7915 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
7916 if (YC->getValueAPF().isZero())
7917 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
7918
7919 return SDValue();
7920}
7921
7922SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
7923 SDValue Ptr, SDValue SV, unsigned Align) {
7924 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
7925 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
7926}
7927
7928SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7929 ArrayRef<SDUse> Ops) {
7930 switch (Ops.size()) {
7931 case 0: return getNode(Opcode, DL, VT);
7932 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
7933 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
7934 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
7935 default: break;
7936 }
7937
7938 // Copy from an SDUse array into an SDValue array for use with
7939 // the regular getNode logic.
7940 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
7941 return getNode(Opcode, DL, VT, NewOps);
7942}
7943
7944SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7945 ArrayRef<SDValue> Ops) {
7946 SDNodeFlags Flags;
7947 if (Inserter)
7948 Flags = Inserter->getFlags();
7949 return getNode(Opcode, DL, VT, Ops, Flags);
7950}
7951
7952SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7953 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
7954 unsigned NumOps = Ops.size();
7955 switch (NumOps) {
7956 case 0: return getNode(Opcode, DL, VT);
7957 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
7958 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
7959 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
7960 default: break;
7961 }
7962
7963#ifndef NDEBUG
7964 for (auto &Op : Ops)
7965 assert(Op.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (Op.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Op.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7966, __extension__ __PRETTY_FUNCTION__))
7966 "Operand is DELETED_NODE!")(static_cast <bool> (Op.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Op.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7966, __extension__ __PRETTY_FUNCTION__))
;
7967#endif
7968
7969 switch (Opcode) {
7970 default: break;
7971 case ISD::BUILD_VECTOR:
7972 // Attempt to simplify BUILD_VECTOR.
7973 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7974 return V;
7975 break;
7976 case ISD::CONCAT_VECTORS:
7977 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7978 return V;
7979 break;
7980 case ISD::SELECT_CC:
7981 assert(NumOps == 5 && "SELECT_CC takes 5 operands!")(static_cast <bool> (NumOps == 5 && "SELECT_CC takes 5 operands!"
) ? void (0) : __assert_fail ("NumOps == 5 && \"SELECT_CC takes 5 operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7981, __extension__ __PRETTY_FUNCTION__))
;
7982 assert(Ops[0].getValueType() == Ops[1].getValueType() &&(static_cast <bool> (Ops[0].getValueType() == Ops[1].getValueType
() && "LHS and RHS of condition must have same type!"
) ? void (0) : __assert_fail ("Ops[0].getValueType() == Ops[1].getValueType() && \"LHS and RHS of condition must have same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7983, __extension__ __PRETTY_FUNCTION__))
7983 "LHS and RHS of condition must have same type!")(static_cast <bool> (Ops[0].getValueType() == Ops[1].getValueType
() && "LHS and RHS of condition must have same type!"
) ? void (0) : __assert_fail ("Ops[0].getValueType() == Ops[1].getValueType() && \"LHS and RHS of condition must have same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7983, __extension__ __PRETTY_FUNCTION__))
;
7984 assert(Ops[2].getValueType() == Ops[3].getValueType() &&(static_cast <bool> (Ops[2].getValueType() == Ops[3].getValueType
() && "True and False arms of SelectCC must have same type!"
) ? void (0) : __assert_fail ("Ops[2].getValueType() == Ops[3].getValueType() && \"True and False arms of SelectCC must have same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7985, __extension__ __PRETTY_FUNCTION__))
7985 "True and False arms of SelectCC must have same type!")(static_cast <bool> (Ops[2].getValueType() == Ops[3].getValueType
() && "True and False arms of SelectCC must have same type!"
) ? void (0) : __assert_fail ("Ops[2].getValueType() == Ops[3].getValueType() && \"True and False arms of SelectCC must have same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7985, __extension__ __PRETTY_FUNCTION__))
;
7986 assert(Ops[2].getValueType() == VT &&(static_cast <bool> (Ops[2].getValueType() == VT &&
"select_cc node must be of same type as true and false value!"
) ? void (0) : __assert_fail ("Ops[2].getValueType() == VT && \"select_cc node must be of same type as true and false value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7987, __extension__ __PRETTY_FUNCTION__))
7987 "select_cc node must be of same type as true and false value!")(static_cast <bool> (Ops[2].getValueType() == VT &&
"select_cc node must be of same type as true and false value!"
) ? void (0) : __assert_fail ("Ops[2].getValueType() == VT && \"select_cc node must be of same type as true and false value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7987, __extension__ __PRETTY_FUNCTION__))
;
7988 break;
7989 case ISD::BR_CC:
7990 assert(NumOps == 5 && "BR_CC takes 5 operands!")(static_cast <bool> (NumOps == 5 && "BR_CC takes 5 operands!"
) ? void (0) : __assert_fail ("NumOps == 5 && \"BR_CC takes 5 operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7990, __extension__ __PRETTY_FUNCTION__))
;
7991 assert(Ops[2].getValueType() == Ops[3].getValueType() &&(static_cast <bool> (Ops[2].getValueType() == Ops[3].getValueType
() && "LHS/RHS of comparison should match types!") ? void
(0) : __assert_fail ("Ops[2].getValueType() == Ops[3].getValueType() && \"LHS/RHS of comparison should match types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7992, __extension__ __PRETTY_FUNCTION__))
7992 "LHS/RHS of comparison should match types!")(static_cast <bool> (Ops[2].getValueType() == Ops[3].getValueType
() && "LHS/RHS of comparison should match types!") ? void
(0) : __assert_fail ("Ops[2].getValueType() == Ops[3].getValueType() && \"LHS/RHS of comparison should match types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 7992, __extension__ __PRETTY_FUNCTION__))
;
7993 break;
7994 }
7995
7996 // Memoize nodes.
7997 SDNode *N;
7998 SDVTList VTs = getVTList(VT);
7999
8000 if (VT != MVT::Glue) {
8001 FoldingSetNodeID ID;
8002 AddNodeIDNode(ID, Opcode, VTs, Ops);
8003 void *IP = nullptr;
8004
8005 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
8006 return SDValue(E, 0);
8007
8008 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
8009 createOperands(N, Ops);
8010
8011 CSEMap.InsertNode(N, IP);
8012 } else {
8013 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
8014 createOperands(N, Ops);
8015 }
8016
8017 N->setFlags(Flags);
8018 InsertNode(N);
8019 SDValue V(N, 0);
8020 NewSDValueDbgMsg(V, "Creating new node: ", this);
8021 return V;
8022}
8023
8024SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
8025 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
8026 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
8027}
8028
8029SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8030 ArrayRef<SDValue> Ops) {
8031 SDNodeFlags Flags;
8032 if (Inserter)
8033 Flags = Inserter->getFlags();
8034 return getNode(Opcode, DL, VTList, Ops, Flags);
8035}
8036
8037SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8038 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
8039 if (VTList.NumVTs == 1)
8040 return getNode(Opcode, DL, VTList.VTs[0], Ops);
8041
8042#ifndef NDEBUG
8043 for (auto &Op : Ops)
8044 assert(Op.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (Op.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Op.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8045, __extension__ __PRETTY_FUNCTION__))
8045 "Operand is DELETED_NODE!")(static_cast <bool> (Op.getOpcode() != ISD::DELETED_NODE
&& "Operand is DELETED_NODE!") ? void (0) : __assert_fail
("Op.getOpcode() != ISD::DELETED_NODE && \"Operand is DELETED_NODE!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8045, __extension__ __PRETTY_FUNCTION__))
;
8046#endif
8047
8048 switch (Opcode) {
8049 case ISD::STRICT_FP_EXTEND:
8050 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&(static_cast <bool> (VTList.NumVTs == 2 && Ops.
size() == 2 && "Invalid STRICT_FP_EXTEND!") ? void (0
) : __assert_fail ("VTList.NumVTs == 2 && Ops.size() == 2 && \"Invalid STRICT_FP_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8051, __extension__ __PRETTY_FUNCTION__))
8051 "Invalid STRICT_FP_EXTEND!")(static_cast <bool> (VTList.NumVTs == 2 && Ops.
size() == 2 && "Invalid STRICT_FP_EXTEND!") ? void (0
) : __assert_fail ("VTList.NumVTs == 2 && Ops.size() == 2 && \"Invalid STRICT_FP_EXTEND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8051, __extension__ __PRETTY_FUNCTION__))
;
8052 assert(VTList.VTs[0].isFloatingPoint() &&(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"
) ? void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8053, __extension__ __PRETTY_FUNCTION__))
8053 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!")(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"
) ? void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && \"Invalid FP cast!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8053, __extension__ __PRETTY_FUNCTION__))
;
8054 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&(static_cast <bool> (VTList.VTs[0].isVector() == Ops[1]
.getValueType().isVector() && "STRICT_FP_EXTEND result type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && \"STRICT_FP_EXTEND result type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8056, __extension__ __PRETTY_FUNCTION__))
8055 "STRICT_FP_EXTEND result type should be vector iff the operand "(static_cast <bool> (VTList.VTs[0].isVector() == Ops[1]
.getValueType().isVector() && "STRICT_FP_EXTEND result type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && \"STRICT_FP_EXTEND result type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8056, __extension__ __PRETTY_FUNCTION__))
8056 "type is vector!")(static_cast <bool> (VTList.VTs[0].isVector() == Ops[1]
.getValueType().isVector() && "STRICT_FP_EXTEND result type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && \"STRICT_FP_EXTEND result type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8056, __extension__ __PRETTY_FUNCTION__))
;
8057 assert((!VTList.VTs[0].isVector() ||(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8060, __extension__ __PRETTY_FUNCTION__))
8058 VTList.VTs[0].getVectorNumElements() ==(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8060, __extension__ __PRETTY_FUNCTION__))
8059 Ops[1].getValueType().getVectorNumElements()) &&(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8060, __extension__ __PRETTY_FUNCTION__))
8060 "Vector element count mismatch!")(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8060, __extension__ __PRETTY_FUNCTION__))
;
8061 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&(static_cast <bool> (Ops[1].getValueType().bitsLT(VTList
.VTs[0]) && "Invalid fpext node, dst <= src!") ? void
(0) : __assert_fail ("Ops[1].getValueType().bitsLT(VTList.VTs[0]) && \"Invalid fpext node, dst <= src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8062, __extension__ __PRETTY_FUNCTION__))
8062 "Invalid fpext node, dst <= src!")(static_cast <bool> (Ops[1].getValueType().bitsLT(VTList
.VTs[0]) && "Invalid fpext node, dst <= src!") ? void
(0) : __assert_fail ("Ops[1].getValueType().bitsLT(VTList.VTs[0]) && \"Invalid fpext node, dst <= src!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8062, __extension__ __PRETTY_FUNCTION__))
;
8063 break;
8064 case ISD::STRICT_FP_ROUND:
8065 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!")(static_cast <bool> (VTList.NumVTs == 2 && Ops.
size() == 3 && "Invalid STRICT_FP_ROUND!") ? void (0)
: __assert_fail ("VTList.NumVTs == 2 && Ops.size() == 3 && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8065, __extension__ __PRETTY_FUNCTION__))
;
8066 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&(static_cast <bool> (VTList.VTs[0].isVector() == Ops[1]
.getValueType().isVector() && "STRICT_FP_ROUND result type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && \"STRICT_FP_ROUND result type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8068, __extension__ __PRETTY_FUNCTION__))
8067 "STRICT_FP_ROUND result type should be vector iff the operand "(static_cast <bool> (VTList.VTs[0].isVector() == Ops[1]
.getValueType().isVector() && "STRICT_FP_ROUND result type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && \"STRICT_FP_ROUND result type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8068, __extension__ __PRETTY_FUNCTION__))
8068 "type is vector!")(static_cast <bool> (VTList.VTs[0].isVector() == Ops[1]
.getValueType().isVector() && "STRICT_FP_ROUND result type should be vector iff the operand "
"type is vector!") ? void (0) : __assert_fail ("VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && \"STRICT_FP_ROUND result type should be vector iff the operand \" \"type is vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8068, __extension__ __PRETTY_FUNCTION__))
;
8069 assert((!VTList.VTs[0].isVector() ||(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8072, __extension__ __PRETTY_FUNCTION__))
8070 VTList.VTs[0].getVectorNumElements() ==(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8072, __extension__ __PRETTY_FUNCTION__))
8071 Ops[1].getValueType().getVectorNumElements()) &&(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8072, __extension__ __PRETTY_FUNCTION__))
8072 "Vector element count mismatch!")(static_cast <bool> ((!VTList.VTs[0].isVector() || VTList
.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements
()) && "Vector element count mismatch!") ? void (0) :
__assert_fail ("(!VTList.VTs[0].isVector() || VTList.VTs[0].getVectorNumElements() == Ops[1].getValueType().getVectorNumElements()) && \"Vector element count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8072, __extension__ __PRETTY_FUNCTION__))
;
8073 assert(VTList.VTs[0].isFloatingPoint() &&(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
8074 Ops[1].getValueType().isFloatingPoint() &&(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
8075 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
8076 isa<ConstantSDNode>(Ops[2]) &&(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
8077 (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 ||(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
8078 cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) &&(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
8079 "Invalid STRICT_FP_ROUND!")(static_cast <bool> (VTList.VTs[0].isFloatingPoint() &&
Ops[1].getValueType().isFloatingPoint() && VTList.VTs
[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode
>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->
getZExtValue() == 1) && "Invalid STRICT_FP_ROUND!") ?
void (0) : __assert_fail ("VTList.VTs[0].isFloatingPoint() && Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && \"Invalid STRICT_FP_ROUND!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8079, __extension__ __PRETTY_FUNCTION__))
;
8080 break;
8081#if 0
8082 // FIXME: figure out how to safely handle things like
8083 // int foo(int x) { return 1 << (x & 255); }
8084 // int bar() { return foo(256); }
8085 case ISD::SRA_PARTS:
8086 case ISD::SRL_PARTS:
8087 case ISD::SHL_PARTS:
8088 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
8089 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
8090 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
8091 else if (N3.getOpcode() == ISD::AND)
8092 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
8093 // If the and is only masking out bits that cannot effect the shift,
8094 // eliminate the and.
8095 unsigned NumBits = VT.getScalarSizeInBits()*2;
8096 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
8097 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
8098 }
8099 break;
8100#endif
8101 }
8102
8103 // Memoize the node unless it returns a flag.
8104 SDNode *N;
8105 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8106 FoldingSetNodeID ID;
8107 AddNodeIDNode(ID, Opcode, VTList, Ops);
8108 void *IP = nullptr;
8109 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
8110 return SDValue(E, 0);
8111
8112 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
8113 createOperands(N, Ops);
8114 CSEMap.InsertNode(N, IP);
8115 } else {
8116 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
8117 createOperands(N, Ops);
8118 }
8119
8120 N->setFlags(Flags);
8121 InsertNode(N);
8122 SDValue V(N, 0);
8123 NewSDValueDbgMsg(V, "Creating new node: ", this);
8124 return V;
8125}
8126
8127SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
8128 SDVTList VTList) {
8129 return getNode(Opcode, DL, VTList, None);
8130}
8131
8132SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8133 SDValue N1) {
8134 SDValue Ops[] = { N1 };
8135 return getNode(Opcode, DL, VTList, Ops);
8136}
8137
8138SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8139 SDValue N1, SDValue N2) {
8140 SDValue Ops[] = { N1, N2 };
8141 return getNode(Opcode, DL, VTList, Ops);
8142}
8143
8144SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8145 SDValue N1, SDValue N2, SDValue N3) {
8146 SDValue Ops[] = { N1, N2, N3 };
8147 return getNode(Opcode, DL, VTList, Ops);
8148}
8149
8150SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8151 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8152 SDValue Ops[] = { N1, N2, N3, N4 };
8153 return getNode(Opcode, DL, VTList, Ops);
8154}
8155
8156SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8157 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8158 SDValue N5) {
8159 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8160 return getNode(Opcode, DL, VTList, Ops);
8161}
8162
8163SDVTList SelectionDAG::getVTList(EVT VT) {
8164 return makeVTList(SDNode::getValueTypeList(VT), 1);
8165}
8166
8167SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
8168 FoldingSetNodeID ID;
8169 ID.AddInteger(2U);
8170 ID.AddInteger(VT1.getRawBits());
8171 ID.AddInteger(VT2.getRawBits());
8172
8173 void *IP = nullptr;
8174 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8175 if (!Result) {
8176 EVT *Array = Allocator.Allocate<EVT>(2);
8177 Array[0] = VT1;
8178 Array[1] = VT2;
8179 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
8180 VTListMap.InsertNode(Result, IP);
8181 }
8182 return Result->getSDVTList();
8183}
8184
8185SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
8186 FoldingSetNodeID ID;
8187 ID.AddInteger(3U);
8188 ID.AddInteger(VT1.getRawBits());
8189 ID.AddInteger(VT2.getRawBits());
8190 ID.AddInteger(VT3.getRawBits());
8191
8192 void *IP = nullptr;
8193 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8194 if (!Result) {
8195 EVT *Array = Allocator.Allocate<EVT>(3);
8196 Array[0] = VT1;
8197 Array[1] = VT2;
8198 Array[2] = VT3;
8199 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
8200 VTListMap.InsertNode(Result, IP);
8201 }
8202 return Result->getSDVTList();
8203}
8204
8205SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
8206 FoldingSetNodeID ID;
8207 ID.AddInteger(4U);
8208 ID.AddInteger(VT1.getRawBits());
8209 ID.AddInteger(VT2.getRawBits());
8210 ID.AddInteger(VT3.getRawBits());
8211 ID.AddInteger(VT4.getRawBits());
8212
8213 void *IP = nullptr;
8214 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8215 if (!Result) {
8216 EVT *Array = Allocator.Allocate<EVT>(4);
8217 Array[0] = VT1;
8218 Array[1] = VT2;
8219 Array[2] = VT3;
8220 Array[3] = VT4;
8221 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
8222 VTListMap.InsertNode(Result, IP);
8223 }
8224 return Result->getSDVTList();
8225}
8226
8227SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
8228 unsigned NumVTs = VTs.size();
8229 FoldingSetNodeID ID;
8230 ID.AddInteger(NumVTs);
8231 for (unsigned index = 0; index < NumVTs; index++) {
8232 ID.AddInteger(VTs[index].getRawBits());
8233 }
8234
8235 void *IP = nullptr;
8236 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8237 if (!Result) {
8238 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
8239 llvm::copy(VTs, Array);
8240 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
8241 VTListMap.InsertNode(Result, IP);
8242 }
8243 return Result->getSDVTList();
8244}
8245
8246
8247/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
8248/// specified operands. If the resultant node already exists in the DAG,
8249/// this does not modify the specified node, instead it returns the node that
8250/// already exists. If the resultant node does not exist in the DAG, the
8251/// input node is returned. As a degenerate case, if you specify the same
8252/// input operands as the node already has, the input node is returned.
8253SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
8254 assert(N->getNumOperands() == 1 && "Update with wrong number of operands")(static_cast <bool> (N->getNumOperands() == 1 &&
"Update with wrong number of operands") ? void (0) : __assert_fail
("N->getNumOperands() == 1 && \"Update with wrong number of operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8254, __extension__ __PRETTY_FUNCTION__))
;
8255
8256 // Check to see if there is no change.
8257 if (Op == N->getOperand(0)) return N;
8258
8259 // See if the modified node already exists.
8260 void *InsertPos = nullptr;
8261 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
8262 return Existing;
8263
8264 // Nope it doesn't. Remove the node from its current place in the maps.
8265 if (InsertPos)
8266 if (!RemoveNodeFromCSEMaps(N))
8267 InsertPos = nullptr;
8268
8269 // Now we update the operands.
8270 N->OperandList[0].set(Op);
8271
8272 updateDivergence(N);
8273 // If this gets put into a CSE map, add it.
8274 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
8275 return N;
8276}
8277
8278SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
8279 assert(N->getNumOperands() == 2 && "Update with wrong number of operands")(static_cast <bool> (N->getNumOperands() == 2 &&
"Update with wrong number of operands") ? void (0) : __assert_fail
("N->getNumOperands() == 2 && \"Update with wrong number of operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8279, __extension__ __PRETTY_FUNCTION__))
;
8280
8281 // Check to see if there is no change.
8282 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
8283 return N; // No operands changed, just return the input node.
8284
8285 // See if the modified node already exists.
8286 void *InsertPos = nullptr;
8287 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
8288 return Existing;
8289
8290 // Nope it doesn't. Remove the node from its current place in the maps.
8291 if (InsertPos)
8292 if (!RemoveNodeFromCSEMaps(N))
8293 InsertPos = nullptr;
8294
8295 // Now we update the operands.
8296 if (N->OperandList[0] != Op1)
8297 N->OperandList[0].set(Op1);
8298 if (N->OperandList[1] != Op2)
8299 N->OperandList[1].set(Op2);
8300
8301 updateDivergence(N);
8302 // If this gets put into a CSE map, add it.
8303 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
8304 return N;
8305}
8306
8307SDNode *SelectionDAG::
8308UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
8309 SDValue Ops[] = { Op1, Op2, Op3 };
8310 return UpdateNodeOperands(N, Ops);
8311}
8312
8313SDNode *SelectionDAG::
8314UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
8315 SDValue Op3, SDValue Op4) {
8316 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
8317 return UpdateNodeOperands(N, Ops);
8318}
8319
8320SDNode *SelectionDAG::
8321UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
8322 SDValue Op3, SDValue Op4, SDValue Op5) {
8323 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
8324 return UpdateNodeOperands(N, Ops);
8325}
8326
8327SDNode *SelectionDAG::
8328UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
8329 unsigned NumOps = Ops.size();
8330 assert(N->getNumOperands() == NumOps &&(static_cast <bool> (N->getNumOperands() == NumOps &&
"Update with wrong number of operands") ? void (0) : __assert_fail
("N->getNumOperands() == NumOps && \"Update with wrong number of operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8331, __extension__ __PRETTY_FUNCTION__))
8331 "Update with wrong number of operands")(static_cast <bool> (N->getNumOperands() == NumOps &&
"Update with wrong number of operands") ? void (0) : __assert_fail
("N->getNumOperands() == NumOps && \"Update with wrong number of operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8331, __extension__ __PRETTY_FUNCTION__))
;
8332
8333 // If no operands changed just return the input node.
8334 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
8335 return N;
8336
8337 // See if the modified node already exists.
8338 void *InsertPos = nullptr;
8339 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
8340 return Existing;
8341
8342 // Nope it doesn't. Remove the node from its current place in the maps.
8343 if (InsertPos)
8344 if (!RemoveNodeFromCSEMaps(N))
8345 InsertPos = nullptr;
8346
8347 // Now we update the operands.
8348 for (unsigned i = 0; i != NumOps; ++i)
8349 if (N->OperandList[i] != Ops[i])
8350 N->OperandList[i].set(Ops[i]);
8351
8352 updateDivergence(N);
8353 // If this gets put into a CSE map, add it.
8354 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
8355 return N;
8356}
8357
8358/// DropOperands - Release the operands and set this node to have
8359/// zero operands.
8360void SDNode::DropOperands() {
8361 // Unlike the code in MorphNodeTo that does this, we don't need to
8362 // watch for dead nodes here.
8363 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
8364 SDUse &Use = *I++;
8365 Use.set(SDValue());
8366 }
8367}
8368
8369void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
8370 ArrayRef<MachineMemOperand *> NewMemRefs) {
8371 if (NewMemRefs.empty()) {
8372 N->clearMemRefs();
8373 return;
8374 }
8375
8376 // Check if we can avoid allocating by storing a single reference directly.
8377 if (NewMemRefs.size() == 1) {
8378 N->MemRefs = NewMemRefs[0];
8379 N->NumMemRefs = 1;
8380 return;
8381 }
8382
8383 MachineMemOperand **MemRefsBuffer =
8384 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
8385 llvm::copy(NewMemRefs, MemRefsBuffer);
8386 N->MemRefs = MemRefsBuffer;
8387 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
8388}
8389
8390/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
8391/// machine opcode.
8392///
8393SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8394 EVT VT) {
8395 SDVTList VTs = getVTList(VT);
8396 return SelectNodeTo(N, MachineOpc, VTs, None);
8397}
8398
8399SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8400 EVT VT, SDValue Op1) {
8401 SDVTList VTs = getVTList(VT);
8402 SDValue Ops[] = { Op1 };
8403 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8404}
8405
8406SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8407 EVT VT, SDValue Op1,
8408 SDValue Op2) {
8409 SDVTList VTs = getVTList(VT);
8410 SDValue Ops[] = { Op1, Op2 };
8411 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8412}
8413
8414SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8415 EVT VT, SDValue Op1,
8416 SDValue Op2, SDValue Op3) {
8417 SDVTList VTs = getVTList(VT);
8418 SDValue Ops[] = { Op1, Op2, Op3 };
8419 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8420}
8421
8422SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8423 EVT VT, ArrayRef<SDValue> Ops) {
8424 SDVTList VTs = getVTList(VT);
8425 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8426}
8427
8428SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8429 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
8430 SDVTList VTs = getVTList(VT1, VT2);
8431 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8432}
8433
8434SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8435 EVT VT1, EVT VT2) {
8436 SDVTList VTs = getVTList(VT1, VT2);
8437 return SelectNodeTo(N, MachineOpc, VTs, None);
8438}
8439
8440SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8441 EVT VT1, EVT VT2, EVT VT3,
8442 ArrayRef<SDValue> Ops) {
8443 SDVTList VTs = getVTList(VT1, VT2, VT3);
8444 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8445}
8446
8447SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8448 EVT VT1, EVT VT2,
8449 SDValue Op1, SDValue Op2) {
8450 SDVTList VTs = getVTList(VT1, VT2);
8451 SDValue Ops[] = { Op1, Op2 };
8452 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8453}
8454
8455SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8456 SDVTList VTs,ArrayRef<SDValue> Ops) {
8457 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
8458 // Reset the NodeID to -1.
8459 New->setNodeId(-1);
8460 if (New != N) {
8461 ReplaceAllUsesWith(N, New);
8462 RemoveDeadNode(N);
8463 }
8464 return New;
8465}
8466
8467/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
8468/// the line number information on the merged node since it is not possible to
8469/// preserve the information that operation is associated with multiple lines.
8470/// This will make the debugger working better at -O0, were there is a higher
8471/// probability having other instructions associated with that line.
8472///
8473/// For IROrder, we keep the smaller of the two
8474SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
8475 DebugLoc NLoc = N->getDebugLoc();
8476 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
8477 N->setDebugLoc(DebugLoc());
8478 }
8479 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
8480 N->setIROrder(Order);
8481 return N;
8482}
8483
8484/// MorphNodeTo - This *mutates* the specified node to have the specified
8485/// return type, opcode, and operands.
8486///
8487/// Note that MorphNodeTo returns the resultant node. If there is already a
8488/// node of the specified opcode and operands, it returns that node instead of
8489/// the current one. Note that the SDLoc need not be the same.
8490///
8491/// Using MorphNodeTo is faster than creating a new node and swapping it in
8492/// with ReplaceAllUsesWith both because it often avoids allocating a new
8493/// node, and because it doesn't require CSE recalculation for any of
8494/// the node's users.
8495///
8496/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
8497/// As a consequence it isn't appropriate to use from within the DAG combiner or
8498/// the legalizer which maintain worklists that would need to be updated when
8499/// deleting things.
8500SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
8501 SDVTList VTs, ArrayRef<SDValue> Ops) {
8502 // If an identical node already exists, use it.
8503 void *IP = nullptr;
8504 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
8505 FoldingSetNodeID ID;
8506 AddNodeIDNode(ID, Opc, VTs, Ops);
8507 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
8508 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
8509 }
8510
8511 if (!RemoveNodeFromCSEMaps(N))
8512 IP = nullptr;
8513
8514 // Start the morphing.
8515 N->NodeType = Opc;
8516 N->ValueList = VTs.VTs;
8517 N->NumValues = VTs.NumVTs;
8518
8519 // Clear the operands list, updating used nodes to remove this from their
8520 // use list. Keep track of any operands that become dead as a result.
8521 SmallPtrSet<SDNode*, 16> DeadNodeSet;
8522 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
8523 SDUse &Use = *I++;
8524 SDNode *Used = Use.getNode();
8525 Use.set(SDValue());
8526 if (Used->use_empty())
8527 DeadNodeSet.insert(Used);
8528 }
8529
8530 // For MachineNode, initialize the memory references information.
8531 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
8532 MN->clearMemRefs();
8533
8534 // Swap for an appropriately sized array from the recycler.
8535 removeOperands(N);
8536 createOperands(N, Ops);
8537
8538 // Delete any nodes that are still dead after adding the uses for the
8539 // new operands.
8540 if (!DeadNodeSet.empty()) {
8541 SmallVector<SDNode *, 16> DeadNodes;
8542 for (SDNode *N : DeadNodeSet)
8543 if (N->use_empty())
8544 DeadNodes.push_back(N);
8545 RemoveDeadNodes(DeadNodes);
8546 }
8547
8548 if (IP)
8549 CSEMap.InsertNode(N, IP); // Memoize the new node.
8550 return N;
8551}
8552
8553SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
8554 unsigned OrigOpc = Node->getOpcode();
8555 unsigned NewOpc;
8556 switch (OrigOpc) {
8557 default:
8558 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!")::llvm::llvm_unreachable_internal("mutateStrictFPToFP called with unexpected opcode!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8558)
;
8559#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8560 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
8561#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8562 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
8563#include "llvm/IR/ConstrainedOps.def"
8564 }
8565
8566 assert(Node->getNumValues() == 2 && "Unexpected number of results!")(static_cast <bool> (Node->getNumValues() == 2 &&
"Unexpected number of results!") ? void (0) : __assert_fail (
"Node->getNumValues() == 2 && \"Unexpected number of results!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8566, __extension__ __PRETTY_FUNCTION__))
;
8567
8568 // We're taking this node out of the chain, so we need to re-link things.
8569 SDValue InputChain = Node->getOperand(0);
8570 SDValue OutputChain = SDValue(Node, 1);
8571 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
8572
8573 SmallVector<SDValue, 3> Ops;
8574 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
8575 Ops.push_back(Node->getOperand(i));
8576
8577 SDVTList VTs = getVTList(Node->getValueType(0));
8578 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
8579
8580 // MorphNodeTo can operate in two ways: if an existing node with the
8581 // specified operands exists, it can just return it. Otherwise, it
8582 // updates the node in place to have the requested operands.
8583 if (Res == Node) {
8584 // If we updated the node in place, reset the node ID. To the isel,
8585 // this should be just like a newly allocated machine node.
8586 Res->setNodeId(-1);
8587 } else {
8588 ReplaceAllUsesWith(Node, Res);
8589 RemoveDeadNode(Node);
8590 }
8591
8592 return Res;
8593}
8594
8595/// getMachineNode - These are used for target selectors to create a new node
8596/// with specified return type(s), MachineInstr opcode, and operands.
8597///
8598/// Note that getMachineNode returns the resultant node. If there is already a
8599/// node of the specified opcode and operands, it returns that node instead of
8600/// the current one.
8601MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8602 EVT VT) {
8603 SDVTList VTs = getVTList(VT);
8604 return getMachineNode(Opcode, dl, VTs, None);
8605}
8606
8607MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8608 EVT VT, SDValue Op1) {
8609 SDVTList VTs = getVTList(VT);
8610 SDValue Ops[] = { Op1 };
8611 return getMachineNode(Opcode, dl, VTs, Ops);
8612}
8613
8614MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8615 EVT VT, SDValue Op1, SDValue Op2) {
8616 SDVTList VTs = getVTList(VT);
8617 SDValue Ops[] = { Op1, Op2 };
8618 return getMachineNode(Opcode, dl, VTs, Ops);
8619}
8620
8621MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8622 EVT VT, SDValue Op1, SDValue Op2,
8623 SDValue Op3) {
8624 SDVTList VTs = getVTList(VT);
8625 SDValue Ops[] = { Op1, Op2, Op3 };
8626 return getMachineNode(Opcode, dl, VTs, Ops);
8627}
8628
8629MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8630 EVT VT, ArrayRef<SDValue> Ops) {
8631 SDVTList VTs = getVTList(VT);
8632 return getMachineNode(Opcode, dl, VTs, Ops);
8633}
8634
8635MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8636 EVT VT1, EVT VT2, SDValue Op1,
8637 SDValue Op2) {
8638 SDVTList VTs = getVTList(VT1, VT2);
8639 SDValue Ops[] = { Op1, Op2 };
8640 return getMachineNode(Opcode, dl, VTs, Ops);
8641}
8642
8643MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8644 EVT VT1, EVT VT2, SDValue Op1,
8645 SDValue Op2, SDValue Op3) {
8646 SDVTList VTs = getVTList(VT1, VT2);
8647 SDValue Ops[] = { Op1, Op2, Op3 };
8648 return getMachineNode(Opcode, dl, VTs, Ops);
8649}
8650
8651MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8652 EVT VT1, EVT VT2,
8653 ArrayRef<SDValue> Ops) {
8654 SDVTList VTs = getVTList(VT1, VT2);
8655 return getMachineNode(Opcode, dl, VTs, Ops);
8656}
8657
8658MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8659 EVT VT1, EVT VT2, EVT VT3,
8660 SDValue Op1, SDValue Op2) {
8661 SDVTList VTs = getVTList(VT1, VT2, VT3);
8662 SDValue Ops[] = { Op1, Op2 };
8663 return getMachineNode(Opcode, dl, VTs, Ops);
8664}
8665
8666MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8667 EVT VT1, EVT VT2, EVT VT3,
8668 SDValue Op1, SDValue Op2,
8669 SDValue Op3) {
8670 SDVTList VTs = getVTList(VT1, VT2, VT3);
8671 SDValue Ops[] = { Op1, Op2, Op3 };
8672 return getMachineNode(Opcode, dl, VTs, Ops);
8673}
8674
8675MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8676 EVT VT1, EVT VT2, EVT VT3,
8677 ArrayRef<SDValue> Ops) {
8678 SDVTList VTs = getVTList(VT1, VT2, VT3);
8679 return getMachineNode(Opcode, dl, VTs, Ops);
8680}
8681
8682MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8683 ArrayRef<EVT> ResultTys,
8684 ArrayRef<SDValue> Ops) {
8685 SDVTList VTs = getVTList(ResultTys);
8686 return getMachineNode(Opcode, dl, VTs, Ops);
8687}
8688
8689MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
8690 SDVTList VTs,
8691 ArrayRef<SDValue> Ops) {
8692 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
8693 MachineSDNode *N;
8694 void *IP = nullptr;
8695
8696 if (DoCSE) {
8697 FoldingSetNodeID ID;
8698 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
8699 IP = nullptr;
8700 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
8701 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
8702 }
8703 }
8704
8705 // Allocate a new MachineSDNode.
8706 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
8707 createOperands(N, Ops);
8708
8709 if (DoCSE)
8710 CSEMap.InsertNode(N, IP);
8711
8712 InsertNode(N);
8713 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
8714 return N;
8715}
8716
8717/// getTargetExtractSubreg - A convenience function for creating
8718/// TargetOpcode::EXTRACT_SUBREG nodes.
8719SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8720 SDValue Operand) {
8721 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8722 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
8723 VT, Operand, SRIdxVal);
8724 return SDValue(Subreg, 0);
8725}
8726
8727/// getTargetInsertSubreg - A convenience function for creating
8728/// TargetOpcode::INSERT_SUBREG nodes.
8729SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8730 SDValue Operand, SDValue Subreg) {
8731 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8732 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
8733 VT, Operand, Subreg, SRIdxVal);
8734 return SDValue(Result, 0);
8735}
8736
8737/// getNodeIfExists - Get the specified node if it's already available, or
8738/// else return NULL.
8739SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8740 ArrayRef<SDValue> Ops) {
8741 SDNodeFlags Flags;
8742 if (Inserter)
8743 Flags = Inserter->getFlags();
8744 return getNodeIfExists(Opcode, VTList, Ops, Flags);
8745}
8746
8747SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8748 ArrayRef<SDValue> Ops,
8749 const SDNodeFlags Flags) {
8750 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
8751 FoldingSetNodeID ID;
8752 AddNodeIDNode(ID, Opcode, VTList, Ops);
8753 void *IP = nullptr;
8754 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
8755 E->intersectFlagsWith(Flags);
8756 return E;
8757 }
8758 }
8759 return nullptr;
8760}
8761
8762/// doesNodeExist - Check if a node exists without modifying its flags.
8763bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
8764 ArrayRef<SDValue> Ops) {
8765 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
8766 FoldingSetNodeID ID;
8767 AddNodeIDNode(ID, Opcode, VTList, Ops);
8768 void *IP = nullptr;
8769 if (FindNodeOrInsertPos(ID, SDLoc(), IP))
8770 return true;
8771 }
8772 return false;
8773}
8774
8775/// getDbgValue - Creates a SDDbgValue node.
8776///
8777/// SDNode
8778SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
8779 SDNode *N, unsigned R, bool IsIndirect,
8780 const DebugLoc &DL, unsigned O) {
8781 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8782, __extension__ __PRETTY_FUNCTION__))
8782 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8782, __extension__ __PRETTY_FUNCTION__))
;
8783 return new (DbgInfo->getAlloc())
8784 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
8785 {}, IsIndirect, DL, O,
8786 /*IsVariadic=*/false);
8787}
8788
8789/// Constant
8790SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
8791 DIExpression *Expr,
8792 const Value *C,
8793 const DebugLoc &DL, unsigned O) {
8794 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8795, __extension__ __PRETTY_FUNCTION__))
8795 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8795, __extension__ __PRETTY_FUNCTION__))
;
8796 return new (DbgInfo->getAlloc())
8797 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
8798 /*IsIndirect=*/false, DL, O,
8799 /*IsVariadic=*/false);
8800}
8801
8802/// FrameIndex
8803SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
8804 DIExpression *Expr, unsigned FI,
8805 bool IsIndirect,
8806 const DebugLoc &DL,
8807 unsigned O) {
8808 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8809, __extension__ __PRETTY_FUNCTION__))
8809 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8809, __extension__ __PRETTY_FUNCTION__))
;
8810 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
8811}
8812
8813/// FrameIndex with dependencies
8814SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
8815 DIExpression *Expr, unsigned FI,
8816 ArrayRef<SDNode *> Dependencies,
8817 bool IsIndirect,
8818 const DebugLoc &DL,
8819 unsigned O) {
8820 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8821, __extension__ __PRETTY_FUNCTION__))
8821 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8821, __extension__ __PRETTY_FUNCTION__))
;
8822 return new (DbgInfo->getAlloc())
8823 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
8824 Dependencies, IsIndirect, DL, O,
8825 /*IsVariadic=*/false);
8826}
8827
8828/// VReg
8829SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
8830 unsigned VReg, bool IsIndirect,
8831 const DebugLoc &DL, unsigned O) {
8832 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8833, __extension__ __PRETTY_FUNCTION__))
8833 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8833, __extension__ __PRETTY_FUNCTION__))
;
8834 return new (DbgInfo->getAlloc())
8835 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
8836 {}, IsIndirect, DL, O,
8837 /*IsVariadic=*/false);
8838}
8839
8840SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
8841 ArrayRef<SDDbgOperand> Locs,
8842 ArrayRef<SDNode *> Dependencies,
8843 bool IsIndirect, const DebugLoc &DL,
8844 unsigned O, bool IsVariadic) {
8845 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8846, __extension__ __PRETTY_FUNCTION__))
8846 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILocalVariable>(Var)->
isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? void (0) : __assert_fail ("cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8846, __extension__ __PRETTY_FUNCTION__))
;
8847 return new (DbgInfo->getAlloc())
8848 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
8849 DL, O, IsVariadic);
8850}
8851
8852void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
8853 unsigned OffsetInBits, unsigned SizeInBits,
8854 bool InvalidateDbg) {
8855 SDNode *FromNode = From.getNode();
8856 SDNode *ToNode = To.getNode();
8857 assert(FromNode && ToNode && "Can't modify dbg values")(static_cast <bool> (FromNode && ToNode &&
"Can't modify dbg values") ? void (0) : __assert_fail ("FromNode && ToNode && \"Can't modify dbg values\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8857, __extension__ __PRETTY_FUNCTION__))
;
8858
8859 // PR35338
8860 // TODO: assert(From != To && "Redundant dbg value transfer");
8861 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
8862 if (From == To || FromNode == ToNode)
8863 return;
8864
8865 if (!FromNode->getHasDebugValue())
8866 return;
8867
8868 SDDbgOperand FromLocOp =
8869 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
8870 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo());
8871
8872 SmallVector<SDDbgValue *, 2> ClonedDVs;
8873 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
8874 if (Dbg->isInvalidated())
8875 continue;
8876
8877 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
8878
8879 // Create a new location ops vector that is equal to the old vector, but
8880 // with each instance of FromLocOp replaced with ToLocOp.
8881 bool Changed = false;
8882 auto NewLocOps = Dbg->copyLocationOps();
8883 std::replace_if(
8884 NewLocOps.begin(), NewLocOps.end(),
8885 [&Changed, FromLocOp](const SDDbgOperand &Op) {
8886 bool Match = Op == FromLocOp;
8887 Changed |= Match;
8888 return Match;
8889 },
8890 ToLocOp);
8891 // Ignore this SDDbgValue if we didn't find a matching location.
8892 if (!Changed)
8893 continue;
8894
8895 DIVariable *Var = Dbg->getVariable();
8896 auto *Expr = Dbg->getExpression();
8897 // If a fragment is requested, update the expression.
8898 if (SizeInBits) {
8899 // When splitting a larger (e.g., sign-extended) value whose
8900 // lower bits are described with an SDDbgValue, do not attempt
8901 // to transfer the SDDbgValue to the upper bits.
8902 if (auto FI = Expr->getFragmentInfo())
8903 if (OffsetInBits + SizeInBits > FI->SizeInBits)
8904 continue;
8905 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
8906 SizeInBits);
8907 if (!Fragment)
8908 continue;
8909 Expr = *Fragment;
8910 }
8911
8912 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
8913 // Clone the SDDbgValue and move it to To.
8914 SDDbgValue *Clone = getDbgValueList(
8915 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
8916 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
8917 Dbg->isVariadic());
8918 ClonedDVs.push_back(Clone);
8919
8920 if (InvalidateDbg) {
8921 // Invalidate value and indicate the SDDbgValue should not be emitted.
8922 Dbg->setIsInvalidated();
8923 Dbg->setIsEmitted();
8924 }
8925 }
8926
8927 for (SDDbgValue *Dbg : ClonedDVs) {
8928 assert(is_contained(Dbg->getSDNodes(), ToNode) &&(static_cast <bool> (is_contained(Dbg->getSDNodes(),
ToNode) && "Transferred DbgValues should depend on the new SDNode"
) ? void (0) : __assert_fail ("is_contained(Dbg->getSDNodes(), ToNode) && \"Transferred DbgValues should depend on the new SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8929, __extension__ __PRETTY_FUNCTION__))
8929 "Transferred DbgValues should depend on the new SDNode")(static_cast <bool> (is_contained(Dbg->getSDNodes(),
ToNode) && "Transferred DbgValues should depend on the new SDNode"
) ? void (0) : __assert_fail ("is_contained(Dbg->getSDNodes(), ToNode) && \"Transferred DbgValues should depend on the new SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8929, __extension__ __PRETTY_FUNCTION__))
;
8930 AddDbgValue(Dbg, false);
8931 }
8932}
8933
8934void SelectionDAG::salvageDebugInfo(SDNode &N) {
8935 if (!N.getHasDebugValue())
8936 return;
8937
8938 SmallVector<SDDbgValue *, 2> ClonedDVs;
8939 for (auto DV : GetDbgValues(&N)) {
8940 if (DV->isInvalidated())
8941 continue;
8942 switch (N.getOpcode()) {
8943 default:
8944 break;
8945 case ISD::ADD:
8946 SDValue N0 = N.getOperand(0);
8947 SDValue N1 = N.getOperand(1);
8948 if (!isConstantIntBuildVectorOrConstantInt(N0) &&
8949 isConstantIntBuildVectorOrConstantInt(N1)) {
8950 uint64_t Offset = N.getConstantOperandVal(1);
8951
8952 // Rewrite an ADD constant node into a DIExpression. Since we are
8953 // performing arithmetic to compute the variable's *value* in the
8954 // DIExpression, we need to mark the expression with a
8955 // DW_OP_stack_value.
8956 auto *DIExpr = DV->getExpression();
8957 auto NewLocOps = DV->copyLocationOps();
8958 bool Changed = false;
8959 for (size_t i = 0; i < NewLocOps.size(); ++i) {
8960 // We're not given a ResNo to compare against because the whole
8961 // node is going away. We know that any ISD::ADD only has one
8962 // result, so we can assume any node match is using the result.
8963 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
8964 NewLocOps[i].getSDNode() != &N)
8965 continue;
8966 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
8967 SmallVector<uint64_t, 3> ExprOps;
8968 DIExpression::appendOffset(ExprOps, Offset);
8969 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
8970 Changed = true;
8971 }
8972 (void)Changed;
8973 assert(Changed && "Salvage target doesn't use N")(static_cast <bool> (Changed && "Salvage target doesn't use N"
) ? void (0) : __assert_fail ("Changed && \"Salvage target doesn't use N\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8973, __extension__ __PRETTY_FUNCTION__))
;
8974
8975 auto AdditionalDependencies = DV->getAdditionalDependencies();
8976 SDDbgValue *Clone = getDbgValueList(DV->getVariable(), DIExpr,
8977 NewLocOps, AdditionalDependencies,
8978 DV->isIndirect(), DV->getDebugLoc(),
8979 DV->getOrder(), DV->isVariadic());
8980 ClonedDVs.push_back(Clone);
8981 DV->setIsInvalidated();
8982 DV->setIsEmitted();
8983 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("selectiondag")) { dbgs() << "SALVAGE: Rewriting"; N0.
getNode()->dumprFull(this); dbgs() << " into " <<
*DIExpr << '\n'; } } while (false)
8984 N0.getNode()->dumprFull(this);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("selectiondag")) { dbgs() << "SALVAGE: Rewriting"; N0.
getNode()->dumprFull(this); dbgs() << " into " <<
*DIExpr << '\n'; } } while (false)
8985 dbgs() << " into " << *DIExpr << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("selectiondag")) { dbgs() << "SALVAGE: Rewriting"; N0.
getNode()->dumprFull(this); dbgs() << " into " <<
*DIExpr << '\n'; } } while (false)
;
8986 }
8987 }
8988 }
8989
8990 for (SDDbgValue *Dbg : ClonedDVs) {
8991 assert(!Dbg->getSDNodes().empty() &&(static_cast <bool> (!Dbg->getSDNodes().empty() &&
"Salvaged DbgValue should depend on a new SDNode") ? void (0
) : __assert_fail ("!Dbg->getSDNodes().empty() && \"Salvaged DbgValue should depend on a new SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8992, __extension__ __PRETTY_FUNCTION__))
8992 "Salvaged DbgValue should depend on a new SDNode")(static_cast <bool> (!Dbg->getSDNodes().empty() &&
"Salvaged DbgValue should depend on a new SDNode") ? void (0
) : __assert_fail ("!Dbg->getSDNodes().empty() && \"Salvaged DbgValue should depend on a new SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 8992, __extension__ __PRETTY_FUNCTION__))
;
8993 AddDbgValue(Dbg, false);
8994 }
8995}
8996
8997/// Creates a SDDbgLabel node.
8998SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
8999 const DebugLoc &DL, unsigned O) {
9000 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (cast<DILabel>(Label)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9001, __extension__ __PRETTY_FUNCTION__))
9001 "Expected inlined-at fields to agree")(static_cast <bool> (cast<DILabel>(Label)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9001, __extension__ __PRETTY_FUNCTION__))
;
9002 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
9003}
9004
9005namespace {
9006
9007/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
9008/// pointed to by a use iterator is deleted, increment the use iterator
9009/// so that it doesn't dangle.
9010///
9011class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
9012 SDNode::use_iterator &UI;
9013 SDNode::use_iterator &UE;
9014
9015 void NodeDeleted(SDNode *N, SDNode *E) override {
9016 // Increment the iterator as needed.
9017 while (UI != UE && N == *UI)
9018 ++UI;
9019 }
9020
9021public:
9022 RAUWUpdateListener(SelectionDAG &d,
9023 SDNode::use_iterator &ui,
9024 SDNode::use_iterator &ue)
9025 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
9026};
9027
9028} // end anonymous namespace
9029
9030/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
9031/// This can cause recursive merging of nodes in the DAG.
9032///
9033/// This version assumes From has a single result value.
9034///
9035void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
9036 SDNode *From = FromN.getNode();
9037 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&(static_cast <bool> (From->getNumValues() == 1 &&
FromN.getResNo() == 0 && "Cannot replace with this method!"
) ? void (0) : __assert_fail ("From->getNumValues() == 1 && FromN.getResNo() == 0 && \"Cannot replace with this method!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9038, __extension__ __PRETTY_FUNCTION__))
9038 "Cannot replace with this method!")(static_cast <bool> (From->getNumValues() == 1 &&
FromN.getResNo() == 0 && "Cannot replace with this method!"
) ? void (0) : __assert_fail ("From->getNumValues() == 1 && FromN.getResNo() == 0 && \"Cannot replace with this method!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9038, __extension__ __PRETTY_FUNCTION__))
;
9039 assert(From != To.getNode() && "Cannot replace uses of with self")(static_cast <bool> (From != To.getNode() && "Cannot replace uses of with self"
) ? void (0) : __assert_fail ("From != To.getNode() && \"Cannot replace uses of with self\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9039, __extension__ __PRETTY_FUNCTION__))
;
9040
9041 // Preserve Debug Values
9042 transferDbgValues(FromN, To);
9043
9044 // Iterate over all the existing uses of From. New uses will be added
9045 // to the beginning of the use list, which we avoid visiting.
9046 // This specifically avoids visiting uses of From that arise while the
9047 // replacement is happening, because any such uses would be the result
9048 // of CSE: If an existing node looks like From after one of its operands
9049 // is replaced by To, we don't want to replace of all its users with To
9050 // too. See PR3018 for more info.
9051 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
9052 RAUWUpdateListener Listener(*this, UI, UE);
9053 while (UI != UE) {
9054 SDNode *User = *UI;
9055
9056 // This node is about to morph, remove its old self from the CSE maps.
9057 RemoveNodeFromCSEMaps(User);
9058
9059 // A user can appear in a use list multiple times, and when this
9060 // happens the uses are usually next to each other in the list.
9061 // To help reduce the number of CSE recomputations, process all
9062 // the uses of this user that we can find this way.
9063 do {
9064 SDUse &Use = UI.getUse();
9065 ++UI;
9066 Use.set(To);
9067 if (To->isDivergent() != From->isDivergent())
9068 updateDivergence(User);
9069 } while (UI != UE && *UI == User);
9070 // Now that we have modified User, add it back to the CSE maps. If it
9071 // already exists there, recursively merge the results together.
9072 AddModifiedNodeToCSEMaps(User);
9073 }
9074
9075 // If we just RAUW'd the root, take note.
9076 if (FromN == getRoot())
9077 setRoot(To);
9078}
9079
9080/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
9081/// This can cause recursive merging of nodes in the DAG.
9082///
9083/// This version assumes that for each value of From, there is a
9084/// corresponding value in To in the same position with the same type.
9085///
9086void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
9087#ifndef NDEBUG
9088 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
9089 assert((!From->hasAnyUseOfValue(i) ||(static_cast <bool> ((!From->hasAnyUseOfValue(i) || From
->getValueType(i) == To->getValueType(i)) && "Cannot use this version of ReplaceAllUsesWith!"
) ? void (0) : __assert_fail ("(!From->hasAnyUseOfValue(i) || From->getValueType(i) == To->getValueType(i)) && \"Cannot use this version of ReplaceAllUsesWith!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9091, __extension__ __PRETTY_FUNCTION__))
9090 From->getValueType(i) == To->getValueType(i)) &&(static_cast <bool> ((!From->hasAnyUseOfValue(i) || From
->getValueType(i) == To->getValueType(i)) && "Cannot use this version of ReplaceAllUsesWith!"
) ? void (0) : __assert_fail ("(!From->hasAnyUseOfValue(i) || From->getValueType(i) == To->getValueType(i)) && \"Cannot use this version of ReplaceAllUsesWith!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9091, __extension__ __PRETTY_FUNCTION__))
9091 "Cannot use this version of ReplaceAllUsesWith!")(static_cast <bool> ((!From->hasAnyUseOfValue(i) || From
->getValueType(i) == To->getValueType(i)) && "Cannot use this version of ReplaceAllUsesWith!"
) ? void (0) : __assert_fail ("(!From->hasAnyUseOfValue(i) || From->getValueType(i) == To->getValueType(i)) && \"Cannot use this version of ReplaceAllUsesWith!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9091, __extension__ __PRETTY_FUNCTION__))
;
9092#endif
9093
9094 // Handle the trivial case.
9095 if (From == To)
9096 return;
9097
9098 // Preserve Debug Info. Only do this if there's a use.
9099 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
9100 if (From->hasAnyUseOfValue(i)) {
9101 assert((i < To->getNumValues()) && "Invalid To location")(static_cast <bool> ((i < To->getNumValues()) &&
"Invalid To location") ? void (0) : __assert_fail ("(i < To->getNumValues()) && \"Invalid To location\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9101, __extension__ __PRETTY_FUNCTION__))
;
9102 transferDbgValues(SDValue(From, i), SDValue(To, i));
9103 }
9104
9105 // Iterate over just the existing users of From. See the comments in
9106 // the ReplaceAllUsesWith above.
9107 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
9108 RAUWUpdateListener Listener(*this, UI, UE);
9109 while (UI != UE) {
9110 SDNode *User = *UI;
9111
9112 // This node is about to morph, remove its old self from the CSE maps.
9113 RemoveNodeFromCSEMaps(User);
9114
9115 // A user can appear in a use list multiple times, and when this
9116 // happens the uses are usually next to each other in the list.
9117 // To help reduce the number of CSE recomputations, process all
9118 // the uses of this user that we can find this way.
9119 do {
9120 SDUse &Use = UI.getUse();
9121 ++UI;
9122 Use.setNode(To);
9123 if (To->isDivergent() != From->isDivergent())
9124 updateDivergence(User);
9125 } while (UI != UE && *UI == User);
9126
9127 // Now that we have modified User, add it back to the CSE maps. If it
9128 // already exists there, recursively merge the results together.
9129 AddModifiedNodeToCSEMaps(User);
9130 }
9131
9132 // If we just RAUW'd the root, take note.
9133 if (From == getRoot().getNode())
9134 setRoot(SDValue(To, getRoot().getResNo()));
9135}
9136
9137/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
9138/// This can cause recursive merging of nodes in the DAG.
9139///
9140/// This version can replace From with any result values. To must match the
9141/// number and types of values returned by From.
9142void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
9143 if (From->getNumValues() == 1) // Handle the simple case efficiently.
9144 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
9145
9146 // Preserve Debug Info.
9147 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
9148 transferDbgValues(SDValue(From, i), To[i]);
9149
9150 // Iterate over just the existing users of From. See the comments in
9151 // the ReplaceAllUsesWith above.
9152 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
9153 RAUWUpdateListener Listener(*this, UI, UE);
9154 while (UI != UE) {
9155 SDNode *User = *UI;
9156
9157 // This node is about to morph, remove its old self from the CSE maps.
9158 RemoveNodeFromCSEMaps(User);
9159
9160 // A user can appear in a use list multiple times, and when this happens the
9161 // uses are usually next to each other in the list. To help reduce the
9162 // number of CSE and divergence recomputations, process all the uses of this
9163 // user that we can find this way.
9164 bool To_IsDivergent = false;
9165 do {
9166 SDUse &Use = UI.getUse();
9167 const SDValue &ToOp = To[Use.getResNo()];
9168 ++UI;
9169 Use.set(ToOp);
9170 To_IsDivergent |= ToOp->isDivergent();
9171 } while (UI != UE && *UI == User);
9172
9173 if (To_IsDivergent != From->isDivergent())
9174 updateDivergence(User);
9175
9176 // Now that we have modified User, add it back to the CSE maps. If it
9177 // already exists there, recursively merge the results together.
9178 AddModifiedNodeToCSEMaps(User);
9179 }
9180
9181 // If we just RAUW'd the root, take note.
9182 if (From == getRoot().getNode())
9183 setRoot(SDValue(To[getRoot().getResNo()]));
9184}
9185
9186/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
9187/// uses of other values produced by From.getNode() alone. The Deleted
9188/// vector is handled the same way as for ReplaceAllUsesWith.
9189void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
9190 // Handle the really simple, really trivial case efficiently.
9191 if (From == To) return;
9192
9193 // Handle the simple, trivial, case efficiently.
9194 if (From.getNode()->getNumValues() == 1) {
9195 ReplaceAllUsesWith(From, To);
9196 return;
9197 }
9198
9199 // Preserve Debug Info.
9200 transferDbgValues(From, To);
9201
9202 // Iterate over just the existing users of From. See the comments in
9203 // the ReplaceAllUsesWith above.
9204 SDNode::use_iterator UI = From.getNode()->use_begin(),
9205 UE = From.getNode()->use_end();
9206 RAUWUpdateListener Listener(*this, UI, UE);
9207 while (UI != UE) {
9208 SDNode *User = *UI;
9209 bool UserRemovedFromCSEMaps = false;
9210
9211 // A user can appear in a use list multiple times, and when this
9212 // happens the uses are usually next to each other in the list.
9213 // To help reduce the number of CSE recomputations, process all
9214 // the uses of this user that we can find this way.
9215 do {
9216 SDUse &Use = UI.getUse();
9217
9218 // Skip uses of different values from the same node.
9219 if (Use.getResNo() != From.getResNo()) {
9220 ++UI;
9221 continue;
9222 }
9223
9224 // If this node hasn't been modified yet, it's still in the CSE maps,
9225 // so remove its old self from the CSE maps.
9226 if (!UserRemovedFromCSEMaps) {
9227 RemoveNodeFromCSEMaps(User);
9228 UserRemovedFromCSEMaps = true;
9229 }
9230
9231 ++UI;
9232 Use.set(To);
9233 if (To->isDivergent() != From->isDivergent())
9234 updateDivergence(User);
9235 } while (UI != UE && *UI == User);
9236 // We are iterating over all uses of the From node, so if a use
9237 // doesn't use the specific value, no changes are made.
9238 if (!UserRemovedFromCSEMaps)
9239 continue;
9240
9241 // Now that we have modified User, add it back to the CSE maps. If it
9242 // already exists there, recursively merge the results together.
9243 AddModifiedNodeToCSEMaps(User);
9244 }
9245
9246 // If we just RAUW'd the root, take note.
9247 if (From == getRoot())
9248 setRoot(To);
9249}
9250
9251namespace {
9252
9253 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
9254 /// to record information about a use.
9255 struct UseMemo {
9256 SDNode *User;
9257 unsigned Index;
9258 SDUse *Use;
9259 };
9260
9261 /// operator< - Sort Memos by User.
9262 bool operator<(const UseMemo &L, const UseMemo &R) {
9263 return (intptr_t)L.User < (intptr_t)R.User;
9264 }
9265
9266} // end anonymous namespace
9267
9268bool SelectionDAG::calculateDivergence(SDNode *N) {
9269 if (TLI->isSDNodeAlwaysUniform(N)) {
9270 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) &&(static_cast <bool> (!TLI->isSDNodeSourceOfDivergence
(N, FLI, DA) && "Conflicting divergence information!"
) ? void (0) : __assert_fail ("!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && \"Conflicting divergence information!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9271, __extension__ __PRETTY_FUNCTION__))
9271 "Conflicting divergence information!")(static_cast <bool> (!TLI->isSDNodeSourceOfDivergence
(N, FLI, DA) && "Conflicting divergence information!"
) ? void (0) : __assert_fail ("!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && \"Conflicting divergence information!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9271, __extension__ __PRETTY_FUNCTION__))
;
9272 return false;
9273 }
9274 if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA))
9275 return true;
9276 for (auto &Op : N->ops()) {
9277 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
9278 return true;
9279 }
9280 return false;
9281}
9282
9283void SelectionDAG::updateDivergence(SDNode *N) {
9284 SmallVector<SDNode *, 16> Worklist(1, N);
9285 do {
9286 N = Worklist.pop_back_val();
9287 bool IsDivergent = calculateDivergence(N);
9288 if (N->SDNodeBits.IsDivergent != IsDivergent) {
9289 N->SDNodeBits.IsDivergent = IsDivergent;
9290 llvm::append_range(Worklist, N->uses());
9291 }
9292 } while (!Worklist.empty());
9293}
9294
9295void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
9296 DenseMap<SDNode *, unsigned> Degree;
9297 Order.reserve(AllNodes.size());
9298 for (auto &N : allnodes()) {
9299 unsigned NOps = N.getNumOperands();
9300 Degree[&N] = NOps;
9301 if (0 == NOps)
9302 Order.push_back(&N);
9303 }
9304 for (size_t I = 0; I != Order.size(); ++I) {
9305 SDNode *N = Order[I];
9306 for (auto U : N->uses()) {
9307 unsigned &UnsortedOps = Degree[U];
9308 if (0 == --UnsortedOps)
9309 Order.push_back(U);
9310 }
9311 }
9312}
9313
9314#ifndef NDEBUG
9315void SelectionDAG::VerifyDAGDiverence() {
9316 std::vector<SDNode *> TopoOrder;
9317 CreateTopologicalOrder(TopoOrder);
9318 for (auto *N : TopoOrder) {
9319 assert(calculateDivergence(N) == N->isDivergent() &&(static_cast <bool> (calculateDivergence(N) == N->isDivergent
() && "Divergence bit inconsistency detected") ? void
(0) : __assert_fail ("calculateDivergence(N) == N->isDivergent() && \"Divergence bit inconsistency detected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9320, __extension__ __PRETTY_FUNCTION__))
9320 "Divergence bit inconsistency detected")(static_cast <bool> (calculateDivergence(N) == N->isDivergent
() && "Divergence bit inconsistency detected") ? void
(0) : __assert_fail ("calculateDivergence(N) == N->isDivergent() && \"Divergence bit inconsistency detected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9320, __extension__ __PRETTY_FUNCTION__))
;
9321 }
9322}
9323#endif
9324
9325/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
9326/// uses of other values produced by From.getNode() alone. The same value
9327/// may appear in both the From and To list. The Deleted vector is
9328/// handled the same way as for ReplaceAllUsesWith.
9329void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
9330 const SDValue *To,
9331 unsigned Num){
9332 // Handle the simple, trivial case efficiently.
9333 if (Num == 1)
9334 return ReplaceAllUsesOfValueWith(*From, *To);
9335
9336 transferDbgValues(*From, *To);
9337
9338 // Read up all the uses and make records of them. This helps
9339 // processing new uses that are introduced during the
9340 // replacement process.
9341 SmallVector<UseMemo, 4> Uses;
9342 for (unsigned i = 0; i != Num; ++i) {
9343 unsigned FromResNo = From[i].getResNo();
9344 SDNode *FromNode = From[i].getNode();
9345 for (SDNode::use_iterator UI = FromNode->use_begin(),
9346 E = FromNode->use_end(); UI != E; ++UI) {
9347 SDUse &Use = UI.getUse();
9348 if (Use.getResNo() == FromResNo) {
9349 UseMemo Memo = { *UI, i, &Use };
9350 Uses.push_back(Memo);
9351 }
9352 }
9353 }
9354
9355 // Sort the uses, so that all the uses from a given User are together.
9356 llvm::sort(Uses);
9357
9358 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
9359 UseIndex != UseIndexEnd; ) {
9360 // We know that this user uses some value of From. If it is the right
9361 // value, update it.
9362 SDNode *User = Uses[UseIndex].User;
9363
9364 // This node is about to morph, remove its old self from the CSE maps.
9365 RemoveNodeFromCSEMaps(User);
9366
9367 // The Uses array is sorted, so all the uses for a given User
9368 // are next to each other in the list.
9369 // To help reduce the number of CSE recomputations, process all
9370 // the uses of this user that we can find this way.
9371 do {
9372 unsigned i = Uses[UseIndex].Index;
9373 SDUse &Use = *Uses[UseIndex].Use;
9374 ++UseIndex;
9375
9376 Use.set(To[i]);
9377 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
9378
9379 // Now that we have modified User, add it back to the CSE maps. If it
9380 // already exists there, recursively merge the results together.
9381 AddModifiedNodeToCSEMaps(User);
9382 }
9383}
9384
9385/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
9386/// based on their topological order. It returns the maximum id and a vector
9387/// of the SDNodes* in assigned order by reference.
9388unsigned SelectionDAG::AssignTopologicalOrder() {
9389 unsigned DAGSize = 0;
9390
9391 // SortedPos tracks the progress of the algorithm. Nodes before it are
9392 // sorted, nodes after it are unsorted. When the algorithm completes
9393 // it is at the end of the list.
9394 allnodes_iterator SortedPos = allnodes_begin();
9395
9396 // Visit all the nodes. Move nodes with no operands to the front of
9397 // the list immediately. Annotate nodes that do have operands with their
9398 // operand count. Before we do this, the Node Id fields of the nodes
9399 // may contain arbitrary values. After, the Node Id fields for nodes
9400 // before SortedPos will contain the topological sort index, and the
9401 // Node Id fields for nodes At SortedPos and after will contain the
9402 // count of outstanding operands.
9403 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
9404 SDNode *N = &*I++;
9405 checkForCycles(N, this);
9406 unsigned Degree = N->getNumOperands();
9407 if (Degree == 0) {
9408 // A node with no uses, add it to the result array immediately.
9409 N->setNodeId(DAGSize++);
9410 allnodes_iterator Q(N);
9411 if (Q != SortedPos)
9412 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
9413 assert(SortedPos != AllNodes.end() && "Overran node list")(static_cast <bool> (SortedPos != AllNodes.end() &&
"Overran node list") ? void (0) : __assert_fail ("SortedPos != AllNodes.end() && \"Overran node list\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9413, __extension__ __PRETTY_FUNCTION__))
;
9414 ++SortedPos;
9415 } else {
9416 // Temporarily use the Node Id as scratch space for the degree count.
9417 N->setNodeId(Degree);
9418 }
9419 }
9420
9421 // Visit all the nodes. As we iterate, move nodes into sorted order,
9422 // such that by the time the end is reached all nodes will be sorted.
9423 for (SDNode &Node : allnodes()) {
9424 SDNode *N = &Node;
9425 checkForCycles(N, this);
9426 // N is in sorted position, so all its uses have one less operand
9427 // that needs to be sorted.
9428 for (SDNode *P : N->uses()) {
9429 unsigned Degree = P->getNodeId();
9430 assert(Degree != 0 && "Invalid node degree")(static_cast <bool> (Degree != 0 && "Invalid node degree"
) ? void (0) : __assert_fail ("Degree != 0 && \"Invalid node degree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9430, __extension__ __PRETTY_FUNCTION__))
;
9431 --Degree;
9432 if (Degree == 0) {
9433 // All of P's operands are sorted, so P may sorted now.
9434 P->setNodeId(DAGSize++);
9435 if (P->getIterator() != SortedPos)
9436 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
9437 assert(SortedPos != AllNodes.end() && "Overran node list")(static_cast <bool> (SortedPos != AllNodes.end() &&
"Overran node list") ? void (0) : __assert_fail ("SortedPos != AllNodes.end() && \"Overran node list\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9437, __extension__ __PRETTY_FUNCTION__))
;
9438 ++SortedPos;
9439 } else {
9440 // Update P's outstanding operand count.
9441 P->setNodeId(Degree);
9442 }
9443 }
9444 if (Node.getIterator() == SortedPos) {
9445#ifndef NDEBUG
9446 allnodes_iterator I(N);
9447 SDNode *S = &*++I;
9448 dbgs() << "Overran sorted position:\n";
9449 S->dumprFull(this); dbgs() << "\n";
9450 dbgs() << "Checking if this is due to cycles\n";
9451 checkForCycles(this, true);
9452#endif
9453 llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9453)
;
9454 }
9455 }
9456
9457 assert(SortedPos == AllNodes.end() &&(static_cast <bool> (SortedPos == AllNodes.end() &&
"Topological sort incomplete!") ? void (0) : __assert_fail (
"SortedPos == AllNodes.end() && \"Topological sort incomplete!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9458, __extension__ __PRETTY_FUNCTION__))
9458 "Topological sort incomplete!")(static_cast <bool> (SortedPos == AllNodes.end() &&
"Topological sort incomplete!") ? void (0) : __assert_fail (
"SortedPos == AllNodes.end() && \"Topological sort incomplete!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9458, __extension__ __PRETTY_FUNCTION__))
;
9459 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&(static_cast <bool> (AllNodes.front().getOpcode() == ISD
::EntryToken && "First node in topological sort is not the entry token!"
) ? void (0) : __assert_fail ("AllNodes.front().getOpcode() == ISD::EntryToken && \"First node in topological sort is not the entry token!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9460, __extension__ __PRETTY_FUNCTION__))
9460 "First node in topological sort is not the entry token!")(static_cast <bool> (AllNodes.front().getOpcode() == ISD
::EntryToken && "First node in topological sort is not the entry token!"
) ? void (0) : __assert_fail ("AllNodes.front().getOpcode() == ISD::EntryToken && \"First node in topological sort is not the entry token!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9460, __extension__ __PRETTY_FUNCTION__))
;
9461 assert(AllNodes.front().getNodeId() == 0 &&(static_cast <bool> (AllNodes.front().getNodeId() == 0 &&
"First node in topological sort has non-zero id!") ? void (0
) : __assert_fail ("AllNodes.front().getNodeId() == 0 && \"First node in topological sort has non-zero id!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9462, __extension__ __PRETTY_FUNCTION__))
9462 "First node in topological sort has non-zero id!")(static_cast <bool> (AllNodes.front().getNodeId() == 0 &&
"First node in topological sort has non-zero id!") ? void (0
) : __assert_fail ("AllNodes.front().getNodeId() == 0 && \"First node in topological sort has non-zero id!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9462, __extension__ __PRETTY_FUNCTION__))
;
9463 assert(AllNodes.front().getNumOperands() == 0 &&(static_cast <bool> (AllNodes.front().getNumOperands() ==
0 && "First node in topological sort has operands!")
? void (0) : __assert_fail ("AllNodes.front().getNumOperands() == 0 && \"First node in topological sort has operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9464, __extension__ __PRETTY_FUNCTION__))
9464 "First node in topological sort has operands!")(static_cast <bool> (AllNodes.front().getNumOperands() ==
0 && "First node in topological sort has operands!")
? void (0) : __assert_fail ("AllNodes.front().getNumOperands() == 0 && \"First node in topological sort has operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9464, __extension__ __PRETTY_FUNCTION__))
;
9465 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&(static_cast <bool> (AllNodes.back().getNodeId() == (int
)DAGSize-1 && "Last node in topologic sort has unexpected id!"
) ? void (0) : __assert_fail ("AllNodes.back().getNodeId() == (int)DAGSize-1 && \"Last node in topologic sort has unexpected id!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9466, __extension__ __PRETTY_FUNCTION__))
9466 "Last node in topologic sort has unexpected id!")(static_cast <bool> (AllNodes.back().getNodeId() == (int
)DAGSize-1 && "Last node in topologic sort has unexpected id!"
) ? void (0) : __assert_fail ("AllNodes.back().getNodeId() == (int)DAGSize-1 && \"Last node in topologic sort has unexpected id!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9466, __extension__ __PRETTY_FUNCTION__))
;
9467 assert(AllNodes.back().use_empty() &&(static_cast <bool> (AllNodes.back().use_empty() &&
"Last node in topologic sort has users!") ? void (0) : __assert_fail
("AllNodes.back().use_empty() && \"Last node in topologic sort has users!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9468, __extension__ __PRETTY_FUNCTION__))
9468 "Last node in topologic sort has users!")(static_cast <bool> (AllNodes.back().use_empty() &&
"Last node in topologic sort has users!") ? void (0) : __assert_fail
("AllNodes.back().use_empty() && \"Last node in topologic sort has users!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9468, __extension__ __PRETTY_FUNCTION__))
;
9469 assert(DAGSize == allnodes_size() && "Node count mismatch!")(static_cast <bool> (DAGSize == allnodes_size() &&
"Node count mismatch!") ? void (0) : __assert_fail ("DAGSize == allnodes_size() && \"Node count mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9469, __extension__ __PRETTY_FUNCTION__))
;
9470 return DAGSize;
9471}
9472
9473/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
9474/// value is produced by SD.
9475void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
9476 for (SDNode *SD : DB->getSDNodes()) {
9477 if (!SD)
9478 continue;
9479 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue())(static_cast <bool> (DbgInfo->getSDDbgValues(SD).empty
() || SD->getHasDebugValue()) ? void (0) : __assert_fail (
"DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9479, __extension__ __PRETTY_FUNCTION__))
;
9480 SD->setHasDebugValue(true);
9481 }
9482 DbgInfo->add(DB, isParameter);
9483}
9484
9485void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
9486
9487SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
9488 SDValue NewMemOpChain) {
9489 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node")(static_cast <bool> (isa<MemSDNode>(NewMemOpChain
) && "Expected a memop node") ? void (0) : __assert_fail
("isa<MemSDNode>(NewMemOpChain) && \"Expected a memop node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9489, __extension__ __PRETTY_FUNCTION__))
;
9490 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT")(static_cast <bool> (NewMemOpChain.getValueType() == MVT
::Other && "Expected a token VT") ? void (0) : __assert_fail
("NewMemOpChain.getValueType() == MVT::Other && \"Expected a token VT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9490, __extension__ __PRETTY_FUNCTION__))
;
9491 // The new memory operation must have the same position as the old load in
9492 // terms of memory dependency. Create a TokenFactor for the old load and new
9493 // memory operation and update uses of the old load's output chain to use that
9494 // TokenFactor.
9495 if (OldChain == NewMemOpChain || OldChain.use_empty())
9496 return NewMemOpChain;
9497
9498 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
9499 OldChain, NewMemOpChain);
9500 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
9501 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
9502 return TokenFactor;
9503}
9504
9505SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
9506 SDValue NewMemOp) {
9507 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node")(static_cast <bool> (isa<MemSDNode>(NewMemOp.getNode
()) && "Expected a memop node") ? void (0) : __assert_fail
("isa<MemSDNode>(NewMemOp.getNode()) && \"Expected a memop node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9507, __extension__ __PRETTY_FUNCTION__))
;
9508 SDValue OldChain = SDValue(OldLoad, 1);
9509 SDValue NewMemOpChain = NewMemOp.getValue(1);
9510 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
9511}
9512
9513SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
9514 Function **OutFunction) {
9515 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol")(static_cast <bool> (isa<ExternalSymbolSDNode>(Op
) && "Node should be an ExternalSymbol") ? void (0) :
__assert_fail ("isa<ExternalSymbolSDNode>(Op) && \"Node should be an ExternalSymbol\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9515, __extension__ __PRETTY_FUNCTION__))
;
9516
9517 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
9518 auto *Module = MF->getFunction().getParent();
9519 auto *Function = Module->getFunction(Symbol);
9520
9521 if (OutFunction != nullptr)
9522 *OutFunction = Function;
9523
9524 if (Function != nullptr) {
9525 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
9526 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
9527 }
9528
9529 std::string ErrorStr;
9530 raw_string_ostream ErrorFormatter(ErrorStr);
9531
9532 ErrorFormatter << "Undefined external symbol ";
9533 ErrorFormatter << '"' << Symbol << '"';
9534 ErrorFormatter.flush();
9535
9536 report_fatal_error(ErrorStr);
9537}
9538
9539//===----------------------------------------------------------------------===//
9540// SDNode Class
9541//===----------------------------------------------------------------------===//
9542
9543bool llvm::isNullConstant(SDValue V) {
9544 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9545 return Const != nullptr && Const->isNullValue();
9546}
9547
9548bool llvm::isNullFPConstant(SDValue V) {
9549 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
9550 return Const != nullptr && Const->isZero() && !Const->isNegative();
9551}
9552
9553bool llvm::isAllOnesConstant(SDValue V) {
9554 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9555 return Const != nullptr && Const->isAllOnesValue();
9556}
9557
9558bool llvm::isOneConstant(SDValue V) {
9559 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9560 return Const != nullptr && Const->isOne();
9561}
9562
9563SDValue llvm::peekThroughBitcasts(SDValue V) {
9564 while (V.getOpcode() == ISD::BITCAST)
9565 V = V.getOperand(0);
9566 return V;
9567}
9568
9569SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
9570 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
9571 V = V.getOperand(0);
9572 return V;
9573}
9574
9575SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
9576 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
9577 V = V.getOperand(0);
9578 return V;
9579}
9580
9581bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
9582 if (V.getOpcode() != ISD::XOR)
9583 return false;
9584 V = peekThroughBitcasts(V.getOperand(1));
9585 unsigned NumBits = V.getScalarValueSizeInBits();
9586 ConstantSDNode *C =
9587 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
9588 return C && (C->getAPIntValue().countTrailingOnes() >= NumBits);
9589}
9590
9591ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
9592 bool AllowTruncation) {
9593 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
9594 return CN;
9595
9596 // SplatVectors can truncate their operands. Ignore that case here unless
9597 // AllowTruncation is set.
9598 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
9599 EVT VecEltVT = N->getValueType(0).getVectorElementType();
9600 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
9601 EVT CVT = CN->getValueType(0);
9602 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension")(static_cast <bool> (CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"
) ? void (0) : __assert_fail ("CVT.bitsGE(VecEltVT) && \"Illegal splat_vector element extension\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9602, __extension__ __PRETTY_FUNCTION__))
;
9603 if (AllowTruncation || CVT == VecEltVT)
9604 return CN;
9605 }
9606 }
9607
9608 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9609 BitVector UndefElements;
9610 ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
9611
9612 // BuildVectors can truncate their operands. Ignore that case here unless
9613 // AllowTruncation is set.
9614 if (CN && (UndefElements.none() || AllowUndefs)) {
9615 EVT CVT = CN->getValueType(0);
9616 EVT NSVT = N.getValueType().getScalarType();
9617 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension")(static_cast <bool> (CVT.bitsGE(NSVT) && "Illegal build vector element extension"
) ? void (0) : __assert_fail ("CVT.bitsGE(NSVT) && \"Illegal build vector element extension\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9617, __extension__ __PRETTY_FUNCTION__))
;
9618 if (AllowTruncation || (CVT == NSVT))
9619 return CN;
9620 }
9621 }
9622
9623 return nullptr;
9624}
9625
9626ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
9627 bool AllowUndefs,
9628 bool AllowTruncation) {
9629 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
9630 return CN;
9631
9632 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9633 BitVector UndefElements;
9634 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
9635
9636 // BuildVectors can truncate their operands. Ignore that case here unless
9637 // AllowTruncation is set.
9638 if (CN && (UndefElements.none() || AllowUndefs)) {
9639 EVT CVT = CN->getValueType(0);
9640 EVT NSVT = N.getValueType().getScalarType();
9641 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension")(static_cast <bool> (CVT.bitsGE(NSVT) && "Illegal build vector element extension"
) ? void (0) : __assert_fail ("CVT.bitsGE(NSVT) && \"Illegal build vector element extension\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9641, __extension__ __PRETTY_FUNCTION__))
;
9642 if (AllowTruncation || (CVT == NSVT))
9643 return CN;
9644 }
9645 }
9646
9647 return nullptr;
9648}
9649
9650ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
9651 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
9652 return CN;
9653
9654 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9655 BitVector UndefElements;
9656 ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
9657 if (CN && (UndefElements.none() || AllowUndefs))
9658 return CN;
9659 }
9660
9661 if (N.getOpcode() == ISD::SPLAT_VECTOR)
9662 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
9663 return CN;
9664
9665 return nullptr;
9666}
9667
9668ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
9669 const APInt &DemandedElts,
9670 bool AllowUndefs) {
9671 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
9672 return CN;
9673
9674 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9675 BitVector UndefElements;
9676 ConstantFPSDNode *CN =
9677 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
9678 if (CN && (UndefElements.none() || AllowUndefs))
9679 return CN;
9680 }
9681
9682 return nullptr;
9683}
9684
9685bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
9686 // TODO: may want to use peekThroughBitcast() here.
9687 ConstantSDNode *C =
9688 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
9689 return C && C->isNullValue();
9690}
9691
9692bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
9693 // TODO: may want to use peekThroughBitcast() here.
9694 unsigned BitWidth = N.getScalarValueSizeInBits();
9695 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
9696 return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth;
9697}
9698
9699bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
9700 N = peekThroughBitcasts(N);
9701 unsigned BitWidth = N.getScalarValueSizeInBits();
9702 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
9703 return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth;
9704}
9705
9706HandleSDNode::~HandleSDNode() {
9707 DropOperands();
9708}
9709
9710GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
9711 const DebugLoc &DL,
9712 const GlobalValue *GA, EVT VT,
9713 int64_t o, unsigned TF)
9714 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
9715 TheGlobal = GA;
9716}
9717
9718AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
9719 EVT VT, unsigned SrcAS,
9720 unsigned DestAS)
9721 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
9722 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
9723
9724MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
9725 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
9726 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
9727 MemSDNodeBits.IsVolatile = MMO->isVolatile();
9728 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
9729 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
9730 MemSDNodeBits.IsInvariant = MMO->isInvariant();
9731
9732 // We check here that the size of the memory operand fits within the size of
9733 // the MMO. This is because the MMO might indicate only a possible address
9734 // range instead of specifying the affected memory addresses precisely.
9735 // TODO: Make MachineMemOperands aware of scalable vectors.
9736 assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() &&(static_cast <bool> (memvt.getStoreSize().getKnownMinSize
() <= MMO->getSize() && "Size mismatch!") ? void
(0) : __assert_fail ("memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() && \"Size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9737, __extension__ __PRETTY_FUNCTION__))
9737 "Size mismatch!")(static_cast <bool> (memvt.getStoreSize().getKnownMinSize
() <= MMO->getSize() && "Size mismatch!") ? void
(0) : __assert_fail ("memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() && \"Size mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9737, __extension__ __PRETTY_FUNCTION__))
;
9738}
9739
9740/// Profile - Gather unique data for the node.
9741///
9742void SDNode::Profile(FoldingSetNodeID &ID) const {
9743 AddNodeIDNode(ID, this);
9744}
9745
9746namespace {
9747
9748 struct EVTArray {
9749 std::vector<EVT> VTs;
9750
9751 EVTArray() {
9752 VTs.reserve(MVT::VALUETYPE_SIZE);
9753 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
9754 VTs.push_back(MVT((MVT::SimpleValueType)i));
9755 }
9756 };
9757
9758} // end anonymous namespace
9759
9760static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
9761static ManagedStatic<EVTArray> SimpleVTArray;
9762static ManagedStatic<sys::SmartMutex<true>> VTMutex;
9763
9764/// getValueTypeList - Return a pointer to the specified value type.
9765///
9766const EVT *SDNode::getValueTypeList(EVT VT) {
9767 if (VT.isExtended()) {
9768 sys::SmartScopedLock<true> Lock(*VTMutex);
9769 return &(*EVTs->insert(VT).first);
9770 }
9771 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!")(static_cast <bool> (VT.getSimpleVT() < MVT::VALUETYPE_SIZE
&& "Value type out of range!") ? void (0) : __assert_fail
("VT.getSimpleVT() < MVT::VALUETYPE_SIZE && \"Value type out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9771, __extension__ __PRETTY_FUNCTION__))
;
9772 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
9773}
9774
9775/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
9776/// indicated value. This method ignores uses of other values defined by this
9777/// operation.
9778bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
9779 assert(Value < getNumValues() && "Bad value!")(static_cast <bool> (Value < getNumValues() &&
"Bad value!") ? void (0) : __assert_fail ("Value < getNumValues() && \"Bad value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9779, __extension__ __PRETTY_FUNCTION__))
;
9780
9781 // TODO: Only iterate over uses of a given value of the node
9782 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
9783 if (UI.getUse().getResNo() == Value) {
9784 if (NUses == 0)
9785 return false;
9786 --NUses;
9787 }
9788 }
9789
9790 // Found exactly the right number of uses?
9791 return NUses == 0;
9792}
9793
9794/// hasAnyUseOfValue - Return true if there are any use of the indicated
9795/// value. This method ignores uses of other values defined by this operation.
9796bool SDNode::hasAnyUseOfValue(unsigned Value) const {
9797 assert(Value < getNumValues() && "Bad value!")(static_cast <bool> (Value < getNumValues() &&
"Bad value!") ? void (0) : __assert_fail ("Value < getNumValues() && \"Bad value!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9797, __extension__ __PRETTY_FUNCTION__))
;
9798
9799 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
9800 if (UI.getUse().getResNo() == Value)
9801 return true;
9802
9803 return false;
9804}
9805
9806/// isOnlyUserOf - Return true if this node is the only use of N.
9807bool SDNode::isOnlyUserOf(const SDNode *N) const {
9808 bool Seen = false;
9809 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9810 SDNode *User = *I;
9811 if (User == this)
9812 Seen = true;
9813 else
9814 return false;
9815 }
9816
9817 return Seen;
9818}
9819
9820/// Return true if the only users of N are contained in Nodes.
9821bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
9822 bool Seen = false;
9823 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9824 SDNode *User = *I;
9825 if (llvm::is_contained(Nodes, User))
9826 Seen = true;
9827 else
9828 return false;
9829 }
9830
9831 return Seen;
9832}
9833
9834/// isOperand - Return true if this node is an operand of N.
9835bool SDValue::isOperandOf(const SDNode *N) const {
9836 return is_contained(N->op_values(), *this);
9837}
9838
9839bool SDNode::isOperandOf(const SDNode *N) const {
9840 return any_of(N->op_values(),
9841 [this](SDValue Op) { return this == Op.getNode(); });
9842}
9843
9844/// reachesChainWithoutSideEffects - Return true if this operand (which must
9845/// be a chain) reaches the specified operand without crossing any
9846/// side-effecting instructions on any chain path. In practice, this looks
9847/// through token factors and non-volatile loads. In order to remain efficient,
9848/// this only looks a couple of nodes in, it does not do an exhaustive search.
9849///
9850/// Note that we only need to examine chains when we're searching for
9851/// side-effects; SelectionDAG requires that all side-effects are represented
9852/// by chains, even if another operand would force a specific ordering. This
9853/// constraint is necessary to allow transformations like splitting loads.
9854bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
9855 unsigned Depth) const {
9856 if (*this == Dest) return true;
9857
9858 // Don't search too deeply, we just want to be able to see through
9859 // TokenFactor's etc.
9860 if (Depth == 0) return false;
9861
9862 // If this is a token factor, all inputs to the TF happen in parallel.
9863 if (getOpcode() == ISD::TokenFactor) {
9864 // First, try a shallow search.
9865 if (is_contained((*this)->ops(), Dest)) {
9866 // We found the chain we want as an operand of this TokenFactor.
9867 // Essentially, we reach the chain without side-effects if we could
9868 // serialize the TokenFactor into a simple chain of operations with
9869 // Dest as the last operation. This is automatically true if the
9870 // chain has one use: there are no other ordering constraints.
9871 // If the chain has more than one use, we give up: some other
9872 // use of Dest might force a side-effect between Dest and the current
9873 // node.
9874 if (Dest.hasOneUse())
9875 return true;
9876 }
9877 // Next, try a deep search: check whether every operand of the TokenFactor
9878 // reaches Dest.
9879 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
9880 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
9881 });
9882 }
9883
9884 // Loads don't have side effects, look through them.
9885 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
9886 if (Ld->isUnordered())
9887 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
9888 }
9889 return false;
9890}
9891
9892bool SDNode::hasPredecessor(const SDNode *N) const {
9893 SmallPtrSet<const SDNode *, 32> Visited;
9894 SmallVector<const SDNode *, 16> Worklist;
9895 Worklist.push_back(this);
9896 return hasPredecessorHelper(N, Visited, Worklist);
9897}
9898
9899void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
9900 this->Flags.intersectWith(Flags);
9901}
9902
9903SDValue
9904SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
9905 ArrayRef<ISD::NodeType> CandidateBinOps,
9906 bool AllowPartials) {
9907 // The pattern must end in an extract from index 0.
9908 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9909 !isNullConstant(Extract->getOperand(1)))
9910 return SDValue();
9911
9912 // Match against one of the candidate binary ops.
9913 SDValue Op = Extract->getOperand(0);
9914 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
9915 return Op.getOpcode() == unsigned(BinOp);
9916 }))
9917 return SDValue();
9918
9919 // Floating-point reductions may require relaxed constraints on the final step
9920 // of the reduction because they may reorder intermediate operations.
9921 unsigned CandidateBinOp = Op.getOpcode();
9922 if (Op.getValueType().isFloatingPoint()) {
9923 SDNodeFlags Flags = Op->getFlags();
9924 switch (CandidateBinOp) {
9925 case ISD::FADD:
9926 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
9927 return SDValue();
9928 break;
9929 default:
9930 llvm_unreachable("Unhandled FP opcode for binop reduction")::llvm::llvm_unreachable_internal("Unhandled FP opcode for binop reduction"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 9930)
;
9931 }
9932 }
9933
9934 // Matching failed - attempt to see if we did enough stages that a partial
9935 // reduction from a subvector is possible.
9936 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
9937 if (!AllowPartials || !Op)
9938 return SDValue();
9939 EVT OpVT = Op.getValueType();
9940 EVT OpSVT = OpVT.getScalarType();
9941 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
9942 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
9943 return SDValue();
9944 BinOp = (ISD::NodeType)CandidateBinOp;
9945 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
9946 getVectorIdxConstant(0, SDLoc(Op)));
9947 };
9948
9949 // At each stage, we're looking for something that looks like:
9950 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
9951 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
9952 // i32 undef, i32 undef, i32 undef, i32 undef>
9953 // %a = binop <8 x i32> %op, %s
9954 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
9955 // we expect something like:
9956 // <4,5,6,7,u,u,u,u>
9957 // <2,3,u,u,u,u,u,u>
9958 // <1,u,u,u,u,u,u,u>
9959 // While a partial reduction match would be:
9960 // <2,3,u,u,u,u,u,u>
9961 // <1,u,u,u,u,u,u,u>
9962 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
9963 SDValue PrevOp;
9964 for (unsigned i = 0; i < Stages; ++i) {
9965 unsigned MaskEnd = (1 << i);
9966
9967 if (Op.getOpcode() != CandidateBinOp)
9968 return PartialReduction(PrevOp, MaskEnd);
9969
9970 SDValue Op0 = Op.getOperand(0);
9971 SDValue Op1 = Op.getOperand(1);
9972
9973 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
9974 if (Shuffle) {
9975 Op = Op1;
9976 } else {
9977 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
9978 Op = Op0;
9979 }
9980
9981 // The first operand of the shuffle should be the same as the other operand
9982 // of the binop.
9983 if (!Shuffle || Shuffle->getOperand(0) != Op)
9984 return PartialReduction(PrevOp, MaskEnd);
9985
9986 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
9987 for (int Index = 0; Index < (int)MaskEnd; ++Index)
9988 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
9989 return PartialReduction(PrevOp, MaskEnd);
9990
9991 PrevOp = Op;
9992 }
9993
9994 // Handle subvector reductions, which tend to appear after the shuffle
9995 // reduction stages.
9996 while (Op.getOpcode() == CandidateBinOp) {
9997 unsigned NumElts = Op.getValueType().getVectorNumElements();
9998 SDValue Op0 = Op.getOperand(0);
9999 SDValue Op1 = Op.getOperand(1);
10000 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
10001 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
10002 Op0.getOperand(0) != Op1.getOperand(0))
10003 break;
10004 SDValue Src = Op0.getOperand(0);
10005 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
10006 if (NumSrcElts != (2 * NumElts))
10007 break;
10008 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
10009 Op1.getConstantOperandAPInt(1) == NumElts) &&
10010 !(Op1.getConstantOperandAPInt(1) == 0 &&
10011 Op0.getConstantOperandAPInt(1) == NumElts))
10012 break;
10013 Op = Src;
10014 }
10015
10016 BinOp = (ISD::NodeType)CandidateBinOp;
10017 return Op;
10018}
10019
10020SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
10021 assert(N->getNumValues() == 1 &&(static_cast <bool> (N->getNumValues() == 1 &&
"Can't unroll a vector with multiple results!") ? void (0) :
__assert_fail ("N->getNumValues() == 1 && \"Can't unroll a vector with multiple results!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10022, __extension__ __PRETTY_FUNCTION__))
10022 "Can't unroll a vector with multiple results!")(static_cast <bool> (N->getNumValues() == 1 &&
"Can't unroll a vector with multiple results!") ? void (0) :
__assert_fail ("N->getNumValues() == 1 && \"Can't unroll a vector with multiple results!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10022, __extension__ __PRETTY_FUNCTION__))
;
10023
10024 EVT VT = N->getValueType(0);
10025 unsigned NE = VT.getVectorNumElements();
10026 EVT EltVT = VT.getVectorElementType();
10027 SDLoc dl(N);
10028
10029 SmallVector<SDValue, 8> Scalars;
10030 SmallVector<SDValue, 4> Operands(N->getNumOperands());
10031
10032 // If ResNE is 0, fully unroll the vector op.
10033 if (ResNE == 0)
10034 ResNE = NE;
10035 else if (NE > ResNE)
10036 NE = ResNE;
10037
10038 unsigned i;
10039 for (i= 0; i != NE; ++i) {
10040 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
10041 SDValue Operand = N->getOperand(j);
10042 EVT OperandVT = Operand.getValueType();
10043 if (OperandVT.isVector()) {
10044 // A vector operand; extract a single element.
10045 EVT OperandEltVT = OperandVT.getVectorElementType();
10046 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
10047 Operand, getVectorIdxConstant(i, dl));
10048 } else {
10049 // A scalar operand; just use it as is.
10050 Operands[j] = Operand;
10051 }
10052 }
10053
10054 switch (N->getOpcode()) {
10055 default: {
10056 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
10057 N->getFlags()));
10058 break;
10059 }
10060 case ISD::VSELECT:
10061 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
10062 break;
10063 case ISD::SHL:
10064 case ISD::SRA:
10065 case ISD::SRL:
10066 case ISD::ROTL:
10067 case ISD::ROTR:
10068 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
10069 getShiftAmountOperand(Operands[0].getValueType(),
10070 Operands[1])));
10071 break;
10072 case ISD::SIGN_EXTEND_INREG: {
10073 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
10074 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
10075 Operands[0],
10076 getValueType(ExtVT)));
10077 }
10078 }
10079 }
10080
10081 for (; i < ResNE; ++i)
10082 Scalars.push_back(getUNDEF(EltVT));
10083
10084 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
10085 return getBuildVector(VecVT, dl, Scalars);
10086}
10087
10088std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
10089 SDNode *N, unsigned ResNE) {
10090 unsigned Opcode = N->getOpcode();
10091 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||(static_cast <bool> ((Opcode == ISD::UADDO || Opcode ==
ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
Opcode == ISD::UMULO || Opcode == ISD::SMULO) && "Expected an overflow opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::UADDO || Opcode == ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO || Opcode == ISD::UMULO || Opcode == ISD::SMULO) && \"Expected an overflow opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10094, __extension__ __PRETTY_FUNCTION__))
10092 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||(static_cast <bool> ((Opcode == ISD::UADDO || Opcode ==
ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
Opcode == ISD::UMULO || Opcode == ISD::SMULO) && "Expected an overflow opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::UADDO || Opcode == ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO || Opcode == ISD::UMULO || Opcode == ISD::SMULO) && \"Expected an overflow opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10094, __extension__ __PRETTY_FUNCTION__))
10093 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&(static_cast <bool> ((Opcode == ISD::UADDO || Opcode ==
ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
Opcode == ISD::UMULO || Opcode == ISD::SMULO) && "Expected an overflow opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::UADDO || Opcode == ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO || Opcode == ISD::UMULO || Opcode == ISD::SMULO) && \"Expected an overflow opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10094, __extension__ __PRETTY_FUNCTION__))
10094 "Expected an overflow opcode")(static_cast <bool> ((Opcode == ISD::UADDO || Opcode ==
ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
Opcode == ISD::UMULO || Opcode == ISD::SMULO) && "Expected an overflow opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::UADDO || Opcode == ISD::SADDO || Opcode == ISD::USUBO || Opcode == ISD::SSUBO || Opcode == ISD::UMULO || Opcode == ISD::SMULO) && \"Expected an overflow opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10094, __extension__ __PRETTY_FUNCTION__))
;
10095
10096 EVT ResVT = N->getValueType(0);
10097 EVT OvVT = N->getValueType(1);
10098 EVT ResEltVT = ResVT.getVectorElementType();
10099 EVT OvEltVT = OvVT.getVectorElementType();
10100 SDLoc dl(N);
10101
10102 // If ResNE is 0, fully unroll the vector op.
10103 unsigned NE = ResVT.getVectorNumElements();
10104 if (ResNE == 0)
10105 ResNE = NE;
10106 else if (NE > ResNE)
10107 NE = ResNE;
10108
10109 SmallVector<SDValue, 8> LHSScalars;
10110 SmallVector<SDValue, 8> RHSScalars;
10111 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
10112 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
10113
10114 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
10115 SDVTList VTs = getVTList(ResEltVT, SVT);
10116 SmallVector<SDValue, 8> ResScalars;
10117 SmallVector<SDValue, 8> OvScalars;
10118 for (unsigned i = 0; i < NE; ++i) {
10119 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
10120 SDValue Ov =
10121 getSelect(dl, OvEltVT, Res.getValue(1),
10122 getBoolConstant(true, dl, OvEltVT, ResVT),
10123 getConstant(0, dl, OvEltVT));
10124
10125 ResScalars.push_back(Res);
10126 OvScalars.push_back(Ov);
10127 }
10128
10129 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
10130 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
10131
10132 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
10133 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
10134 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
10135 getBuildVector(NewOvVT, dl, OvScalars));
10136}
10137
10138bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
10139 LoadSDNode *Base,
10140 unsigned Bytes,
10141 int Dist) const {
10142 if (LD->isVolatile() || Base->isVolatile())
10143 return false;
10144 // TODO: probably too restrictive for atomics, revisit
10145 if (!LD->isSimple())
10146 return false;
10147 if (LD->isIndexed() || Base->isIndexed())
10148 return false;
10149 if (LD->getChain() != Base->getChain())
10150 return false;
10151 EVT VT = LD->getValueType(0);
10152 if (VT.getSizeInBits() / 8 != Bytes)
10153 return false;
10154
10155 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
10156 auto LocDecomp = BaseIndexOffset::match(LD, *this);
10157
10158 int64_t Offset = 0;
10159 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
10160 return (Dist * Bytes == Offset);
10161 return false;
10162}
10163
10164/// InferPtrAlignment - Infer alignment of a load / store address. Return None
10165/// if it cannot be inferred.
10166MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
10167 // If this is a GlobalAddress + cst, return the alignment.
10168 const GlobalValue *GV = nullptr;
10169 int64_t GVOffset = 0;
10170 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
10171 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
10172 KnownBits Known(PtrWidth);
10173 llvm::computeKnownBits(GV, Known, getDataLayout());
10174 unsigned AlignBits = Known.countMinTrailingZeros();
10175 if (AlignBits)
10176 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
10177 }
10178
10179 // If this is a direct reference to a stack slot, use information about the
10180 // stack slot's alignment.
10181 int FrameIdx = INT_MIN(-2147483647 -1);
10182 int64_t FrameOffset = 0;
10183 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
10184 FrameIdx = FI->getIndex();
10185 } else if (isBaseWithConstantOffset(Ptr) &&
10186 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
10187 // Handle FI+Cst
10188 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
10189 FrameOffset = Ptr.getConstantOperandVal(1);
10190 }
10191
10192 if (FrameIdx != INT_MIN(-2147483647 -1)) {
10193 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
10194 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
10195 }
10196
10197 return None;
10198}
10199
10200/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
10201/// which is split (or expanded) into two not necessarily identical pieces.
10202std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
10203 // Currently all types are split in half.
10204 EVT LoVT, HiVT;
10205 if (!VT.isVector())
10206 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
10207 else
10208 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
10209
10210 return std::make_pair(LoVT, HiVT);
10211}
10212
10213/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
10214/// type, dependent on an enveloping VT that has been split into two identical
10215/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
10216std::pair<EVT, EVT>
10217SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
10218 bool *HiIsEmpty) const {
10219 EVT EltTp = VT.getVectorElementType();
10220 // Examples:
10221 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
10222 // custom VL=9 with enveloping VL=8/8 yields 8/1
10223 // custom VL=10 with enveloping VL=8/8 yields 8/2
10224 // etc.
10225 ElementCount VTNumElts = VT.getVectorElementCount();
10226 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
10227 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&(static_cast <bool> (VTNumElts.isScalable() == EnvNumElts
.isScalable() && "Mixing fixed width and scalable vectors when enveloping a type"
) ? void (0) : __assert_fail ("VTNumElts.isScalable() == EnvNumElts.isScalable() && \"Mixing fixed width and scalable vectors when enveloping a type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10228, __extension__ __PRETTY_FUNCTION__))
10228 "Mixing fixed width and scalable vectors when enveloping a type")(static_cast <bool> (VTNumElts.isScalable() == EnvNumElts
.isScalable() && "Mixing fixed width and scalable vectors when enveloping a type"
) ? void (0) : __assert_fail ("VTNumElts.isScalable() == EnvNumElts.isScalable() && \"Mixing fixed width and scalable vectors when enveloping a type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10228, __extension__ __PRETTY_FUNCTION__))
;
10229 EVT LoVT, HiVT;
10230 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
10231 LoVT = EnvVT;
10232 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
10233 *HiIsEmpty = false;
10234 } else {
10235 // Flag that hi type has zero storage size, but return split envelop type
10236 // (this would be easier if vector types with zero elements were allowed).
10237 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
10238 HiVT = EnvVT;
10239 *HiIsEmpty = true;
10240 }
10241 return std::make_pair(LoVT, HiVT);
10242}
10243
10244/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
10245/// low/high part.
10246std::pair<SDValue, SDValue>
10247SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
10248 const EVT &HiVT) {
10249 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&(static_cast <bool> (LoVT.isScalableVector() == HiVT.isScalableVector
() && LoVT.isScalableVector() == N.getValueType().isScalableVector
() && "Splitting vector with an invalid mixture of fixed and scalable "
"vector types") ? void (0) : __assert_fail ("LoVT.isScalableVector() == HiVT.isScalableVector() && LoVT.isScalableVector() == N.getValueType().isScalableVector() && \"Splitting vector with an invalid mixture of fixed and scalable \" \"vector types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10252, __extension__ __PRETTY_FUNCTION__))
10250 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&(static_cast <bool> (LoVT.isScalableVector() == HiVT.isScalableVector
() && LoVT.isScalableVector() == N.getValueType().isScalableVector
() && "Splitting vector with an invalid mixture of fixed and scalable "
"vector types") ? void (0) : __assert_fail ("LoVT.isScalableVector() == HiVT.isScalableVector() && LoVT.isScalableVector() == N.getValueType().isScalableVector() && \"Splitting vector with an invalid mixture of fixed and scalable \" \"vector types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10252, __extension__ __PRETTY_FUNCTION__))
10251 "Splitting vector with an invalid mixture of fixed and scalable "(static_cast <bool> (LoVT.isScalableVector() == HiVT.isScalableVector
() && LoVT.isScalableVector() == N.getValueType().isScalableVector
() && "Splitting vector with an invalid mixture of fixed and scalable "
"vector types") ? void (0) : __assert_fail ("LoVT.isScalableVector() == HiVT.isScalableVector() && LoVT.isScalableVector() == N.getValueType().isScalableVector() && \"Splitting vector with an invalid mixture of fixed and scalable \" \"vector types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10252, __extension__ __PRETTY_FUNCTION__))
10252 "vector types")(static_cast <bool> (LoVT.isScalableVector() == HiVT.isScalableVector
() && LoVT.isScalableVector() == N.getValueType().isScalableVector
() && "Splitting vector with an invalid mixture of fixed and scalable "
"vector types") ? void (0) : __assert_fail ("LoVT.isScalableVector() == HiVT.isScalableVector() && LoVT.isScalableVector() == N.getValueType().isScalableVector() && \"Splitting vector with an invalid mixture of fixed and scalable \" \"vector types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10252, __extension__ __PRETTY_FUNCTION__))
;
10253 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=(static_cast <bool> (LoVT.getVectorMinNumElements() + HiVT
.getVectorMinNumElements() <= N.getValueType().getVectorMinNumElements
() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= N.getValueType().getVectorMinNumElements() && \"More vector elements requested than available!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10255, __extension__ __PRETTY_FUNCTION__))
10254 N.getValueType().getVectorMinNumElements() &&(static_cast <bool> (LoVT.getVectorMinNumElements() + HiVT
.getVectorMinNumElements() <= N.getValueType().getVectorMinNumElements
() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= N.getValueType().getVectorMinNumElements() && \"More vector elements requested than available!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10255, __extension__ __PRETTY_FUNCTION__))
10255 "More vector elements requested than available!")(static_cast <bool> (LoVT.getVectorMinNumElements() + HiVT
.getVectorMinNumElements() <= N.getValueType().getVectorMinNumElements
() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= N.getValueType().getVectorMinNumElements() && \"More vector elements requested than available!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10255, __extension__ __PRETTY_FUNCTION__))
;
10256 SDValue Lo, Hi;
10257 Lo =
10258 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL));
10259 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
10260 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
10261 // IDX with the runtime scaling factor of the result vector type. For
10262 // fixed-width result vectors, that runtime scaling factor is 1.
10263 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
10264 getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
10265 return std::make_pair(Lo, Hi);
10266}
10267
10268/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
10269SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
10270 EVT VT = N.getValueType();
10271 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
10272 NextPowerOf2(VT.getVectorNumElements()));
10273 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
10274 getVectorIdxConstant(0, DL));
10275}
10276
10277void SelectionDAG::ExtractVectorElements(SDValue Op,
10278 SmallVectorImpl<SDValue> &Args,
10279 unsigned Start, unsigned Count,
10280 EVT EltVT) {
10281 EVT VT = Op.getValueType();
10282 if (Count == 0)
10283 Count = VT.getVectorNumElements();
10284 if (EltVT == EVT())
10285 EltVT = VT.getVectorElementType();
10286 SDLoc SL(Op);
10287 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
10288 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
10289 getVectorIdxConstant(i, SL)));
10290 }
10291}
10292
10293// getAddressSpace - Return the address space this GlobalAddress belongs to.
10294unsigned GlobalAddressSDNode::getAddressSpace() const {
10295 return getGlobal()->getType()->getAddressSpace();
10296}
10297
10298Type *ConstantPoolSDNode::getType() const {
10299 if (isMachineConstantPoolEntry())
10300 return Val.MachineCPVal->getType();
10301 return Val.ConstVal->getType();
10302}
10303
10304bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
10305 unsigned &SplatBitSize,
10306 bool &HasAnyUndefs,
10307 unsigned MinSplatBits,
10308 bool IsBigEndian) const {
10309 EVT VT = getValueType(0);
10310 assert(VT.isVector() && "Expected a vector type")(static_cast <bool> (VT.isVector() && "Expected a vector type"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected a vector type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10310, __extension__ __PRETTY_FUNCTION__))
;
10311 unsigned VecWidth = VT.getSizeInBits();
10312 if (MinSplatBits > VecWidth)
10313 return false;
10314
10315 // FIXME: The widths are based on this node's type, but build vectors can
10316 // truncate their operands.
10317 SplatValue = APInt(VecWidth, 0);
10318 SplatUndef = APInt(VecWidth, 0);
10319
10320 // Get the bits. Bits with undefined values (when the corresponding element
10321 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
10322 // in SplatValue. If any of the values are not constant, give up and return
10323 // false.
10324 unsigned int NumOps = getNumOperands();
10325 assert(NumOps > 0 && "isConstantSplat has 0-size build vector")(static_cast <bool> (NumOps > 0 && "isConstantSplat has 0-size build vector"
) ? void (0) : __assert_fail ("NumOps > 0 && \"isConstantSplat has 0-size build vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10325, __extension__ __PRETTY_FUNCTION__))
;
10326 unsigned EltWidth = VT.getScalarSizeInBits();
10327
10328 for (unsigned j = 0; j < NumOps; ++j) {
10329 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
10330 SDValue OpVal = getOperand(i);
10331 unsigned BitPos = j * EltWidth;
10332
10333 if (OpVal.isUndef())
10334 SplatUndef.setBits(BitPos, BitPos + EltWidth);
10335 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
10336 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
10337 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
10338 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
10339 else
10340 return false;
10341 }
10342
10343 // The build_vector is all constants or undefs. Find the smallest element
10344 // size that splats the vector.
10345 HasAnyUndefs = (SplatUndef != 0);
10346
10347 // FIXME: This does not work for vectors with elements less than 8 bits.
10348 while (VecWidth > 8) {
10349 unsigned HalfSize = VecWidth / 2;
10350 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
10351 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
10352 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
10353 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
10354
10355 // If the two halves do not match (ignoring undef bits), stop here.
10356 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
10357 MinSplatBits > HalfSize)
10358 break;
10359
10360 SplatValue = HighValue | LowValue;
10361 SplatUndef = HighUndef & LowUndef;
10362
10363 VecWidth = HalfSize;
10364 }
10365
10366 SplatBitSize = VecWidth;
10367 return true;
10368}
10369
10370SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
10371 BitVector *UndefElements) const {
10372 unsigned NumOps = getNumOperands();
10373 if (UndefElements) {
10374 UndefElements->clear();
10375 UndefElements->resize(NumOps);
10376 }
10377 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size")(static_cast <bool> (NumOps == DemandedElts.getBitWidth
() && "Unexpected vector size") ? void (0) : __assert_fail
("NumOps == DemandedElts.getBitWidth() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10377, __extension__ __PRETTY_FUNCTION__))
;
10378 if (!DemandedElts)
10379 return SDValue();
10380 SDValue Splatted;
10381 for (unsigned i = 0; i != NumOps; ++i) {
10382 if (!DemandedElts[i])
10383 continue;
10384 SDValue Op = getOperand(i);
10385 if (Op.isUndef()) {
10386 if (UndefElements)
10387 (*UndefElements)[i] = true;
10388 } else if (!Splatted) {
10389 Splatted = Op;
10390 } else if (Splatted != Op) {
10391 return SDValue();
10392 }
10393 }
10394
10395 if (!Splatted) {
10396 unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros();
10397 assert(getOperand(FirstDemandedIdx).isUndef() &&(static_cast <bool> (getOperand(FirstDemandedIdx).isUndef
() && "Can only have a splat without a constant for all undefs."
) ? void (0) : __assert_fail ("getOperand(FirstDemandedIdx).isUndef() && \"Can only have a splat without a constant for all undefs.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10398, __extension__ __PRETTY_FUNCTION__))
10398 "Can only have a splat without a constant for all undefs.")(static_cast <bool> (getOperand(FirstDemandedIdx).isUndef
() && "Can only have a splat without a constant for all undefs."
) ? void (0) : __assert_fail ("getOperand(FirstDemandedIdx).isUndef() && \"Can only have a splat without a constant for all undefs.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10398, __extension__ __PRETTY_FUNCTION__))
;
10399 return getOperand(FirstDemandedIdx);
10400 }
10401
10402 return Splatted;
10403}
10404
10405SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
10406 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
10407 return getSplatValue(DemandedElts, UndefElements);
10408}
10409
10410bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
10411 SmallVectorImpl<SDValue> &Sequence,
10412 BitVector *UndefElements) const {
10413 unsigned NumOps = getNumOperands();
10414 Sequence.clear();
10415 if (UndefElements) {
10416 UndefElements->clear();
10417 UndefElements->resize(NumOps);
10418 }
10419 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size")(static_cast <bool> (NumOps == DemandedElts.getBitWidth
() && "Unexpected vector size") ? void (0) : __assert_fail
("NumOps == DemandedElts.getBitWidth() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10419, __extension__ __PRETTY_FUNCTION__))
;
10420 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
10421 return false;
10422
10423 // Set the undefs even if we don't find a sequence (like getSplatValue).
10424 if (UndefElements)
10425 for (unsigned I = 0; I != NumOps; ++I)
10426 if (DemandedElts[I] && getOperand(I).isUndef())
10427 (*UndefElements)[I] = true;
10428
10429 // Iteratively widen the sequence length looking for repetitions.
10430 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
10431 Sequence.append(SeqLen, SDValue());
10432 for (unsigned I = 0; I != NumOps; ++I) {
10433 if (!DemandedElts[I])
10434 continue;
10435 SDValue &SeqOp = Sequence[I % SeqLen];
10436 SDValue Op = getOperand(I);
10437 if (Op.isUndef()) {
10438 if (!SeqOp)
10439 SeqOp = Op;
10440 continue;
10441 }
10442 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
10443 Sequence.clear();
10444 break;
10445 }
10446 SeqOp = Op;
10447 }
10448 if (!Sequence.empty())
10449 return true;
10450 }
10451
10452 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern")(static_cast <bool> (Sequence.empty() && "Failed to empty non-repeating sequence pattern"
) ? void (0) : __assert_fail ("Sequence.empty() && \"Failed to empty non-repeating sequence pattern\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10452, __extension__ __PRETTY_FUNCTION__))
;
10453 return false;
10454}
10455
10456bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
10457 BitVector *UndefElements) const {
10458 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
10459 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
10460}
10461
10462ConstantSDNode *
10463BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
10464 BitVector *UndefElements) const {
10465 return dyn_cast_or_null<ConstantSDNode>(
10466 getSplatValue(DemandedElts, UndefElements));
10467}
10468
10469ConstantSDNode *
10470BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
10471 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
10472}
10473
10474ConstantFPSDNode *
10475BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
10476 BitVector *UndefElements) const {
10477 return dyn_cast_or_null<ConstantFPSDNode>(
10478 getSplatValue(DemandedElts, UndefElements));
10479}
10480
10481ConstantFPSDNode *
10482BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
10483 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
10484}
10485
10486int32_t
10487BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
10488 uint32_t BitWidth) const {
10489 if (ConstantFPSDNode *CN =
10490 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
10491 bool IsExact;
10492 APSInt IntVal(BitWidth);
10493 const APFloat &APF = CN->getValueAPF();
10494 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
10495 APFloat::opOK ||
10496 !IsExact)
10497 return -1;
10498
10499 return IntVal.exactLogBase2();
10500 }
10501 return -1;
10502}
10503
10504bool BuildVectorSDNode::isConstant() const {
10505 for (const SDValue &Op : op_values()) {
10506 unsigned Opc = Op.getOpcode();
10507 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
10508 return false;
10509 }
10510 return true;
10511}
10512
10513bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
10514 // Find the first non-undef value in the shuffle mask.
10515 unsigned i, e;
10516 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
10517 /* search */;
10518
10519 // If all elements are undefined, this shuffle can be considered a splat
10520 // (although it should eventually get simplified away completely).
10521 if (i == e)
10522 return true;
10523
10524 // Make sure all remaining elements are either undef or the same as the first
10525 // non-undef value.
10526 for (int Idx = Mask[i]; i != e; ++i)
10527 if (Mask[i] >= 0 && Mask[i] != Idx)
10528 return false;
10529 return true;
10530}
10531
10532// Returns the SDNode if it is a constant integer BuildVector
10533// or constant integer.
10534SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
10535 if (isa<ConstantSDNode>(N))
10536 return N.getNode();
10537 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
10538 return N.getNode();
10539 // Treat a GlobalAddress supporting constant offset folding as a
10540 // constant integer.
10541 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
10542 if (GA->getOpcode() == ISD::GlobalAddress &&
10543 TLI->isOffsetFoldingLegal(GA))
10544 return GA;
10545 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
10546 isa<ConstantSDNode>(N.getOperand(0)))
10547 return N.getNode();
10548 return nullptr;
10549}
10550
10551// Returns the SDNode if it is a constant float BuildVector
10552// or constant float.
10553SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
10554 if (isa<ConstantFPSDNode>(N))
10555 return N.getNode();
10556
10557 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
10558 return N.getNode();
10559
10560 return nullptr;
10561}
10562
10563void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
10564 assert(!Node->OperandList && "Node already has operands")(static_cast <bool> (!Node->OperandList && "Node already has operands"
) ? void (0) : __assert_fail ("!Node->OperandList && \"Node already has operands\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10564, __extension__ __PRETTY_FUNCTION__))
;
10565 assert(SDNode::getMaxNumOperands() >= Vals.size() &&(static_cast <bool> (SDNode::getMaxNumOperands() >= Vals
.size() && "too many operands to fit into SDNode") ? void
(0) : __assert_fail ("SDNode::getMaxNumOperands() >= Vals.size() && \"too many operands to fit into SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10566, __extension__ __PRETTY_FUNCTION__))
10566 "too many operands to fit into SDNode")(static_cast <bool> (SDNode::getMaxNumOperands() >= Vals
.size() && "too many operands to fit into SDNode") ? void
(0) : __assert_fail ("SDNode::getMaxNumOperands() >= Vals.size() && \"too many operands to fit into SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10566, __extension__ __PRETTY_FUNCTION__))
;
10567 SDUse *Ops = OperandRecycler.allocate(
10568 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
10569
10570 bool IsDivergent = false;
10571 for (unsigned I = 0; I != Vals.size(); ++I) {
10572 Ops[I].setUser(Node);
10573 Ops[I].setInitial(Vals[I]);
10574 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
10575 IsDivergent |= Ops[I].getNode()->isDivergent();
10576 }
10577 Node->NumOperands = Vals.size();
10578 Node->OperandList = Ops;
10579 if (!TLI->isSDNodeAlwaysUniform(Node)) {
10580 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA);
10581 Node->SDNodeBits.IsDivergent = IsDivergent;
10582 }
10583 checkForCycles(Node);
10584}
10585
10586SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
10587 SmallVectorImpl<SDValue> &Vals) {
10588 size_t Limit = SDNode::getMaxNumOperands();
10589 while (Vals.size() > Limit) {
10590 unsigned SliceIdx = Vals.size() - Limit;
10591 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
10592 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
10593 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
10594 Vals.emplace_back(NewTF);
10595 }
10596 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
10597}
10598
10599SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
10600 EVT VT, SDNodeFlags Flags) {
10601 switch (Opcode) {
10602 default:
10603 return SDValue();
10604 case ISD::ADD:
10605 case ISD::OR:
10606 case ISD::XOR:
10607 case ISD::UMAX:
10608 return getConstant(0, DL, VT);
10609 case ISD::MUL:
10610 return getConstant(1, DL, VT);
10611 case ISD::AND:
10612 case ISD::UMIN:
10613 return getAllOnesConstant(DL, VT);
10614 case ISD::SMAX:
10615 return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);
10616 case ISD::SMIN:
10617 return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);
10618 case ISD::FADD:
10619 return getConstantFP(-0.0, DL, VT);
10620 case ISD::FMUL:
10621 return getConstantFP(1.0, DL, VT);
10622 case ISD::FMINNUM:
10623 case ISD::FMAXNUM: {
10624 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
10625 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
10626 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
10627 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
10628 APFloat::getLargest(Semantics);
10629 if (Opcode == ISD::FMAXNUM)
10630 NeutralAF.changeSign();
10631
10632 return getConstantFP(NeutralAF, DL, VT);
10633 }
10634 }
10635}
10636
10637#ifndef NDEBUG
10638static void checkForCyclesHelper(const SDNode *N,
10639 SmallPtrSetImpl<const SDNode*> &Visited,
10640 SmallPtrSetImpl<const SDNode*> &Checked,
10641 const llvm::SelectionDAG *DAG) {
10642 // If this node has already been checked, don't check it again.
10643 if (Checked.count(N))
10644 return;
10645
10646 // If a node has already been visited on this depth-first walk, reject it as
10647 // a cycle.
10648 if (!Visited.insert(N).second) {
10649 errs() << "Detected cycle in SelectionDAG\n";
10650 dbgs() << "Offending node:\n";
10651 N->dumprFull(DAG); dbgs() << "\n";
10652 abort();
10653 }
10654
10655 for (const SDValue &Op : N->op_values())
10656 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
10657
10658 Checked.insert(N);
10659 Visited.erase(N);
10660}
10661#endif
10662
10663void llvm::checkForCycles(const llvm::SDNode *N,
10664 const llvm::SelectionDAG *DAG,
10665 bool force) {
10666#ifndef NDEBUG
10667 bool check = force;
10668#ifdef EXPENSIVE_CHECKS
10669 check = true;
10670#endif // EXPENSIVE_CHECKS
10671 if (check) {
10672 assert(N && "Checking nonexistent SDNode")(static_cast <bool> (N && "Checking nonexistent SDNode"
) ? void (0) : __assert_fail ("N && \"Checking nonexistent SDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp"
, 10672, __extension__ __PRETTY_FUNCTION__))
;
10673 SmallPtrSet<const SDNode*, 32> visited;
10674 SmallPtrSet<const SDNode*, 32> checked;
10675 checkForCyclesHelper(N, visited, checked, DAG);
10676 }
10677#endif // !NDEBUG
10678}
10679
10680void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
10681 checkForCycles(DAG->getRoot().getNode(), DAG, force);
10682}

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h

1//===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SelectionDAG class, and transitively defines the
10// SDNode class and subclasses.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_SELECTIONDAG_H
15#define LLVM_CODEGEN_SELECTIONDAG_H
16
17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
23#include "llvm/ADT/SetVector.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/ilist.h"
27#include "llvm/ADT/iterator.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/CodeGen/DAGCombine.h"
30#include "llvm/CodeGen/ISDOpcodes.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/SelectionDAGNodes.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/Instructions.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/Support/Allocator.h"
39#include "llvm/Support/ArrayRecycler.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/CodeGen.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MachineValueType.h"
45#include "llvm/Support/RecyclingAllocator.h"
46#include <algorithm>
47#include <cassert>
48#include <cstdint>
49#include <functional>
50#include <map>
51#include <string>
52#include <tuple>
53#include <utility>
54#include <vector>
55
56namespace llvm {
57
58class AAResults;
59class BlockAddress;
60class BlockFrequencyInfo;
61class Constant;
62class ConstantFP;
63class ConstantInt;
64class DataLayout;
65struct fltSemantics;
66class FunctionLoweringInfo;
67class GlobalValue;
68struct KnownBits;
69class LegacyDivergenceAnalysis;
70class LLVMContext;
71class MachineBasicBlock;
72class MachineConstantPoolValue;
73class MCSymbol;
74class OptimizationRemarkEmitter;
75class ProfileSummaryInfo;
76class SDDbgValue;
77class SDDbgOperand;
78class SDDbgLabel;
79class SelectionDAG;
80class SelectionDAGTargetInfo;
81class TargetLibraryInfo;
82class TargetLowering;
83class TargetMachine;
84class TargetSubtargetInfo;
85class Value;
86
87class SDVTListNode : public FoldingSetNode {
88 friend struct FoldingSetTrait<SDVTListNode>;
89
90 /// A reference to an Interned FoldingSetNodeID for this node.
91 /// The Allocator in SelectionDAG holds the data.
92 /// SDVTList contains all types which are frequently accessed in SelectionDAG.
93 /// The size of this list is not expected to be big so it won't introduce
94 /// a memory penalty.
95 FoldingSetNodeIDRef FastID;
96 const EVT *VTs;
97 unsigned int NumVTs;
98 /// The hash value for SDVTList is fixed, so cache it to avoid
99 /// hash calculation.
100 unsigned HashValue;
101
102public:
103 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) :
104 FastID(ID), VTs(VT), NumVTs(Num) {
105 HashValue = ID.ComputeHash();
106 }
107
108 SDVTList getSDVTList() {
109 SDVTList result = {VTs, NumVTs};
110 return result;
111 }
112};
113
114/// Specialize FoldingSetTrait for SDVTListNode
115/// to avoid computing temp FoldingSetNodeID and hash value.
116template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> {
117 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) {
118 ID = X.FastID;
119 }
120
121 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID,
122 unsigned IDHash, FoldingSetNodeID &TempID) {
123 if (X.HashValue != IDHash)
124 return false;
125 return ID == X.FastID;
126 }
127
128 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) {
129 return X.HashValue;
130 }
131};
132
133template <> struct ilist_alloc_traits<SDNode> {
134 static void deleteNode(SDNode *) {
135 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!")::llvm::llvm_unreachable_internal("ilist_traits<SDNode> shouldn't see a deleteNode call!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 135)
;
136 }
137};
138
139/// Keeps track of dbg_value information through SDISel. We do
140/// not build SDNodes for these so as not to perturb the generated code;
141/// instead the info is kept off to the side in this structure. Each SDNode may
142/// have one or more associated dbg_value entries. This information is kept in
143/// DbgValMap.
144/// Byval parameters are handled separately because they don't use alloca's,
145/// which busts the normal mechanism. There is good reason for handling all
146/// parameters separately: they may not have code generated for them, they
147/// should always go at the beginning of the function regardless of other code
148/// motion, and debug info for them is potentially useful even if the parameter
149/// is unused. Right now only byval parameters are handled separately.
150class SDDbgInfo {
151 BumpPtrAllocator Alloc;
152 SmallVector<SDDbgValue*, 32> DbgValues;
153 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
154 SmallVector<SDDbgLabel*, 4> DbgLabels;
155 using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>;
156 DbgValMapType DbgValMap;
157
158public:
159 SDDbgInfo() = default;
160 SDDbgInfo(const SDDbgInfo &) = delete;
161 SDDbgInfo &operator=(const SDDbgInfo &) = delete;
162
163 void add(SDDbgValue *V, bool isParameter);
164
165 void add(SDDbgLabel *L) { DbgLabels.push_back(L); }
166
167 /// Invalidate all DbgValues attached to the node and remove
168 /// it from the Node-to-DbgValues map.
169 void erase(const SDNode *Node);
170
171 void clear() {
172 DbgValMap.clear();
173 DbgValues.clear();
174 ByvalParmDbgValues.clear();
175 DbgLabels.clear();
176 Alloc.Reset();
177 }
178
179 BumpPtrAllocator &getAlloc() { return Alloc; }
180
181 bool empty() const {
182 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty();
183 }
184
185 ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const {
186 auto I = DbgValMap.find(Node);
187 if (I != DbgValMap.end())
188 return I->second;
189 return ArrayRef<SDDbgValue*>();
190 }
191
192 using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator;
193 using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator;
194
195 DbgIterator DbgBegin() { return DbgValues.begin(); }
196 DbgIterator DbgEnd() { return DbgValues.end(); }
197 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); }
198 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); }
199 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); }
200 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); }
201};
202
203void checkForCycles(const SelectionDAG *DAG, bool force = false);
204
205/// This is used to represent a portion of an LLVM function in a low-level
206/// Data Dependence DAG representation suitable for instruction selection.
207/// This DAG is constructed as the first step of instruction selection in order
208/// to allow implementation of machine specific optimizations
209/// and code simplifications.
210///
211/// The representation used by the SelectionDAG is a target-independent
212/// representation, which has some similarities to the GCC RTL representation,
213/// but is significantly more simple, powerful, and is a graph form instead of a
214/// linear form.
215///
216class SelectionDAG {
217 const TargetMachine &TM;
218 const SelectionDAGTargetInfo *TSI = nullptr;
219 const TargetLowering *TLI = nullptr;
220 const TargetLibraryInfo *LibInfo = nullptr;
221 MachineFunction *MF;
222 Pass *SDAGISelPass = nullptr;
223 LLVMContext *Context;
224 CodeGenOpt::Level OptLevel;
225
226 LegacyDivergenceAnalysis * DA = nullptr;
227 FunctionLoweringInfo * FLI = nullptr;
228
229 /// The function-level optimization remark emitter. Used to emit remarks
230 /// whenever manipulating the DAG.
231 OptimizationRemarkEmitter *ORE;
232
233 ProfileSummaryInfo *PSI = nullptr;
234 BlockFrequencyInfo *BFI = nullptr;
235
236 /// The starting token.
237 SDNode EntryNode;
238
239 /// The root of the entire DAG.
240 SDValue Root;
241
242 /// A linked list of nodes in the current DAG.
243 ilist<SDNode> AllNodes;
244
245 /// The AllocatorType for allocating SDNodes. We use
246 /// pool allocation with recycling.
247 using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode,
248 sizeof(LargestSDNode),
249 alignof(MostAlignedSDNode)>;
250
251 /// Pool allocation for nodes.
252 NodeAllocatorType NodeAllocator;
253
254 /// This structure is used to memoize nodes, automatically performing
255 /// CSE with existing nodes when a duplicate is requested.
256 FoldingSet<SDNode> CSEMap;
257
258 /// Pool allocation for machine-opcode SDNode operands.
259 BumpPtrAllocator OperandAllocator;
260 ArrayRecycler<SDUse> OperandRecycler;
261
262 /// Pool allocation for misc. objects that are created once per SelectionDAG.
263 BumpPtrAllocator Allocator;
264
265 /// Tracks dbg_value and dbg_label information through SDISel.
266 SDDbgInfo *DbgInfo;
267
268 using CallSiteInfo = MachineFunction::CallSiteInfo;
269 using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl;
270
271 struct CallSiteDbgInfo {
272 CallSiteInfo CSInfo;
273 MDNode *HeapAllocSite = nullptr;
274 bool NoMerge = false;
275 };
276
277 DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo;
278
279 uint16_t NextPersistentId = 0;
280
281public:
282 /// Clients of various APIs that cause global effects on
283 /// the DAG can optionally implement this interface. This allows the clients
284 /// to handle the various sorts of updates that happen.
285 ///
286 /// A DAGUpdateListener automatically registers itself with DAG when it is
287 /// constructed, and removes itself when destroyed in RAII fashion.
288 struct DAGUpdateListener {
289 DAGUpdateListener *const Next;
290 SelectionDAG &DAG;
291
292 explicit DAGUpdateListener(SelectionDAG &D)
293 : Next(D.UpdateListeners), DAG(D) {
294 DAG.UpdateListeners = this;
295 }
296
297 virtual ~DAGUpdateListener() {
298 assert(DAG.UpdateListeners == this &&(static_cast <bool> (DAG.UpdateListeners == this &&
"DAGUpdateListeners must be destroyed in LIFO order") ? void
(0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 299, __extension__ __PRETTY_FUNCTION__))
299 "DAGUpdateListeners must be destroyed in LIFO order")(static_cast <bool> (DAG.UpdateListeners == this &&
"DAGUpdateListeners must be destroyed in LIFO order") ? void
(0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 299, __extension__ __PRETTY_FUNCTION__))
;
300 DAG.UpdateListeners = Next;
301 }
302
303 /// The node N that was deleted and, if E is not null, an
304 /// equivalent node E that replaced it.
305 virtual void NodeDeleted(SDNode *N, SDNode *E);
306
307 /// The node N that was updated.
308 virtual void NodeUpdated(SDNode *N);
309
310 /// The node N that was inserted.
311 virtual void NodeInserted(SDNode *N);
312 };
313
314 struct DAGNodeDeletedListener : public DAGUpdateListener {
315 std::function<void(SDNode *, SDNode *)> Callback;
316
317 DAGNodeDeletedListener(SelectionDAG &DAG,
318 std::function<void(SDNode *, SDNode *)> Callback)
319 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {}
320
321 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); }
322
323 private:
324 virtual void anchor();
325 };
326
327 /// Help to insert SDNodeFlags automatically in transforming. Use
328 /// RAII to save and resume flags in current scope.
329 class FlagInserter {
330 SelectionDAG &DAG;
331 SDNodeFlags Flags;
332 FlagInserter *LastInserter;
333
334 public:
335 FlagInserter(SelectionDAG &SDAG, SDNodeFlags Flags)
336 : DAG(SDAG), Flags(Flags),
337 LastInserter(SDAG.getFlagInserter()) {
338 SDAG.setFlagInserter(this);
339 }
340 FlagInserter(SelectionDAG &SDAG, SDNode *N)
341 : FlagInserter(SDAG, N->getFlags()) {}
342
343 FlagInserter(const FlagInserter &) = delete;
344 FlagInserter &operator=(const FlagInserter &) = delete;
345 ~FlagInserter() { DAG.setFlagInserter(LastInserter); }
346
347 SDNodeFlags getFlags() const { return Flags; }
348 };
349
350 /// When true, additional steps are taken to
351 /// ensure that getConstant() and similar functions return DAG nodes that
352 /// have legal types. This is important after type legalization since
353 /// any illegally typed nodes generated after this point will not experience
354 /// type legalization.
355 bool NewNodesMustHaveLegalTypes = false;
356
357private:
358 /// DAGUpdateListener is a friend so it can manipulate the listener stack.
359 friend struct DAGUpdateListener;
360
361 /// Linked list of registered DAGUpdateListener instances.
362 /// This stack is maintained by DAGUpdateListener RAII.
363 DAGUpdateListener *UpdateListeners = nullptr;
364
365 /// Implementation of setSubgraphColor.
366 /// Return whether we had to truncate the search.
367 bool setSubgraphColorHelper(SDNode *N, const char *Color,
368 DenseSet<SDNode *> &visited,
369 int level, bool &printed);
370
371 template <typename SDNodeT, typename... ArgTypes>
372 SDNodeT *newSDNode(ArgTypes &&... Args) {
373 return new (NodeAllocator.template Allocate<SDNodeT>())
374 SDNodeT(std::forward<ArgTypes>(Args)...);
375 }
376
377 /// Build a synthetic SDNodeT with the given args and extract its subclass
378 /// data as an integer (e.g. for use in a folding set).
379 ///
380 /// The args to this function are the same as the args to SDNodeT's
381 /// constructor, except the second arg (assumed to be a const DebugLoc&) is
382 /// omitted.
383 template <typename SDNodeT, typename... ArgTypes>
384 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder,
385 ArgTypes &&... Args) {
386 // The compiler can reduce this expression to a constant iff we pass an
387 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing
388 // on the subclass data.
389 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...)
390 .getRawSubclassData();
391 }
392
393 template <typename SDNodeTy>
394 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order,
395 SDVTList VTs, EVT MemoryVT,
396 MachineMemOperand *MMO) {
397 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO)
398 .getRawSubclassData();
399 }
400
401 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals);
402
403 void removeOperands(SDNode *Node) {
404 if (!Node->OperandList)
405 return;
406 OperandRecycler.deallocate(
407 ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
408 Node->OperandList);
409 Node->NumOperands = 0;
410 Node->OperandList = nullptr;
411 }
412 void CreateTopologicalOrder(std::vector<SDNode*>& Order);
413
414public:
415 // Maximum depth for recursive analysis such as computeKnownBits, etc.
416 static constexpr unsigned MaxRecursionDepth = 6;
417
418 explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level);
419 SelectionDAG(const SelectionDAG &) = delete;
420 SelectionDAG &operator=(const SelectionDAG &) = delete;
421 ~SelectionDAG();
422
423 /// Prepare this SelectionDAG to process code in the given MachineFunction.
424 void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
425 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
426 LegacyDivergenceAnalysis * Divergence,
427 ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin);
428
429 void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) {
430 FLI = FuncInfo;
431 }
432
433 /// Clear state and free memory necessary to make this
434 /// SelectionDAG ready to process a new block.
435 void clear();
436
437 MachineFunction &getMachineFunction() const { return *MF; }
438 const Pass *getPass() const { return SDAGISelPass; }
439
440 const DataLayout &getDataLayout() const { return MF->getDataLayout(); }
441 const TargetMachine &getTarget() const { return TM; }
442 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); }
443 const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
444 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; }
445 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; }
446 const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; }
447 LLVMContext *getContext() const { return Context; }
448 OptimizationRemarkEmitter &getORE() const { return *ORE; }
449 ProfileSummaryInfo *getPSI() const { return PSI; }
450 BlockFrequencyInfo *getBFI() const { return BFI; }
451
452 FlagInserter *getFlagInserter() { return Inserter; }
453 void setFlagInserter(FlagInserter *FI) { Inserter = FI; }
454
455 /// Just dump dot graph to a user-provided path and title.
456 /// This doesn't open the dot viewer program and
457 /// helps visualization when outside debugging session.
458 /// FileName expects absolute path. If provided
459 /// without any path separators then the file
460 /// will be created in the current directory.
461 /// Error will be emitted if the path is insane.
462#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
463 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dumpDotGraph(const Twine &FileName, const Twine &Title);
464#endif
465
466 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
467 void viewGraph(const std::string &Title);
468 void viewGraph();
469
470#ifndef NDEBUG
471 std::map<const SDNode *, std::string> NodeGraphAttrs;
472#endif
473
474 /// Clear all previously defined node graph attributes.
475 /// Intended to be used from a debugging tool (eg. gdb).
476 void clearGraphAttrs();
477
478 /// Set graph attributes for a node. (eg. "color=red".)
479 void setGraphAttrs(const SDNode *N, const char *Attrs);
480
481 /// Get graph attributes for a node. (eg. "color=red".)
482 /// Used from getNodeAttributes.
483 std::string getGraphAttrs(const SDNode *N) const;
484
485 /// Convenience for setting node color attribute.
486 void setGraphColor(const SDNode *N, const char *Color);
487
488 /// Convenience for setting subgraph color attribute.
489 void setSubgraphColor(SDNode *N, const char *Color);
490
491 using allnodes_const_iterator = ilist<SDNode>::const_iterator;
492
493 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); }
494 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); }
495
496 using allnodes_iterator = ilist<SDNode>::iterator;
497
498 allnodes_iterator allnodes_begin() { return AllNodes.begin(); }
499 allnodes_iterator allnodes_end() { return AllNodes.end(); }
500
501 ilist<SDNode>::size_type allnodes_size() const {
502 return AllNodes.size();
503 }
504
505 iterator_range<allnodes_iterator> allnodes() {
506 return make_range(allnodes_begin(), allnodes_end());
507 }
508 iterator_range<allnodes_const_iterator> allnodes() const {
509 return make_range(allnodes_begin(), allnodes_end());
510 }
511
512 /// Return the root tag of the SelectionDAG.
513 const SDValue &getRoot() const { return Root; }
514
515 /// Return the token chain corresponding to the entry of the function.
516 SDValue getEntryNode() const {
517 return SDValue(const_cast<SDNode *>(&EntryNode), 0);
518 }
519
520 /// Set the current root tag of the SelectionDAG.
521 ///
522 const SDValue &setRoot(SDValue N) {
523 assert((!N.getNode() || N.getValueType() == MVT::Other) &&(static_cast <bool> ((!N.getNode() || N.getValueType() ==
MVT::Other) && "DAG root value is not a chain!") ? void
(0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 524, __extension__ __PRETTY_FUNCTION__))
524 "DAG root value is not a chain!")(static_cast <bool> ((!N.getNode() || N.getValueType() ==
MVT::Other) && "DAG root value is not a chain!") ? void
(0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 524, __extension__ __PRETTY_FUNCTION__))
;
525 if (N.getNode())
526 checkForCycles(N.getNode(), this);
527 Root = N;
528 if (N.getNode())
529 checkForCycles(this);
530 return Root;
531 }
532
533#ifndef NDEBUG
534 void VerifyDAGDiverence();
535#endif
536
537 /// This iterates over the nodes in the SelectionDAG, folding
538 /// certain types of nodes together, or eliminating superfluous nodes. The
539 /// Level argument controls whether Combine is allowed to produce nodes and
540 /// types that are illegal on the target.
541 void Combine(CombineLevel Level, AAResults *AA,
542 CodeGenOpt::Level OptLevel);
543
544 /// This transforms the SelectionDAG into a SelectionDAG that
545 /// only uses types natively supported by the target.
546 /// Returns "true" if it made any changes.
547 ///
548 /// Note that this is an involved process that may invalidate pointers into
549 /// the graph.
550 bool LegalizeTypes();
551
552 /// This transforms the SelectionDAG into a SelectionDAG that is
553 /// compatible with the target instruction selector, as indicated by the
554 /// TargetLowering object.
555 ///
556 /// Note that this is an involved process that may invalidate pointers into
557 /// the graph.
558 void Legalize();
559
560 /// Transforms a SelectionDAG node and any operands to it into a node
561 /// that is compatible with the target instruction selector, as indicated by
562 /// the TargetLowering object.
563 ///
564 /// \returns true if \c N is a valid, legal node after calling this.
565 ///
566 /// This essentially runs a single recursive walk of the \c Legalize process
567 /// over the given node (and its operands). This can be used to incrementally
568 /// legalize the DAG. All of the nodes which are directly replaced,
569 /// potentially including N, are added to the output parameter \c
570 /// UpdatedNodes so that the delta to the DAG can be understood by the
571 /// caller.
572 ///
573 /// When this returns false, N has been legalized in a way that make the
574 /// pointer passed in no longer valid. It may have even been deleted from the
575 /// DAG, and so it shouldn't be used further. When this returns true, the
576 /// N passed in is a legal node, and can be immediately processed as such.
577 /// This may still have done some work on the DAG, and will still populate
578 /// UpdatedNodes with any new nodes replacing those originally in the DAG.
579 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes);
580
581 /// This transforms the SelectionDAG into a SelectionDAG
582 /// that only uses vector math operations supported by the target. This is
583 /// necessary as a separate step from Legalize because unrolling a vector
584 /// operation can introduce illegal types, which requires running
585 /// LegalizeTypes again.
586 ///
587 /// This returns true if it made any changes; in that case, LegalizeTypes
588 /// is called again before Legalize.
589 ///
590 /// Note that this is an involved process that may invalidate pointers into
591 /// the graph.
592 bool LegalizeVectors();
593
594 /// This method deletes all unreachable nodes in the SelectionDAG.
595 void RemoveDeadNodes();
596
597 /// Remove the specified node from the system. This node must
598 /// have no referrers.
599 void DeleteNode(SDNode *N);
600
601 /// Return an SDVTList that represents the list of values specified.
602 SDVTList getVTList(EVT VT);
603 SDVTList getVTList(EVT VT1, EVT VT2);
604 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
605 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4);
606 SDVTList getVTList(ArrayRef<EVT> VTs);
607
608 //===--------------------------------------------------------------------===//
609 // Node creation methods.
610
611 /// Create a ConstantSDNode wrapping a constant value.
612 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
613 ///
614 /// If only legal types can be produced, this does the necessary
615 /// transformations (e.g., if the vector element type is illegal).
616 /// @{
617 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
618 bool isTarget = false, bool isOpaque = false);
619 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
620 bool isTarget = false, bool isOpaque = false);
621
622 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
623 bool IsOpaque = false) {
624 return getConstant(APInt::getAllOnesValue(VT.getScalarSizeInBits()), DL,
625 VT, IsTarget, IsOpaque);
626 }
627
628 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
629 bool isTarget = false, bool isOpaque = false);
630 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL,
631 bool isTarget = false);
632 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL,
633 bool LegalTypes = true);
634 SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
635 bool isTarget = false);
636
637 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT,
638 bool isOpaque = false) {
639 return getConstant(Val, DL, VT, true, isOpaque);
640 }
641 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT,
642 bool isOpaque = false) {
643 return getConstant(Val, DL, VT, true, isOpaque);
644 }
645 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
646 bool isOpaque = false) {
647 return getConstant(Val, DL, VT, true, isOpaque);
648 }
649
650 /// Create a true or false constant of type \p VT using the target's
651 /// BooleanContent for type \p OpVT.
652 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT);
653 /// @}
654
655 /// Create a ConstantFPSDNode wrapping a constant value.
656 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
657 ///
658 /// If only legal types can be produced, this does the necessary
659 /// transformations (e.g., if the vector element type is illegal).
660 /// The forms that take a double should only be used for simple constants
661 /// that can be exactly represented in VT. No checks are made.
662 /// @{
663 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT,
664 bool isTarget = false);
665 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT,
666 bool isTarget = false);
667 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT,
668 bool isTarget = false);
669 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) {
670 return getConstantFP(Val, DL, VT, true);
671 }
672 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) {
673 return getConstantFP(Val, DL, VT, true);
674 }
675 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) {
676 return getConstantFP(Val, DL, VT, true);
677 }
678 /// @}
679
680 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
681 int64_t offset = 0, bool isTargetGA = false,
682 unsigned TargetFlags = 0);
683 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
684 int64_t offset = 0, unsigned TargetFlags = 0) {
685 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags);
686 }
687 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false);
688 SDValue getTargetFrameIndex(int FI, EVT VT) {
689 return getFrameIndex(FI, VT, true);
690 }
691 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false,
692 unsigned TargetFlags = 0);
693 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) {
694 return getJumpTable(JTI, VT, true, TargetFlags);
695 }
696 SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align = None,
697 int Offs = 0, bool isT = false,
698 unsigned TargetFlags = 0);
699 SDValue getTargetConstantPool(const Constant *C, EVT VT,
700 MaybeAlign Align = None, int Offset = 0,
701 unsigned TargetFlags = 0) {
702 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
703 }
704 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT,
705 MaybeAlign Align = None, int Offs = 0,
706 bool isT = false, unsigned TargetFlags = 0);
707 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT,
708 MaybeAlign Align = None, int Offset = 0,
709 unsigned TargetFlags = 0) {
710 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
711 }
712 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
713 unsigned TargetFlags = 0);
714 // When generating a branch to a BB, we don't in general know enough
715 // to provide debug info for the BB at that time, so keep this one around.
716 SDValue getBasicBlock(MachineBasicBlock *MBB);
717 SDValue getExternalSymbol(const char *Sym, EVT VT);
718 SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
719 unsigned TargetFlags = 0);
720 SDValue getMCSymbol(MCSymbol *Sym, EVT VT);
721
722 SDValue getValueType(EVT);
723 SDValue getRegister(unsigned Reg, EVT VT);
724 SDValue getRegisterMask(const uint32_t *RegMask);
725 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
726 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
727 MCSymbol *Label);
728 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0,
729 bool isTarget = false, unsigned TargetFlags = 0);
730 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT,
731 int64_t Offset = 0, unsigned TargetFlags = 0) {
732 return getBlockAddress(BA, VT, Offset, true, TargetFlags);
733 }
734
735 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg,
736 SDValue N) {
737 return getNode(ISD::CopyToReg, dl, MVT::Other, Chain,
738 getRegister(Reg, N.getValueType()), N);
739 }
740
741 // This version of the getCopyToReg method takes an extra operand, which
742 // indicates that there is potentially an incoming glue value (if Glue is not
743 // null) and that there should be a glue result.
744 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
745 SDValue Glue) {
746 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
747 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
748 return getNode(ISD::CopyToReg, dl, VTs,
749 makeArrayRef(Ops, Glue.getNode() ? 4 : 3));
750 }
751
752 // Similar to last getCopyToReg() except parameter Reg is a SDValue
753 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
754 SDValue Glue) {
755 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
756 SDValue Ops[] = { Chain, Reg, N, Glue };
757 return getNode(ISD::CopyToReg, dl, VTs,
758 makeArrayRef(Ops, Glue.getNode() ? 4 : 3));
759 }
760
761 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) {
762 SDVTList VTs = getVTList(VT, MVT::Other);
763 SDValue Ops[] = { Chain, getRegister(Reg, VT) };
764 return getNode(ISD::CopyFromReg, dl, VTs, Ops);
765 }
766
767 // This version of the getCopyFromReg method takes an extra operand, which
768 // indicates that there is potentially an incoming glue value (if Glue is not
769 // null) and that there should be a glue result.
770 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT,
771 SDValue Glue) {
772 SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue);
773 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
774 return getNode(ISD::CopyFromReg, dl, VTs,
775 makeArrayRef(Ops, Glue.getNode() ? 3 : 2));
776 }
777
778 SDValue getCondCode(ISD::CondCode Cond);
779
780 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT,
781 /// which must be a vector type, must match the number of mask elements
782 /// NumElts. An integer mask element equal to -1 is treated as undefined.
783 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
784 ArrayRef<int> Mask);
785
786 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
787 /// which must be a vector type, must match the number of operands in Ops.
788 /// The operands must have the same type as (or, for integers, a type wider
789 /// than) VT's element type.
790 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) {
791 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
792 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
793 }
794
795 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
796 /// which must be a vector type, must match the number of operands in Ops.
797 /// The operands must have the same type as (or, for integers, a type wider
798 /// than) VT's element type.
799 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) {
800 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
801 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
802 }
803
804 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all
805 /// elements. VT must be a vector type. Op's type must be the same as (or,
806 /// for integers, a type wider than) VT's element type.
807 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) {
808 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
809 if (Op.getOpcode() == ISD::UNDEF) {
810 assert((VT.getVectorElementType() == Op.getValueType() ||(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 814, __extension__ __PRETTY_FUNCTION__))
811 (VT.isInteger() &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 814, __extension__ __PRETTY_FUNCTION__))
812 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 814, __extension__ __PRETTY_FUNCTION__))
813 "A splatted value must have a width equal or (for integers) "(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 814, __extension__ __PRETTY_FUNCTION__))
814 "greater than the vector element type!")(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 814, __extension__ __PRETTY_FUNCTION__))
;
815 return getNode(ISD::UNDEF, SDLoc(), VT);
816 }
817
818 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op);
819 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
820 }
821
822 // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all
823 // elements.
824 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) {
825 if (Op.getOpcode() == ISD::UNDEF) {
826 assert((VT.getVectorElementType() == Op.getValueType() ||(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 830, __extension__ __PRETTY_FUNCTION__))
827 (VT.isInteger() &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 830, __extension__ __PRETTY_FUNCTION__))
828 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 830, __extension__ __PRETTY_FUNCTION__))
829 "A splatted value must have a width equal or (for integers) "(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 830, __extension__ __PRETTY_FUNCTION__))
830 "greater than the vector element type!")(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? void (0) : __assert_fail
("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 830, __extension__ __PRETTY_FUNCTION__))
;
831 return getNode(ISD::UNDEF, SDLoc(), VT);
832 }
833 return getNode(ISD::SPLAT_VECTOR, DL, VT, Op);
834 }
835
836 /// Returns a vector of type ResVT whose elements contain the linear sequence
837 /// <0, Step, Step * 2, Step * 3, ...>
838 SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal);
839
840 /// Returns a vector of type ResVT whose elements contain the linear sequence
841 /// <0, 1, 2, 3, ...>
842 SDValue getStepVector(const SDLoc &DL, EVT ResVT);
843
844 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
845 /// the shuffle node in input but with swapped operands.
846 ///
847 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
848 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);
849
850 /// Convert Op, which must be of float type, to the
851 /// float type VT, by either extending or rounding (by truncation).
852 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT);
853
854 /// Convert Op, which must be a STRICT operation of float type, to the
855 /// float type VT, by either extending or rounding (by truncation).
856 std::pair<SDValue, SDValue>
857 getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT);
858
859 /// Convert Op, which must be of integer type, to the
860 /// integer type VT, by either any-extending or truncating it.
861 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
862
863 /// Convert Op, which must be of integer type, to the
864 /// integer type VT, by either sign-extending or truncating it.
865 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
866
867 /// Convert Op, which must be of integer type, to the
868 /// integer type VT, by either zero-extending or truncating it.
869 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
870
871 /// Return the expression required to zero extend the Op
872 /// value assuming it was the smaller SrcTy value.
873 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
874
875 /// Convert Op, which must be of integer type, to the integer type VT, by
876 /// either truncating it or performing either zero or sign extension as
877 /// appropriate extension for the pointer's semantics.
878 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
879
880 /// Return the expression required to extend the Op as a pointer value
881 /// assuming it was the smaller SrcTy value. This may be either a zero extend
882 /// or a sign extend.
883 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
884
885 /// Convert Op, which must be of integer type, to the integer type VT,
886 /// by using an extension appropriate for the target's
887 /// BooleanContent for type OpVT or truncating it.
888 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);
889
890 /// Create a bitwise NOT operation as (XOR Val, -1).
891 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);
892
893 /// Create a logical NOT operation as (XOR Val, BooleanOne).
894 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);
895
896 /// Returns sum of the base pointer and offset.
897 /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default.
898 SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL,
899 const SDNodeFlags Flags = SDNodeFlags());
900 SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL,
901 const SDNodeFlags Flags = SDNodeFlags());
902
903 /// Create an add instruction with appropriate flags when used for
904 /// addressing some offset of an object. i.e. if a load is split into multiple
905 /// components, create an add nuw from the base pointer to the offset.
906 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) {
907 SDNodeFlags Flags;
908 Flags.setNoUnsignedWrap(true);
909 return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
910 }
911
912 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) {
913 // The object itself can't wrap around the address space, so it shouldn't be
914 // possible for the adds of the offsets to the split parts to overflow.
915 SDNodeFlags Flags;
916 Flags.setNoUnsignedWrap(true);
917 return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
918 }
919
920 /// Return a new CALLSEQ_START node, that starts new call frame, in which
921 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and
922 /// OutSize specifies part of the frame set up prior to the sequence.
923 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize,
924 const SDLoc &DL) {
925 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
926 SDValue Ops[] = { Chain,
927 getIntPtrConstant(InSize, DL, true),
928 getIntPtrConstant(OutSize, DL, true) };
929 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
930 }
931
932 /// Return a new CALLSEQ_END node, which always must have a
933 /// glue result (to ensure it's not CSE'd).
934 /// CALLSEQ_END does not have a useful SDLoc.
935 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
936 SDValue InGlue, const SDLoc &DL) {
937 SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue);
938 SmallVector<SDValue, 4> Ops;
939 Ops.push_back(Chain);
940 Ops.push_back(Op1);
941 Ops.push_back(Op2);
942 if (InGlue.getNode())
943 Ops.push_back(InGlue);
944 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
945 }
946
947 /// Return true if the result of this operation is always undefined.
948 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops);
949
950 /// Return an UNDEF node. UNDEF does not have a useful SDLoc.
951 SDValue getUNDEF(EVT VT) {
952 return getNode(ISD::UNDEF, SDLoc(), VT);
953 }
954
955 /// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
956 SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
957 assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() &&(static_cast <bool> (MulImm.getMinSignedBits() <= VT
.getSizeInBits() && "Immediate does not fit VT") ? void
(0) : __assert_fail ("MulImm.getMinSignedBits() <= VT.getSizeInBits() && \"Immediate does not fit VT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 958, __extension__ __PRETTY_FUNCTION__))
958 "Immediate does not fit VT")(static_cast <bool> (MulImm.getMinSignedBits() <= VT
.getSizeInBits() && "Immediate does not fit VT") ? void
(0) : __assert_fail ("MulImm.getMinSignedBits() <= VT.getSizeInBits() && \"Immediate does not fit VT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 958, __extension__ __PRETTY_FUNCTION__))
;
959 return getNode(ISD::VSCALE, DL, VT,
960 getConstant(MulImm.sextOrTrunc(VT.getSizeInBits()), DL, VT));
961 }
962
963 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
964 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
965 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
966 }
967
968 /// Gets or creates the specified node.
969 ///
970 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
971 ArrayRef<SDUse> Ops);
972 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
973 ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
974 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
975 ArrayRef<SDValue> Ops);
976 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
977 ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
978
979 // Use flags from current flag inserter.
980 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
981 ArrayRef<SDValue> Ops);
982 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
983 ArrayRef<SDValue> Ops);
984 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand);
985 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
986 SDValue N2);
987 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
988 SDValue N2, SDValue N3);
989
990 // Specialize based on number of operands.
991 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT);
992 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
993 const SDNodeFlags Flags);
994 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
995 SDValue N2, const SDNodeFlags Flags);
996 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
997 SDValue N2, SDValue N3, const SDNodeFlags Flags);
998 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
999 SDValue N2, SDValue N3, SDValue N4);
1000 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1001 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
1002
1003 // Specialize again based on number of operands for nodes with a VTList
1004 // rather than a single VT.
1005 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList);
1006 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
1007 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1008 SDValue N2);
1009 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1010 SDValue N2, SDValue N3);
1011 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1012 SDValue N2, SDValue N3, SDValue N4);
1013 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
1014 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
1015
1016 /// Compute a TokenFactor to force all the incoming stack arguments to be
1017 /// loaded from the stack. This is used in tail call lowering to protect
1018 /// stack arguments from being clobbered.
1019 SDValue getStackArgumentTokenFactor(SDValue Chain);
1020
1021 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
1022 SDValue Size, Align Alignment, bool isVol,
1023 bool AlwaysInline, bool isTailCall,
1024 MachinePointerInfo DstPtrInfo,
1025 MachinePointerInfo SrcPtrInfo,
1026 const AAMDNodes &AAInfo = AAMDNodes());
1027
1028 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
1029 SDValue Size, Align Alignment, bool isVol, bool isTailCall,
1030 MachinePointerInfo DstPtrInfo,
1031 MachinePointerInfo SrcPtrInfo,
1032 const AAMDNodes &AAInfo = AAMDNodes());
1033
1034 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
1035 SDValue Size, Align Alignment, bool isVol, bool isTailCall,
1036 MachinePointerInfo DstPtrInfo,
1037 const AAMDNodes &AAInfo = AAMDNodes());
1038
1039 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
1040 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
1041 SDValue Size, Type *SizeTy, unsigned ElemSz,
1042 bool isTailCall, MachinePointerInfo DstPtrInfo,
1043 MachinePointerInfo SrcPtrInfo);
1044
1045 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
1046 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
1047 SDValue Size, Type *SizeTy, unsigned ElemSz,
1048 bool isTailCall, MachinePointerInfo DstPtrInfo,
1049 MachinePointerInfo SrcPtrInfo);
1050
1051 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
1052 unsigned DstAlign, SDValue Value, SDValue Size,
1053 Type *SizeTy, unsigned ElemSz, bool isTailCall,
1054 MachinePointerInfo DstPtrInfo);
1055
1056 /// Helper function to make it easier to build SetCC's if you just have an
1057 /// ISD::CondCode instead of an SDValue.
1058 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
1059 ISD::CondCode Cond, SDValue Chain = SDValue(),
1060 bool IsSignaling = false) {
1061 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&(static_cast <bool> (LHS.getValueType().isVector() == RHS
.getValueType().isVector() && "Cannot compare scalars to vectors"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
1062 "Cannot compare scalars to vectors")(static_cast <bool> (LHS.getValueType().isVector() == RHS
.getValueType().isVector() && "Cannot compare scalars to vectors"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
;
1063 assert(LHS.getValueType().isVector() == VT.isVector() &&(static_cast <bool> (LHS.getValueType().isVector() == VT
.isVector() && "Cannot compare scalars to vectors") ?
void (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1064, __extension__ __PRETTY_FUNCTION__))
1064 "Cannot compare scalars to vectors")(static_cast <bool> (LHS.getValueType().isVector() == VT
.isVector() && "Cannot compare scalars to vectors") ?
void (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1064, __extension__ __PRETTY_FUNCTION__))
;
1065 assert(Cond != ISD::SETCC_INVALID &&(static_cast <bool> (Cond != ISD::SETCC_INVALID &&
"Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail
("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1066, __extension__ __PRETTY_FUNCTION__))
1066 "Cannot create a setCC of an invalid node.")(static_cast <bool> (Cond != ISD::SETCC_INVALID &&
"Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail
("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1066, __extension__ __PRETTY_FUNCTION__))
;
1067 if (Chain)
1068 return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL,
1069 {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)});
1070 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
1071 }
1072
1073 /// Helper function to make it easier to build Select's if you just have
1074 /// operands and don't want to check for vector.
1075 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
1076 SDValue RHS) {
1077 assert(LHS.getValueType() == RHS.getValueType() &&(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Cannot use select on differing types") ? void (
0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1078, __extension__ __PRETTY_FUNCTION__))
1078 "Cannot use select on differing types")(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Cannot use select on differing types") ? void (
0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1078, __extension__ __PRETTY_FUNCTION__))
;
1079 assert(VT.isVector() == LHS.getValueType().isVector() &&(static_cast <bool> (VT.isVector() == LHS.getValueType(
).isVector() && "Cannot mix vectors and scalars") ? void
(0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1080, __extension__ __PRETTY_FUNCTION__))
1080 "Cannot mix vectors and scalars")(static_cast <bool> (VT.isVector() == LHS.getValueType(
).isVector() && "Cannot mix vectors and scalars") ? void
(0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1080, __extension__ __PRETTY_FUNCTION__))
;
1081 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
1082 return getNode(Opcode, DL, VT, Cond, LHS, RHS);
1083 }
1084
1085 /// Helper function to make it easier to build SelectCC's if you just have an
1086 /// ISD::CondCode instead of an SDValue.
1087 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
1088 SDValue False, ISD::CondCode Cond) {
1089 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True,
1090 False, getCondCode(Cond));
1091 }
1092
1093 /// Try to simplify a select/vselect into 1 of its operands or a constant.
1094 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
1095
1096 /// Try to simplify a shift into 1 of its operands or a constant.
1097 SDValue simplifyShift(SDValue X, SDValue Y);
1098
1099 /// Try to simplify a floating-point binary operation into 1 of its operands
1100 /// or a constant.
1101 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
1102 SDNodeFlags Flags);
1103
1104 /// VAArg produces a result and token chain, and takes a pointer
1105 /// and a source value as input.
1106 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1107 SDValue SV, unsigned Align);
1108
1109 /// Gets a node for an atomic cmpxchg op. There are two
1110 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a
1111 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
1112 /// a success flag (initially i1), and a chain.
1113 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1114 SDVTList VTs, SDValue Chain, SDValue Ptr,
1115 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
1116
1117 /// Gets a node for an atomic op, produces result (if relevant)
1118 /// and chain and takes 2 operands.
1119 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
1120 SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
1121
1122 /// Gets a node for an atomic op, produces result and chain and
1123 /// takes 1 operand.
1124 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
1125 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);
1126
1127 /// Gets a node for an atomic op, produces result and chain and takes N
1128 /// operands.
1129 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1130 SDVTList VTList, ArrayRef<SDValue> Ops,
1131 MachineMemOperand *MMO);
1132
1133 /// Creates a MemIntrinsicNode that may produce a
1134 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
1135 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
1136 /// less than FIRST_TARGET_MEMORY_OPCODE.
1137 SDValue getMemIntrinsicNode(
1138 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
1139 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
1140 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
1141 MachineMemOperand::MOStore,
1142 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
1143
1144 inline SDValue getMemIntrinsicNode(
1145 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
1146 EVT MemVT, MachinePointerInfo PtrInfo, MaybeAlign Alignment = None,
1147 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
1148 MachineMemOperand::MOStore,
1149 uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
1150 // Ensure that codegen never sees alignment 0
1151 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
1152 Alignment.getValueOr(getEVTAlign(MemVT)), Flags,
1153 Size, AAInfo);
1154 }
1155
1156 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1157 ArrayRef<SDValue> Ops, EVT MemVT,
1158 MachineMemOperand *MMO);
1159
1160 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends
1161 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between
1162 /// offsets `Offset` and `Offset + Size`.
1163 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
1164 int FrameIndex, int64_t Size, int64_t Offset = -1);
1165
1166 /// Creates a PseudoProbeSDNode with function GUID `Guid` and
1167 /// the index of the block `Index` it is probing, as well as the attributes
1168 /// `attr` of the probe.
1169 SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid,
1170 uint64_t Index, uint32_t Attr);
1171
1172 /// Create a MERGE_VALUES node from the given operands.
1173 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);
1174
1175 /// Loads are not normal binary operators: their result type is not
1176 /// determined by their operands, and they produce a value AND a token chain.
1177 ///
1178 /// This function will set the MOLoad flag on MMOFlags, but you can set it if
1179 /// you want. The MOStore flag must not be set.
1180 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1181 MachinePointerInfo PtrInfo,
1182 MaybeAlign Alignment = MaybeAlign(),
1183 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1184 const AAMDNodes &AAInfo = AAMDNodes(),
1185 const MDNode *Ranges = nullptr);
1186 /// FIXME: Remove once transition to Align is over.
1187 inline SDValue
1188 getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1189 MachinePointerInfo PtrInfo, unsigned Alignment,
1190 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1191 const AAMDNodes &AAInfo = AAMDNodes(),
1192 const MDNode *Ranges = nullptr) {
1193 return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags,
1194 AAInfo, Ranges);
1195 }
1196 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1197 MachineMemOperand *MMO);
1198 SDValue
1199 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
1200 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
1201 MaybeAlign Alignment = MaybeAlign(),
1202 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1203 const AAMDNodes &AAInfo = AAMDNodes());
1204 /// FIXME: Remove once transition to Align is over.
1205 inline SDValue
1206 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
1207 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
1208 unsigned Alignment,
1209 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1210 const AAMDNodes &AAInfo = AAMDNodes()) {
1211 return getExtLoad(ExtType, dl, VT, Chain, Ptr, PtrInfo, MemVT,
1212 MaybeAlign(Alignment), MMOFlags, AAInfo);
1213 }
1214 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1215 SDValue Chain, SDValue Ptr, EVT MemVT,
1216 MachineMemOperand *MMO);
1217 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1218 SDValue Offset, ISD::MemIndexedMode AM);
1219 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1220 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1221 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
1222 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1223 const AAMDNodes &AAInfo = AAMDNodes(),
1224 const MDNode *Ranges = nullptr);
1225 inline SDValue getLoad(
1226 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
1227 SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
1228 EVT MemVT, MaybeAlign Alignment = MaybeAlign(),
1229 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1230 const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) {
1231 // Ensures that codegen never sees a None Alignment.
1232 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
3
Calling 'SelectionDAG::getLoad'
1233 Alignment.getValueOr(getEVTAlign(MemVT)), MMOFlags, AAInfo,
1234 Ranges);
1235 }
1236 /// FIXME: Remove once transition to Align is over.
1237 inline SDValue
1238 getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1239 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1240 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment,
1241 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1242 const AAMDNodes &AAInfo = AAMDNodes(),
1243 const MDNode *Ranges = nullptr) {
1244 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
1245 MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges);
1246 }
1247 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1248 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1249 EVT MemVT, MachineMemOperand *MMO);
1250
1251 /// Helper function to build ISD::STORE nodes.
1252 ///
1253 /// This function will set the MOStore flag on MMOFlags, but you can set it if
1254 /// you want. The MOLoad and MOInvariant flags must not be set.
1255
1256 SDValue
1257 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1258 MachinePointerInfo PtrInfo, Align Alignment,
1259 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1260 const AAMDNodes &AAInfo = AAMDNodes());
1261 inline SDValue
1262 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1263 MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
1264 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1265 const AAMDNodes &AAInfo = AAMDNodes()) {
1266 return getStore(Chain, dl, Val, Ptr, PtrInfo,
1267 Alignment.getValueOr(getEVTAlign(Val.getValueType())),
1268 MMOFlags, AAInfo);
1269 }
1270 /// FIXME: Remove once transition to Align is over.
1271 inline SDValue
1272 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1273 MachinePointerInfo PtrInfo, unsigned Alignment,
1274 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1275 const AAMDNodes &AAInfo = AAMDNodes()) {
1276 return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment),
1277 MMOFlags, AAInfo);
1278 }
1279 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1280 MachineMemOperand *MMO);
1281 SDValue
1282 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1283 MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
1284 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1285 const AAMDNodes &AAInfo = AAMDNodes());
1286 inline SDValue
1287 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1288 MachinePointerInfo PtrInfo, EVT SVT,
1289 MaybeAlign Alignment = MaybeAlign(),
1290 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1291 const AAMDNodes &AAInfo = AAMDNodes()) {
1292 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
1293 Alignment.getValueOr(getEVTAlign(SVT)), MMOFlags,
1294 AAInfo);
1295 }
1296 /// FIXME: Remove once transition to Align is over.
1297 inline SDValue
1298 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1299 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment,
1300 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1301 const AAMDNodes &AAInfo = AAMDNodes()) {
1302 return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
1303 MaybeAlign(Alignment), MMOFlags, AAInfo);
1304 }
1305 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1306 SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
1307 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1308 SDValue Offset, ISD::MemIndexedMode AM);
1309
1310 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base,
1311 SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT,
1312 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
1313 ISD::LoadExtType, bool IsExpanding = false);
1314 SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1315 SDValue Offset, ISD::MemIndexedMode AM);
1316 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1317 SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT,
1318 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
1319 bool IsTruncating = false, bool IsCompressing = false);
1320 SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
1321 SDValue Base, SDValue Offset,
1322 ISD::MemIndexedMode AM);
1323 SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
1324 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1325 ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy);
1326 SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
1327 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1328 ISD::MemIndexType IndexType,
1329 bool IsTruncating = false);
1330
1331 /// Construct a node to track a Value* through the backend.
1332 SDValue getSrcValue(const Value *v);
1333
1334 /// Return an MDNodeSDNode which holds an MDNode.
1335 SDValue getMDNode(const MDNode *MD);
1336
1337 /// Return a bitcast using the SDLoc of the value operand, and casting to the
1338 /// provided type. Use getNode to set a custom SDLoc.
1339 SDValue getBitcast(EVT VT, SDValue V);
1340
1341 /// Return an AddrSpaceCastSDNode.
1342 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
1343 unsigned DestAS);
1344
1345 /// Return a freeze using the SDLoc of the value operand.
1346 SDValue getFreeze(SDValue V);
1347
1348 /// Return an AssertAlignSDNode.
1349 SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A);
1350
1351 /// Return the specified value casted to
1352 /// the target's desired shift amount type.
1353 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
1354
1355 /// Expand the specified \c ISD::VAARG node as the Legalize pass would.
1356 SDValue expandVAArg(SDNode *Node);
1357
1358 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would.
1359 SDValue expandVACopy(SDNode *Node);
1360
1361 /// Returs an GlobalAddress of the function from the current module with
1362 /// name matching the given ExternalSymbol. Additionally can provide the
1363 /// matched function.
1364 /// Panics the function doesn't exists.
1365 SDValue getSymbolFunctionGlobalAddress(SDValue Op,
1366 Function **TargetFunction = nullptr);
1367
1368 /// *Mutate* the specified node in-place to have the
1369 /// specified operands. If the resultant node already exists in the DAG,
1370 /// this does not modify the specified node, instead it returns the node that
1371 /// already exists. If the resultant node does not exist in the DAG, the
1372 /// input node is returned. As a degenerate case, if you specify the same
1373 /// input operands as the node already has, the input node is returned.
1374 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op);
1375 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2);
1376 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1377 SDValue Op3);
1378 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1379 SDValue Op3, SDValue Op4);
1380 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1381 SDValue Op3, SDValue Op4, SDValue Op5);
1382 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
1383
1384 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k
1385 /// values or more, move values into new TokenFactors in 64k-1 blocks, until
1386 /// the final TokenFactor has less than 64k operands.
1387 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);
1388
1389 /// *Mutate* the specified machine node's memory references to the provided
1390 /// list.
1391 void setNodeMemRefs(MachineSDNode *N,
1392 ArrayRef<MachineMemOperand *> NewMemRefs);
1393
1394 // Calculate divergence of node \p N based on its operands.
1395 bool calculateDivergence(SDNode *N);
1396
1397 // Propagates the change in divergence to users
1398 void updateDivergence(SDNode * N);
1399
1400 /// These are used for target selectors to *mutate* the
1401 /// specified node to have the specified return type, Target opcode, and
1402 /// operands. Note that target opcodes are stored as
1403 /// ~TargetOpcode in the node opcode field. The resultant node is returned.
1404 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT);
1405 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1);
1406 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1407 SDValue Op1, SDValue Op2);
1408 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1409 SDValue Op1, SDValue Op2, SDValue Op3);
1410 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1411 ArrayRef<SDValue> Ops);
1412 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2);
1413 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1414 EVT VT2, ArrayRef<SDValue> Ops);
1415 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1416 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1417 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1418 EVT VT2, SDValue Op1, SDValue Op2);
1419 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs,
1420 ArrayRef<SDValue> Ops);
1421
1422 /// This *mutates* the specified node to have the specified
1423 /// return type, opcode, and operands.
1424 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
1425 ArrayRef<SDValue> Ops);
1426
1427 /// Mutate the specified strict FP node to its non-strict equivalent,
1428 /// unlinking the node from its chain and dropping the metadata arguments.
1429 /// The node must be a strict FP node.
1430 SDNode *mutateStrictFPToFP(SDNode *Node);
1431
1432 /// These are used for target selectors to create a new node
1433 /// with specified return type(s), MachineInstr opcode, and operands.
1434 ///
1435 /// Note that getMachineNode returns the resultant node. If there is already
1436 /// a node of the specified opcode and operands, it returns that node instead
1437 /// of the current one.
1438 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT);
1439 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1440 SDValue Op1);
1441 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1442 SDValue Op1, SDValue Op2);
1443 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1444 SDValue Op1, SDValue Op2, SDValue Op3);
1445 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1446 ArrayRef<SDValue> Ops);
1447 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1448 EVT VT2, SDValue Op1, SDValue Op2);
1449 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1450 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1451 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1452 EVT VT2, ArrayRef<SDValue> Ops);
1453 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1454 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
1455 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1456 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
1457 SDValue Op3);
1458 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1459 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1460 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl,
1461 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops);
1462 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs,
1463 ArrayRef<SDValue> Ops);
1464
1465 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
1466 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1467 SDValue Operand);
1468
1469 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
1470 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1471 SDValue Operand, SDValue Subreg);
1472
1473 /// Get the specified node if it's already available, or else return NULL.
1474 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList,
1475 ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
1476 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList,
1477 ArrayRef<SDValue> Ops);
1478
1479 /// Check if a node exists without modifying its flags.
1480 bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops);
1481
1482 /// Creates a SDDbgValue node.
1483 SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N,
1484 unsigned R, bool IsIndirect, const DebugLoc &DL,
1485 unsigned O);
1486
1487 /// Creates a constant SDDbgValue node.
1488 SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr,
1489 const Value *C, const DebugLoc &DL,
1490 unsigned O);
1491
1492 /// Creates a FrameIndex SDDbgValue node.
1493 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr,
1494 unsigned FI, bool IsIndirect,
1495 const DebugLoc &DL, unsigned O);
1496
1497 /// Creates a FrameIndex SDDbgValue node.
1498 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr,
1499 unsigned FI,
1500 ArrayRef<SDNode *> Dependencies,
1501 bool IsIndirect, const DebugLoc &DL,
1502 unsigned O);
1503
1504 /// Creates a VReg SDDbgValue node.
1505 SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
1506 unsigned VReg, bool IsIndirect,
1507 const DebugLoc &DL, unsigned O);
1508
1509 /// Creates a SDDbgValue node from a list of locations.
1510 SDDbgValue *getDbgValueList(DIVariable *Var, DIExpression *Expr,
1511 ArrayRef<SDDbgOperand> Locs,
1512 ArrayRef<SDNode *> Dependencies, bool IsIndirect,
1513 const DebugLoc &DL, unsigned O, bool IsVariadic);
1514
1515 /// Creates a SDDbgLabel node.
1516 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O);
1517
1518 /// Transfer debug values from one node to another, while optionally
1519 /// generating fragment expressions for split-up values. If \p InvalidateDbg
1520 /// is set, debug values are invalidated after they are transferred.
1521 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
1522 unsigned SizeInBits = 0, bool InvalidateDbg = true);
1523
1524 /// Remove the specified node from the system. If any of its
1525 /// operands then becomes dead, remove them as well. Inform UpdateListener
1526 /// for each node deleted.
1527 void RemoveDeadNode(SDNode *N);
1528
1529 /// This method deletes the unreachable nodes in the
1530 /// given list, and any nodes that become unreachable as a result.
1531 void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes);
1532
1533 /// Modify anything using 'From' to use 'To' instead.
1534 /// This can cause recursive merging of nodes in the DAG. Use the first
1535 /// version if 'From' is known to have a single result, use the second
1536 /// if you have two nodes with identical results (or if 'To' has a superset
1537 /// of the results of 'From'), use the third otherwise.
1538 ///
1539 /// These methods all take an optional UpdateListener, which (if not null) is
1540 /// informed about nodes that are deleted and modified due to recursive
1541 /// changes in the dag.
1542 ///
1543 /// These functions only replace all existing uses. It's possible that as
1544 /// these replacements are being performed, CSE may cause the From node
1545 /// to be given new uses. These new uses of From are left in place, and
1546 /// not automatically transferred to To.
1547 ///
1548 void ReplaceAllUsesWith(SDValue From, SDValue To);
1549 void ReplaceAllUsesWith(SDNode *From, SDNode *To);
1550 void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
1551
1552 /// Replace any uses of From with To, leaving
1553 /// uses of other values produced by From.getNode() alone.
1554 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
1555
1556 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once.
1557 /// This correctly handles the case where
1558 /// there is an overlap between the From values and the To values.
1559 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
1560 unsigned Num);
1561
1562 /// If an existing load has uses of its chain, create a token factor node with
1563 /// that chain and the new memory node's chain and update users of the old
1564 /// chain to the token factor. This ensures that the new memory node will have
1565 /// the same relative memory dependency position as the old load. Returns the
1566 /// new merged load chain.
1567 SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain);
1568
1569 /// If an existing load has uses of its chain, create a token factor node with
1570 /// that chain and the new memory node's chain and update users of the old
1571 /// chain to the token factor. This ensures that the new memory node will have
1572 /// the same relative memory dependency position as the old load. Returns the
1573 /// new merged load chain.
1574 SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp);
1575
1576 /// Topological-sort the AllNodes list and a
1577 /// assign a unique node id for each node in the DAG based on their
1578 /// topological order. Returns the number of nodes.
1579 unsigned AssignTopologicalOrder();
1580
1581 /// Move node N in the AllNodes list to be immediately
1582 /// before the given iterator Position. This may be used to update the
1583 /// topological ordering when the list of nodes is modified.
1584 void RepositionNode(allnodes_iterator Position, SDNode *N) {
1585 AllNodes.insert(Position, AllNodes.remove(N));
1586 }
1587
1588 /// Returns an APFloat semantics tag appropriate for the given type. If VT is
1589 /// a vector type, the element semantics are returned.
1590 static const fltSemantics &EVTToAPFloatSemantics(EVT VT) {
1591 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
1592 default: llvm_unreachable("Unknown FP format")::llvm::llvm_unreachable_internal("Unknown FP format", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1592)
;
1593 case MVT::f16: return APFloat::IEEEhalf();
1594 case MVT::bf16: return APFloat::BFloat();
1595 case MVT::f32: return APFloat::IEEEsingle();
1596 case MVT::f64: return APFloat::IEEEdouble();
1597 case MVT::f80: return APFloat::x87DoubleExtended();
1598 case MVT::f128: return APFloat::IEEEquad();
1599 case MVT::ppcf128: return APFloat::PPCDoubleDouble();
1600 }
1601 }
1602
1603 /// Add a dbg_value SDNode. If SD is non-null that means the
1604 /// value is produced by SD.
1605 void AddDbgValue(SDDbgValue *DB, bool isParameter);
1606
1607 /// Add a dbg_label SDNode.
1608 void AddDbgLabel(SDDbgLabel *DB);
1609
1610 /// Get the debug values which reference the given SDNode.
1611 ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const {
1612 return DbgInfo->getSDDbgValues(SD);
1613 }
1614
1615public:
1616 /// Return true if there are any SDDbgValue nodes associated
1617 /// with this SelectionDAG.
1618 bool hasDebugValues() const { return !DbgInfo->empty(); }
1619
1620 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); }
1621 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); }
1622
1623 SDDbgInfo::DbgIterator ByvalParmDbgBegin() const {
1624 return DbgInfo->ByvalParmDbgBegin();
1625 }
1626 SDDbgInfo::DbgIterator ByvalParmDbgEnd() const {
1627 return DbgInfo->ByvalParmDbgEnd();
1628 }
1629
1630 SDDbgInfo::DbgLabelIterator DbgLabelBegin() const {
1631 return DbgInfo->DbgLabelBegin();
1632 }
1633 SDDbgInfo::DbgLabelIterator DbgLabelEnd() const {
1634 return DbgInfo->DbgLabelEnd();
1635 }
1636
1637 /// To be invoked on an SDNode that is slated to be erased. This
1638 /// function mirrors \c llvm::salvageDebugInfo.
1639 void salvageDebugInfo(SDNode &N);
1640
1641 void dump() const;
1642
1643 /// In most cases this function returns the ABI alignment for a given type,
1644 /// except for illegal vector types where the alignment exceeds that of the
1645 /// stack. In such cases we attempt to break the vector down to a legal type
1646 /// and return the ABI alignment for that instead.
1647 Align getReducedAlign(EVT VT, bool UseABI);
1648
1649 /// Create a stack temporary based on the size in bytes and the alignment
1650 SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment);
1651
1652 /// Create a stack temporary, suitable for holding the specified value type.
1653 /// If minAlign is specified, the slot size will have at least that alignment.
1654 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);
1655
1656 /// Create a stack temporary suitable for holding either of the specified
1657 /// value types.
1658 SDValue CreateStackTemporary(EVT VT1, EVT VT2);
1659
1660 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
1661 const GlobalAddressSDNode *GA,
1662 const SDNode *N2);
1663
1664 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1665 ArrayRef<SDValue> Ops);
1666
1667 SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1668 ArrayRef<SDValue> Ops,
1669 const SDNodeFlags Flags = SDNodeFlags());
1670
1671 /// Fold floating-point operations with 2 operands when both operands are
1672 /// constants and/or undefined.
1673 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
1674 SDValue N1, SDValue N2);
1675
1676 /// Constant fold a setcc to true or false.
1677 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
1678 const SDLoc &dl);
1679
1680 /// See if the specified operand can be simplified with the knowledge that
1681 /// only the bits specified by DemandedBits are used. If so, return the
1682 /// simpler operand, otherwise return a null SDValue.
1683 ///
1684 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
1685 /// simplify nodes with multiple uses more aggressively.)
1686 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits);
1687
1688 /// See if the specified operand can be simplified with the knowledge that
1689 /// only the bits specified by DemandedBits are used in the elements specified
1690 /// by DemandedElts. If so, return the simpler operand, otherwise return a
1691 /// null SDValue.
1692 ///
1693 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
1694 /// simplify nodes with multiple uses more aggressively.)
1695 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits,
1696 const APInt &DemandedElts);
1697
1698 /// Return true if the sign bit of Op is known to be zero.
1699 /// We use this predicate to simplify operations downstream.
1700 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;
1701
1702 /// Return true if 'Op & Mask' is known to be zero. We
1703 /// use this predicate to simplify operations downstream. Op and Mask are
1704 /// known to be the same type.
1705 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1706 unsigned Depth = 0) const;
1707
1708 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We
1709 /// use this predicate to simplify operations downstream. Op and Mask are
1710 /// known to be the same type.
1711 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1712 const APInt &DemandedElts, unsigned Depth = 0) const;
1713
1714 /// Return true if '(Op & Mask) == Mask'.
1715 /// Op and Mask are known to be the same type.
1716 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
1717 unsigned Depth = 0) const;
1718
1719 /// Determine which bits of Op are known to be either zero or one and return
1720 /// them in Known. For vectors, the known bits are those that are shared by
1721 /// every vector element.
1722 /// Targets can implement the computeKnownBitsForTargetNode method in the
1723 /// TargetLowering class to allow target nodes to be understood.
1724 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const;
1725
1726 /// Determine which bits of Op are known to be either zero or one and return
1727 /// them in Known. The DemandedElts argument allows us to only collect the
1728 /// known bits that are shared by the requested vector elements.
1729 /// Targets can implement the computeKnownBitsForTargetNode method in the
1730 /// TargetLowering class to allow target nodes to be understood.
1731 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
1732 unsigned Depth = 0) const;
1733
1734 /// Used to represent the possible overflow behavior of an operation.
1735 /// Never: the operation cannot overflow.
1736 /// Always: the operation will always overflow.
1737 /// Sometime: the operation may or may not overflow.
1738 enum OverflowKind {
1739 OFK_Never,
1740 OFK_Sometime,
1741 OFK_Always,
1742 };
1743
1744 /// Determine if the result of the addition of 2 node can overflow.
1745 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const;
1746
1747 /// Test if the given value is known to have exactly one bit set. This differs
1748 /// from computeKnownBits in that it doesn't necessarily determine which bit
1749 /// is set.
1750 bool isKnownToBeAPowerOfTwo(SDValue Val) const;
1751
1752 /// Return the number of times the sign bit of the register is replicated into
1753 /// the other bits. We know that at least 1 bit is always equal to the sign
1754 /// bit (itself), but other cases can give us information. For example,
1755 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1756 /// to each other, so we return 3. Targets can implement the
1757 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow
1758 /// target nodes to be understood.
1759 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;
1760
1761 /// Return the number of times the sign bit of the register is replicated into
1762 /// the other bits. We know that at least 1 bit is always equal to the sign
1763 /// bit (itself), but other cases can give us information. For example,
1764 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1765 /// to each other, so we return 3. The DemandedElts argument allows
1766 /// us to only collect the minimum sign bits of the requested vector elements.
1767 /// Targets can implement the ComputeNumSignBitsForTarget method in the
1768 /// TargetLowering class to allow target nodes to be understood.
1769 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
1770 unsigned Depth = 0) const;
1771
1772 /// Return true if this function can prove that \p Op is never poison
1773 /// and, if \p PoisonOnly is false, does not have undef bits.
1774 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly = false,
1775 unsigned Depth = 0) const;
1776
1777 /// Return true if this function can prove that \p Op is never poison
1778 /// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts
1779 /// argument limits the check to the requested vector elements.
1780 bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, const APInt &DemandedElts,
1781 bool PoisonOnly = false,
1782 unsigned Depth = 0) const;
1783
1784 /// Return true if this function can prove that \p Op is never poison.
1785 bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth = 0) const {
1786 return isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ true, Depth);
1787 }
1788
1789 /// Return true if this function can prove that \p Op is never poison. The
1790 /// DemandedElts argument limits the check to the requested vector elements.
1791 bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts,
1792 unsigned Depth = 0) const {
1793 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts,
1794 /*PoisonOnly*/ true, Depth);
1795 }
1796
1797 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode
1798 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that
1799 /// is guaranteed to have the same semantics as an ADD. This handles the
1800 /// equivalence:
1801 /// X|Cst == X+Cst iff X&Cst = 0.
1802 bool isBaseWithConstantOffset(SDValue Op) const;
1803
1804 /// Test whether the given SDValue is known to never be NaN. If \p SNaN is
1805 /// true, returns if \p Op is known to never be a signaling NaN (it may still
1806 /// be a qNaN).
1807 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
1808
1809 /// \returns true if \p Op is known to never be a signaling NaN.
1810 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
1811 return isKnownNeverNaN(Op, true, Depth);
1812 }
1813
1814 /// Test whether the given floating point SDValue is known to never be
1815 /// positive or negative zero.
1816 bool isKnownNeverZeroFloat(SDValue Op) const;
1817
1818 /// Test whether the given SDValue is known to contain non-zero value(s).
1819 bool isKnownNeverZero(SDValue Op) const;
1820
1821 /// Test whether two SDValues are known to compare equal. This
1822 /// is true if they are the same value, or if one is negative zero and the
1823 /// other positive zero.
1824 bool isEqualTo(SDValue A, SDValue B) const;
1825
1826 /// Return true if A and B have no common bits set. As an example, this can
1827 /// allow an 'add' to be transformed into an 'or'.
1828 bool haveNoCommonBitsSet(SDValue A, SDValue B) const;
1829
1830 /// Test whether \p V has a splatted value for all the demanded elements.
1831 ///
1832 /// On success \p UndefElts will indicate the elements that have UNDEF
1833 /// values instead of the splat value, this is only guaranteed to be correct
1834 /// for \p DemandedElts.
1835 ///
1836 /// NOTE: The function will return true for a demanded splat of UNDEF values.
1837 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts,
1838 unsigned Depth = 0);
1839
1840 /// Test whether \p V has a splatted value.
1841 bool isSplatValue(SDValue V, bool AllowUndefs = false);
1842
1843 /// If V is a splatted value, return the source vector and its splat index.
1844 SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
1845
1846 /// If V is a splat vector, return its scalar source operand by extracting
1847 /// that element from the source vector. If LegalTypes is true, this method
1848 /// may only return a legally-typed splat value. If it cannot legalize the
1849 /// splatted value it will return SDValue().
1850 SDValue getSplatValue(SDValue V, bool LegalTypes = false);
1851
1852 /// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount
1853 /// that is less than the element bit-width of the shift node, return it.
1854 const APInt *getValidShiftAmountConstant(SDValue V,
1855 const APInt &DemandedElts) const;
1856
1857 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less
1858 /// than the element bit-width of the shift node, return the minimum value.
1859 const APInt *
1860 getValidMinimumShiftAmountConstant(SDValue V,
1861 const APInt &DemandedElts) const;
1862
1863 /// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less
1864 /// than the element bit-width of the shift node, return the maximum value.
1865 const APInt *
1866 getValidMaximumShiftAmountConstant(SDValue V,
1867 const APInt &DemandedElts) const;
1868
1869 /// Match a binop + shuffle pyramid that represents a horizontal reduction
1870 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p
1871 /// Extract. The reduction must use one of the opcodes listed in /p
1872 /// CandidateBinOps and on success /p BinOp will contain the matching opcode.
1873 /// Returns the vector that is being reduced on, or SDValue() if a reduction
1874 /// was not matched. If \p AllowPartials is set then in the case of a
1875 /// reduction pattern that only matches the first few stages, the extracted
1876 /// subvector of the start of the reduction is returned.
1877 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
1878 ArrayRef<ISD::NodeType> CandidateBinOps,
1879 bool AllowPartials = false);
1880
1881 /// Utility function used by legalize and lowering to
1882 /// "unroll" a vector operation by splitting out the scalars and operating
1883 /// on each element individually. If the ResNE is 0, fully unroll the vector
1884 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
1885 /// If the ResNE is greater than the width of the vector op, unroll the
1886 /// vector op and fill the end of the resulting vector with UNDEFS.
1887 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
1888
1889 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
1890 /// This is a separate function because those opcodes have two results.
1891 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
1892 unsigned ResNE = 0);
1893
1894 /// Return true if loads are next to each other and can be
1895 /// merged. Check that both are nonvolatile and if LD is loading
1896 /// 'Bytes' bytes from a location that is 'Dist' units away from the
1897 /// location that the 'Base' load is loading from.
1898 bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
1899 unsigned Bytes, int Dist) const;
1900
1901 /// Infer alignment of a load / store address. Return None if it cannot be
1902 /// inferred.
1903 MaybeAlign InferPtrAlign(SDValue Ptr) const;
1904
1905 /// Compute the VTs needed for the low/hi parts of a type
1906 /// which is split (or expanded) into two not necessarily identical pieces.
1907 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const;
1908
1909 /// Compute the VTs needed for the low/hi parts of a type, dependent on an
1910 /// enveloping VT that has been split into two identical pieces. Sets the
1911 /// HisIsEmpty flag when hi type has zero storage size.
1912 std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
1913 bool *HiIsEmpty) const;
1914
1915 /// Split the vector with EXTRACT_SUBVECTOR using the provides
1916 /// VTs and return the low/high part.
1917 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
1918 const EVT &LoVT, const EVT &HiVT);
1919
1920 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
1921 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
1922 EVT LoVT, HiVT;
1923 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
1924 return SplitVector(N, DL, LoVT, HiVT);
1925 }
1926
1927 /// Split the node's operand with EXTRACT_SUBVECTOR and
1928 /// return the low/high part.
1929 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
1930 {
1931 return SplitVector(N->getOperand(OpNo), SDLoc(N));
1932 }
1933
1934 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
1935 SDValue WidenVector(const SDValue &N, const SDLoc &DL);
1936
1937 /// Append the extracted elements from Start to Count out of the vector Op in
1938 /// Args. If Count is 0, all of the elements will be extracted. The extracted
1939 /// elements will have type EVT if it is provided, and otherwise their type
1940 /// will be Op's element type.
1941 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
1942 unsigned Start = 0, unsigned Count = 0,
1943 EVT EltVT = EVT());
1944
1945 /// Compute the default alignment value for the given type.
1946 Align getEVTAlign(EVT MemoryVT) const;
1947 /// Compute the default alignment value for the given type.
1948 /// FIXME: Remove once transition to Align is over.
1949 inline unsigned getEVTAlignment(EVT MemoryVT) const {
1950 return getEVTAlign(MemoryVT).value();
1951 }
1952
1953 /// Test whether the given value is a constant int or similar node.
1954 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const;
1955
1956 /// Test whether the given value is a constant FP or similar node.
1957 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) const ;
1958
1959 /// \returns true if \p N is any kind of constant or build_vector of
1960 /// constants, int or float. If a vector, it may not necessarily be a splat.
1961 inline bool isConstantValueOfAnyType(SDValue N) const {
1962 return isConstantIntBuildVectorOrConstantInt(N) ||
1963 isConstantFPBuildVectorOrConstantFP(N);
1964 }
1965
1966 void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) {
1967 SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo);
1968 }
1969
1970 CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) {
1971 auto I = SDCallSiteDbgInfo.find(CallNode);
1972 if (I != SDCallSiteDbgInfo.end())
1973 return std::move(I->second).CSInfo;
1974 return CallSiteInfo();
1975 }
1976
1977 void addHeapAllocSite(const SDNode *Node, MDNode *MD) {
1978 SDCallSiteDbgInfo[Node].HeapAllocSite = MD;
1979 }
1980
1981 /// Return the HeapAllocSite type associated with the SDNode, if it exists.
1982 MDNode *getHeapAllocSite(const SDNode *Node) {
1983 auto It = SDCallSiteDbgInfo.find(Node);
1984 if (It == SDCallSiteDbgInfo.end())
1985 return nullptr;
1986 return It->second.HeapAllocSite;
1987 }
1988
1989 void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) {
1990 if (NoMerge)
1991 SDCallSiteDbgInfo[Node].NoMerge = NoMerge;
1992 }
1993
1994 bool getNoMergeSiteInfo(const SDNode *Node) {
1995 auto I = SDCallSiteDbgInfo.find(Node);
1996 if (I == SDCallSiteDbgInfo.end())
1997 return false;
1998 return I->second.NoMerge;
1999 }
2000
2001 /// Return the current function's default denormal handling kind for the given
2002 /// floating point type.
2003 DenormalMode getDenormalMode(EVT VT) const {
2004 return MF->getDenormalMode(EVTToAPFloatSemantics(VT));
2005 }
2006
2007 bool shouldOptForSize() const;
2008
2009 /// Get the (commutative) neutral element for the given opcode, if it exists.
2010 SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT,
2011 SDNodeFlags Flags);
2012
2013private:
2014 void InsertNode(SDNode *N);
2015 bool RemoveNodeFromCSEMaps(SDNode *N);
2016 void AddModifiedNodeToCSEMaps(SDNode *N);
2017 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
2018 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
2019 void *&InsertPos);
2020 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
2021 void *&InsertPos);
2022 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc);
2023
2024 void DeleteNodeNotInCSEMaps(SDNode *N);
2025 void DeallocateNode(SDNode *N);
2026
2027 void allnodes_clear();
2028
2029 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
2030 /// not, return the insertion token that will make insertion faster. This
2031 /// overload is for nodes other than Constant or ConstantFP, use the other one
2032 /// for those.
2033 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);
2034
2035 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
2036 /// not, return the insertion token that will make insertion faster. Performs
2037 /// additional processing for constant nodes.
2038 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL,
2039 void *&InsertPos);
2040
2041 /// List of non-single value types.
2042 FoldingSet<SDVTListNode> VTListMap;
2043
2044 /// Maps to auto-CSE operations.
2045 std::vector<CondCodeSDNode*> CondCodeNodes;
2046
2047 std::vector<SDNode*> ValueTypeNodes;
2048 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes;
2049 StringMap<SDNode*> ExternalSymbols;
2050
2051 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols;
2052 DenseMap<MCSymbol *, SDNode *> MCSymbols;
2053
2054 FlagInserter *Inserter = nullptr;
2055};
2056
2057template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> {
2058 using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>;
2059
2060 static nodes_iterator nodes_begin(SelectionDAG *G) {
2061 return nodes_iterator(G->allnodes_begin());
2062 }
2063
2064 static nodes_iterator nodes_end(SelectionDAG *G) {
2065 return nodes_iterator(G->allnodes_end());
2066 }
2067};
2068
2069} // end namespace llvm
2070
2071#endif // LLVM_CODEGEN_SELECTIONDAG_H

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h

1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
10// and dyn_cast_or_null<X>() templates.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_CASTING_H
15#define LLVM_SUPPORT_CASTING_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/type_traits.h"
19#include <cassert>
20#include <memory>
21#include <type_traits>
22
23namespace llvm {
24
25//===----------------------------------------------------------------------===//
26// isa<x> Support Templates
27//===----------------------------------------------------------------------===//
28
29// Define a template that can be specialized by smart pointers to reflect the
30// fact that they are automatically dereferenced, and are not involved with the
31// template selection process... the default implementation is a noop.
32//
33template<typename From> struct simplify_type {
34 using SimpleType = From; // The real type this represents...
35
36 // An accessor to get the real value...
37 static SimpleType &getSimplifiedValue(From &Val) { return Val; }
38};
39
40template<typename From> struct simplify_type<const From> {
41 using NonConstSimpleType = typename simplify_type<From>::SimpleType;
42 using SimpleType =
43 typename add_const_past_pointer<NonConstSimpleType>::type;
44 using RetType =
45 typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
46
47 static RetType getSimplifiedValue(const From& Val) {
48 return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
49 }
50};
51
52// The core of the implementation of isa<X> is here; To and From should be
53// the names of classes. This template can be specialized to customize the
54// implementation of isa<> without rewriting it from scratch.
55template <typename To, typename From, typename Enabler = void>
56struct isa_impl {
57 static inline bool doit(const From &Val) {
58 return To::classof(&Val);
59 }
60};
61
62/// Always allow upcasts, and perform no dynamic check for them.
63template <typename To, typename From>
64struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
65 static inline bool doit(const From &) { return true; }
66};
67
68template <typename To, typename From> struct isa_impl_cl {
69 static inline bool doit(const From &Val) {
70 return isa_impl<To, From>::doit(Val);
71 }
72};
73
74template <typename To, typename From> struct isa_impl_cl<To, const From> {
75 static inline bool doit(const From &Val) {
76 return isa_impl<To, From>::doit(Val);
77 }
78};
79
80template <typename To, typename From>
81struct isa_impl_cl<To, const std::unique_ptr<From>> {
82 static inline bool doit(const std::unique_ptr<From> &Val) {
83 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 83, __extension__ __PRETTY_FUNCTION__))
;
84 return isa_impl_cl<To, From>::doit(*Val);
85 }
86};
87
88template <typename To, typename From> struct isa_impl_cl<To, From*> {
89 static inline bool doit(const From *Val) {
90 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 90, __extension__ __PRETTY_FUNCTION__))
;
91 return isa_impl<To, From>::doit(*Val);
92 }
93};
94
95template <typename To, typename From> struct isa_impl_cl<To, From*const> {
96 static inline bool doit(const From *Val) {
97 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 97, __extension__ __PRETTY_FUNCTION__))
;
98 return isa_impl<To, From>::doit(*Val);
99 }
100};
101
102template <typename To, typename From> struct isa_impl_cl<To, const From*> {
103 static inline bool doit(const From *Val) {
104 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 104, __extension__ __PRETTY_FUNCTION__))
;
105 return isa_impl<To, From>::doit(*Val);
106 }
107};
108
109template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
110 static inline bool doit(const From *Val) {
111 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 111, __extension__ __PRETTY_FUNCTION__))
;
112 return isa_impl<To, From>::doit(*Val);
113 }
114};
115
116template<typename To, typename From, typename SimpleFrom>
117struct isa_impl_wrap {
118 // When From != SimplifiedType, we can simplify the type some more by using
119 // the simplify_type template.
120 static bool doit(const From &Val) {
121 return isa_impl_wrap<To, SimpleFrom,
122 typename simplify_type<SimpleFrom>::SimpleType>::doit(
123 simplify_type<const From>::getSimplifiedValue(Val));
124 }
125};
126
127template<typename To, typename FromTy>
128struct isa_impl_wrap<To, FromTy, FromTy> {
129 // When From == SimpleType, we are as simple as we are going to get.
130 static bool doit(const FromTy &Val) {
131 return isa_impl_cl<To,FromTy>::doit(Val);
132 }
133};
134
135// isa<X> - Return true if the parameter to the template is an instance of one
136// of the template type arguments. Used like this:
137//
138// if (isa<Type>(myVal)) { ... }
139// if (isa<Type0, Type1, Type2>(myVal)) { ... }
140//
141template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
142 return isa_impl_wrap<X, const Y,
143 typename simplify_type<const Y>::SimpleType>::doit(Val);
144}
145
146template <typename First, typename Second, typename... Rest, typename Y>
147LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
148 return isa<First>(Val) || isa<Second, Rest...>(Val);
149}
150
151// isa_and_nonnull<X> - Functionally identical to isa, except that a null value
152// is accepted.
153//
154template <typename... X, class Y>
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa_and_nonnull(const Y &Val) {
156 if (!Val)
157 return false;
158 return isa<X...>(Val);
159}
160
161//===----------------------------------------------------------------------===//
162// cast<x> Support Templates
163//===----------------------------------------------------------------------===//
164
165template<class To, class From> struct cast_retty;
166
167// Calculate what type the 'cast' function should return, based on a requested
168// type of To and a source type of From.
169template<class To, class From> struct cast_retty_impl {
170 using ret_type = To &; // Normal case, return Ty&
171};
172template<class To, class From> struct cast_retty_impl<To, const From> {
173 using ret_type = const To &; // Normal case, return Ty&
174};
175
176template<class To, class From> struct cast_retty_impl<To, From*> {
177 using ret_type = To *; // Pointer arg case, return Ty*
178};
179
180template<class To, class From> struct cast_retty_impl<To, const From*> {
181 using ret_type = const To *; // Constant pointer arg case, return const Ty*
182};
183
184template<class To, class From> struct cast_retty_impl<To, const From*const> {
185 using ret_type = const To *; // Constant pointer arg case, return const Ty*
186};
187
188template <class To, class From>
189struct cast_retty_impl<To, std::unique_ptr<From>> {
190private:
191 using PointerType = typename cast_retty_impl<To, From *>::ret_type;
192 using ResultType = std::remove_pointer_t<PointerType>;
193
194public:
195 using ret_type = std::unique_ptr<ResultType>;
196};
197
198template<class To, class From, class SimpleFrom>
199struct cast_retty_wrap {
200 // When the simplified type and the from type are not the same, use the type
201 // simplifier to reduce the type, then reuse cast_retty_impl to get the
202 // resultant type.
203 using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
204};
205
206template<class To, class FromTy>
207struct cast_retty_wrap<To, FromTy, FromTy> {
208 // When the simplified type is equal to the from type, use it directly.
209 using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
210};
211
212template<class To, class From>
213struct cast_retty {
214 using ret_type = typename cast_retty_wrap<
215 To, From, typename simplify_type<From>::SimpleType>::ret_type;
216};
217
218// Ensure the non-simple values are converted using the simplify_type template
219// that may be specialized by smart pointers...
220//
221template<class To, class From, class SimpleFrom> struct cast_convert_val {
222 // This is not a simple type, use the template to simplify it...
223 static typename cast_retty<To, From>::ret_type doit(From &Val) {
224 return cast_convert_val<To, SimpleFrom,
19
Returning without writing to 'Val.Node'
225 typename simplify_type<SimpleFrom>::SimpleType>::doit(
226 simplify_type<From>::getSimplifiedValue(Val));
16
Calling 'simplify_type::getSimplifiedValue'
18
Returning from 'simplify_type::getSimplifiedValue'
227 }
228};
229
230template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
231 // This _is_ a simple type, just cast it.
232 static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
233 typename cast_retty<To, FromTy>::ret_type Res2
234 = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
235 return Res2;
236 }
237};
238
239template <class X> struct is_simple_type {
240 static const bool value =
241 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
242};
243
244// cast<X> - Return the argument parameter cast to the specified type. This
245// casting operator asserts that the type is correct, so it does not return null
246// on failure. It does not allow a null argument (use cast_or_null for that).
247// It is typically used like this:
248//
249// cast<Instruction>(myVal)->getParent()
250//
251template <class X, class Y>
252inline std::enable_if_t<!is_simple_type<Y>::value,
253 typename cast_retty<X, const Y>::ret_type>
254cast(const Y &Val) {
255 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 255, __extension__ __PRETTY_FUNCTION__))
;
256 return cast_convert_val<
257 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
258}
259
260template <class X, class Y>
261inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
262 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 262, __extension__ __PRETTY_FUNCTION__))
;
13
Assuming 'Val' is a 'ConstantSDNode'
14
'?' condition is true
263 return cast_convert_val<X, Y,
15
Calling 'cast_convert_val::doit'
20
Returning from 'cast_convert_val::doit'
21
Returning without writing to 'Val.Node'
264 typename simplify_type<Y>::SimpleType>::doit(Val);
265}
266
267template <class X, class Y>
268inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
269 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 269, __extension__ __PRETTY_FUNCTION__))
;
270 return cast_convert_val<X, Y*,
271 typename simplify_type<Y*>::SimpleType>::doit(Val);
272}
273
274template <class X, class Y>
275inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
276cast(std::unique_ptr<Y> &&Val) {
277 assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val.get()) &&
"cast<Ty>() argument of incompatible type!") ? void (0
) : __assert_fail ("isa<X>(Val.get()) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 277, __extension__ __PRETTY_FUNCTION__))
;
278 using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
279 return ret_type(
280 cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
281 Val.release()));
282}
283
284// cast_or_null<X> - Functionally identical to cast, except that a null value is
285// accepted.
286//
287template <class X, class Y>
288LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
289 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
290cast_or_null(const Y &Val) {
291 if (!Val)
292 return nullptr;
293 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 293, __extension__ __PRETTY_FUNCTION__))
;
294 return cast<X>(Val);
295}
296
297template <class X, class Y>
298LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
299 typename cast_retty<X, Y>::ret_type>
300cast_or_null(Y &Val) {
301 if (!Val)
302 return nullptr;
303 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 303, __extension__ __PRETTY_FUNCTION__))
;
304 return cast<X>(Val);
305}
306
307template <class X, class Y>
308LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
309cast_or_null(Y *Val) {
310 if (!Val) return nullptr;
311 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 311, __extension__ __PRETTY_FUNCTION__))
;
312 return cast<X>(Val);
313}
314
315template <class X, class Y>
316inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
317cast_or_null(std::unique_ptr<Y> &&Val) {
318 if (!Val)
319 return nullptr;
320 return cast<X>(std::move(Val));
321}
322
323// dyn_cast<X> - Return the argument parameter cast to the specified type. This
324// casting operator returns null if the argument is of the wrong type, so it can
325// be used to test for a type as well as cast if successful. This should be
326// used in the context of an if statement like this:
327//
328// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
329//
330
331template <class X, class Y>
332LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
333 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
334dyn_cast(const Y &Val) {
335 return isa<X>(Val) ? cast<X>(Val) : nullptr;
336}
337
338template <class X, class Y>
339LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
340 return isa<X>(Val) ? cast<X>(Val) : nullptr;
10
Assuming 'Val' is a 'ConstantSDNode'
11
'?' condition is true
12
Calling 'cast<llvm::ConstantSDNode, llvm::SDValue>'
22
Returning from 'cast<llvm::ConstantSDNode, llvm::SDValue>'
23
Returning without writing to 'Val.Node'
341}
342
343template <class X, class Y>
344LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
345 return isa<X>(Val) ? cast<X>(Val) : nullptr;
346}
347
348// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
349// value is accepted.
350//
351template <class X, class Y>
352LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
353 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
354dyn_cast_or_null(const Y &Val) {
355 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
356}
357
358template <class X, class Y>
359LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
360 typename cast_retty<X, Y>::ret_type>
361dyn_cast_or_null(Y &Val) {
362 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
363}
364
365template <class X, class Y>
366LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
367dyn_cast_or_null(Y *Val) {
368 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
369}
370
371// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
372// taking ownership of the input pointer iff isa<X>(Val) is true. If the
373// cast is successful, From refers to nullptr on exit and the casted value
374// is returned. If the cast is unsuccessful, the function returns nullptr
375// and From is unchanged.
376template <class X, class Y>
377LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
378 -> decltype(cast<X>(Val)) {
379 if (!isa<X>(Val))
380 return nullptr;
381 return cast<X>(std::move(Val));
382}
383
384template <class X, class Y>
385LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
386 return unique_dyn_cast<X, Y>(Val);
387}
388
389// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
390// a null value is accepted.
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
393 -> decltype(cast<X>(Val)) {
394 if (!Val)
395 return nullptr;
396 return unique_dyn_cast<X, Y>(Val);
397}
398
399template <class X, class Y>
400LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
401 return unique_dyn_cast_or_null<X, Y>(Val);
402}
403
404} // end namespace llvm
405
406#endif // LLVM_SUPPORT_CASTING_H

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
17
Returning without writing to 'Val.Node'
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class MaskedLoadStoreSDNode;
513 friend class MaskedGatherScatterSDNode;
514
515 uint16_t : NumMemSDNodeBits;
516
517 // This storage is shared between disparate class hierarchies to hold an
518 // enumeration specific to the class hierarchy in use.
519 // LSBaseSDNode => enum ISD::MemIndexedMode
520 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
521 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
522 uint16_t AddressingMode : 3;
523 };
524 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
525
526 class LoadSDNodeBitfields {
527 friend class LoadSDNode;
528 friend class MaskedLoadSDNode;
529 friend class MaskedGatherSDNode;
530
531 uint16_t : NumLSBaseSDNodeBits;
532
533 uint16_t ExtTy : 2; // enum ISD::LoadExtType
534 uint16_t IsExpanding : 1;
535 };
536
537 class StoreSDNodeBitfields {
538 friend class StoreSDNode;
539 friend class MaskedStoreSDNode;
540 friend class MaskedScatterSDNode;
541
542 uint16_t : NumLSBaseSDNodeBits;
543
544 uint16_t IsTruncating : 1;
545 uint16_t IsCompressing : 1;
546 };
547
548 union {
549 char RawSDNodeBits[sizeof(uint16_t)];
550 SDNodeBitfields SDNodeBits;
551 ConstantSDNodeBitfields ConstantSDNodeBits;
552 MemSDNodeBitfields MemSDNodeBits;
553 LSBaseSDNodeBitfields LSBaseSDNodeBits;
554 LoadSDNodeBitfields LoadSDNodeBits;
555 StoreSDNodeBitfields StoreSDNodeBits;
556 };
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK
560
561 // RawSDNodeBits must cover the entirety of the union. This means that all of
562 // the union's members must have size <= RawSDNodeBits. We write the RHS as
563 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
564 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
565 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
566 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
567 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
568 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
570
571private:
572 friend class SelectionDAG;
573 // TODO: unfriend HandleSDNode once we fix its operand handling.
574 friend class HandleSDNode;
575
576 /// Unique id per SDNode in the DAG.
577 int NodeId = -1;
578
579 /// The values that are used by this operation.
580 SDUse *OperandList = nullptr;
581
582 /// The types of the values this node defines. SDNode's may
583 /// define multiple values simultaneously.
584 const EVT *ValueList;
585
586 /// List of uses for this SDNode.
587 SDUse *UseList = nullptr;
588
589 /// The number of entries in the Operand/Value list.
590 unsigned short NumOperands = 0;
591 unsigned short NumValues;
592
593 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
594 // original LLVM instructions.
595 // This is used for turning off scheduling, because we'll forgo
596 // the normal scheduling algorithms and output the instructions according to
597 // this ordering.
598 unsigned IROrder;
599
600 /// Source line information.
601 DebugLoc debugLoc;
602
603 /// Return a pointer to the specified value type.
604 static const EVT *getValueTypeList(EVT VT);
605
606 SDNodeFlags Flags;
607
608public:
609 /// Unique and persistent id per SDNode in the DAG.
610 /// Used for debug printing.
611 uint16_t PersistentId;
612
613 //===--------------------------------------------------------------------===//
614 // Accessors
615 //
616
617 /// Return the SelectionDAG opcode value for this node. For
618 /// pre-isel nodes (those for which isMachineOpcode returns false), these
619 /// are the opcode values in the ISD and <target>ISD namespaces. For
620 /// post-isel opcodes, see getMachineOpcode.
621 unsigned getOpcode() const { return (unsigned short)NodeType; }
622
623 /// Test if this node has a target-specific opcode (in the
624 /// \<target\>ISD namespace).
625 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
626
627 /// Test if this node has a target-specific opcode that may raise
628 /// FP exceptions (in the \<target\>ISD namespace and greater than
629 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
630 /// opcode are currently automatically considered to possibly raise
631 /// FP exceptions as well.
632 bool isTargetStrictFPOpcode() const {
633 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
634 }
635
636 /// Test if this node has a target-specific
637 /// memory-referencing opcode (in the \<target\>ISD namespace and
638 /// greater than FIRST_TARGET_MEMORY_OPCODE).
639 bool isTargetMemoryOpcode() const {
640 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
641 }
642
643 /// Return true if the type of the node type undefined.
644 bool isUndef() const { return NodeType == ISD::UNDEF; }
645
646 /// Test if this node is a memory intrinsic (with valid pointer information).
647 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
648 /// non-memory intrinsics (with chains) that are not really instances of
649 /// MemSDNode. For such nodes, we need some extra state to determine the
650 /// proper classof relationship.
651 bool isMemIntrinsic() const {
652 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
653 NodeType == ISD::INTRINSIC_VOID) &&
654 SDNodeBits.IsMemIntrinsic;
655 }
656
657 /// Test if this node is a strict floating point pseudo-op.
658 bool isStrictFPOpcode() {
659 switch (NodeType) {
660 default:
661 return false;
662 case ISD::STRICT_FP16_TO_FP:
663 case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
665 case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
667 return true;
668 }
669 }
670
671 /// Test if this node has a post-isel opcode, directly
672 /// corresponding to a MachineInstr opcode.
673 bool isMachineOpcode() const { return NodeType < 0; }
674
675 /// This may only be called if isMachineOpcode returns
676 /// true. It returns the MachineInstr opcode value that the node's opcode
677 /// corresponds to.
678 unsigned getMachineOpcode() const {
679 assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 679, __extension__ __PRETTY_FUNCTION__))
;
680 return ~NodeType;
681 }
682
683 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
684 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
685
686 bool isDivergent() const { return SDNodeBits.IsDivergent; }
687
688 /// Return true if there are no uses of this node.
689 bool use_empty() const { return UseList == nullptr; }
690
691 /// Return true if there is exactly one use of this node.
692 bool hasOneUse() const { return hasSingleElement(uses()); }
693
694 /// Return the number of uses of this node. This method takes
695 /// time proportional to the number of uses.
696 size_t use_size() const { return std::distance(use_begin(), use_end()); }
697
698 /// Return the unique node id.
699 int getNodeId() const { return NodeId; }
700
701 /// Set unique node id.
702 void setNodeId(int Id) { NodeId = Id; }
703
704 /// Return the node ordering.
705 unsigned getIROrder() const { return IROrder; }
706
707 /// Set the node ordering.
708 void setIROrder(unsigned Order) { IROrder = Order; }
709
710 /// Return the source location info.
711 const DebugLoc &getDebugLoc() const { return debugLoc; }
712
713 /// Set source location info. Try to avoid this, putting
714 /// it in the constructor is preferable.
715 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
716
717 /// This class provides iterator support for SDUse
718 /// operands that use a specific SDNode.
719 class use_iterator {
720 friend class SDNode;
721
722 SDUse *Op = nullptr;
723
724 explicit use_iterator(SDUse *op) : Op(op) {}
725
726 public:
727 using iterator_category = std::forward_iterator_tag;
728 using value_type = SDUse;
729 using difference_type = std::ptrdiff_t;
730 using pointer = value_type *;
731 using reference = value_type &;
732
733 use_iterator() = default;
734 use_iterator(const use_iterator &I) : Op(I.Op) {}
735
736 bool operator==(const use_iterator &x) const {
737 return Op == x.Op;
738 }
739 bool operator!=(const use_iterator &x) const {
740 return !operator==(x);
741 }
742
743 /// Return true if this iterator is at the end of uses list.
744 bool atEnd() const { return Op == nullptr; }
745
746 // Iterator traversal: forward iteration only.
747 use_iterator &operator++() { // Preincrement
748 assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 748, __extension__ __PRETTY_FUNCTION__))
;
749 Op = Op->getNext();
750 return *this;
751 }
752
753 use_iterator operator++(int) { // Postincrement
754 use_iterator tmp = *this; ++*this; return tmp;
755 }
756
757 /// Retrieve a pointer to the current user node.
758 SDNode *operator*() const {
759 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 759, __extension__ __PRETTY_FUNCTION__))
;
760 return Op->getUser();
761 }
762
763 SDNode *operator->() const { return operator*(); }
764
765 SDUse &getUse() const { return *Op; }
766
767 /// Retrieve the operand # of this use in its user.
768 unsigned getOperandNo() const {
769 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __extension__ __PRETTY_FUNCTION__))
;
770 return (unsigned)(Op - Op->getUser()->OperandList);
771 }
772 };
773
774 /// Provide iteration support to walk over all uses of an SDNode.
775 use_iterator use_begin() const {
776 return use_iterator(UseList);
777 }
778
779 static use_iterator use_end() { return use_iterator(nullptr); }
780
781 inline iterator_range<use_iterator> uses() {
782 return make_range(use_begin(), use_end());
783 }
784 inline iterator_range<use_iterator> uses() const {
785 return make_range(use_begin(), use_end());
786 }
787
788 /// Return true if there are exactly NUSES uses of the indicated value.
789 /// This method ignores uses of other values defined by this operation.
790 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
791
792 /// Return true if there are any use of the indicated value.
793 /// This method ignores uses of other values defined by this operation.
794 bool hasAnyUseOfValue(unsigned Value) const;
795
796 /// Return true if this node is the only use of N.
797 bool isOnlyUserOf(const SDNode *N) const;
798
799 /// Return true if this node is an operand of N.
800 bool isOperandOf(const SDNode *N) const;
801
802 /// Return true if this node is a predecessor of N.
803 /// NOTE: Implemented on top of hasPredecessor and every bit as
804 /// expensive. Use carefully.
805 bool isPredecessorOf(const SDNode *N) const {
806 return N->hasPredecessor(this);
807 }
808
809 /// Return true if N is a predecessor of this node.
810 /// N is either an operand of this node, or can be reached by recursively
811 /// traversing up the operands.
812 /// NOTE: This is an expensive method. Use it carefully.
813 bool hasPredecessor(const SDNode *N) const;
814
815 /// Returns true if N is a predecessor of any node in Worklist. This
816 /// helper keeps Visited and Worklist sets externally to allow unions
817 /// searches to be performed in parallel, caching of results across
818 /// queries and incremental addition to Worklist. Stops early if N is
819 /// found but will resume. Remember to clear Visited and Worklists
820 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
821 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
822 /// topologically ordered (Operands have strictly smaller node id) and search
823 /// can be pruned leveraging this.
824 static bool hasPredecessorHelper(const SDNode *N,
825 SmallPtrSetImpl<const SDNode *> &Visited,
826 SmallVectorImpl<const SDNode *> &Worklist,
827 unsigned int MaxSteps = 0,
828 bool TopologicalPrune = false) {
829 SmallVector<const SDNode *, 8> DeferredNodes;
830 if (Visited.count(N))
831 return true;
832
833 // Node Id's are assigned in three places: As a topological
834 // ordering (> 0), during legalization (results in values set to
835 // 0), new nodes (set to -1). If N has a topolgical id then we
836 // know that all nodes with ids smaller than it cannot be
837 // successors and we need not check them. Filter out all node
838 // that can't be matches. We add them to the worklist before exit
839 // in case of multiple calls. Note that during selection the topological id
840 // may be violated if a node's predecessor is selected before it. We mark
841 // this at selection negating the id of unselected successors and
842 // restricting topological pruning to positive ids.
843
844 int NId = N->getNodeId();
845 // If we Invalidated the Id, reconstruct original NId.
846 if (NId < -1)
847 NId = -(NId + 1);
848
849 bool Found = false;
850 while (!Worklist.empty()) {
851 const SDNode *M = Worklist.pop_back_val();
852 int MId = M->getNodeId();
853 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
854 (MId > 0) && (MId < NId)) {
855 DeferredNodes.push_back(M);
856 continue;
857 }
858 for (const SDValue &OpV : M->op_values()) {
859 SDNode *Op = OpV.getNode();
860 if (Visited.insert(Op).second)
861 Worklist.push_back(Op);
862 if (Op == N)
863 Found = true;
864 }
865 if (Found)
866 break;
867 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
868 break;
869 }
870 // Push deferred nodes back on worklist.
871 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
872 // If we bailed early, conservatively return found.
873 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
874 return true;
875 return Found;
876 }
877
878 /// Return true if all the users of N are contained in Nodes.
879 /// NOTE: Requires at least one match, but doesn't require them all.
880 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
881
882 /// Return the number of values used by this operation.
883 unsigned getNumOperands() const { return NumOperands; }
884
885 /// Return the maximum number of operands that a SDNode can hold.
886 static constexpr size_t getMaxNumOperands() {
887 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
888 }
889
890 /// Helper method returns the integer value of a ConstantSDNode operand.
891 inline uint64_t getConstantOperandVal(unsigned Num) const;
892
893 /// Helper method returns the APInt of a ConstantSDNode operand.
894 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
895
896 const SDValue &getOperand(unsigned Num) const {
897 assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 897, __extension__ __PRETTY_FUNCTION__))
;
898 return OperandList[Num];
899 }
900
901 using op_iterator = SDUse *;
902
903 op_iterator op_begin() const { return OperandList; }
904 op_iterator op_end() const { return OperandList+NumOperands; }
905 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
906
907 /// Iterator for directly iterating over the operand SDValue's.
908 struct value_op_iterator
909 : iterator_adaptor_base<value_op_iterator, op_iterator,
910 std::random_access_iterator_tag, SDValue,
911 ptrdiff_t, value_op_iterator *,
912 value_op_iterator *> {
913 explicit value_op_iterator(SDUse *U = nullptr)
914 : iterator_adaptor_base(U) {}
915
916 const SDValue &operator*() const { return I->get(); }
917 };
918
919 iterator_range<value_op_iterator> op_values() const {
920 return make_range(value_op_iterator(op_begin()),
921 value_op_iterator(op_end()));
922 }
923
924 SDVTList getVTList() const {
925 SDVTList X = { ValueList, NumValues };
926 return X;
927 }
928
929 /// If this node has a glue operand, return the node
930 /// to which the glue operand points. Otherwise return NULL.
931 SDNode *getGluedNode() const {
932 if (getNumOperands() != 0 &&
933 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
934 return getOperand(getNumOperands()-1).getNode();
935 return nullptr;
936 }
937
938 /// If this node has a glue value with a user, return
939 /// the user (there is at most one). Otherwise return NULL.
940 SDNode *getGluedUser() const {
941 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
942 if (UI.getUse().get().getValueType() == MVT::Glue)
943 return *UI;
944 return nullptr;
945 }
946
947 SDNodeFlags getFlags() const { return Flags; }
948 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
949
950 /// Clear any flags in this node that aren't also set in Flags.
951 /// If Flags is not in a defined state then this has no effect.
952 void intersectFlagsWith(const SDNodeFlags Flags);
953
954 /// Return the number of values defined/returned by this operator.
955 unsigned getNumValues() const { return NumValues; }
956
957 /// Return the type of a specified result.
958 EVT getValueType(unsigned ResNo) const {
959 assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 959, __extension__ __PRETTY_FUNCTION__))
;
960 return ValueList[ResNo];
961 }
962
963 /// Return the type of a specified result as a simple type.
964 MVT getSimpleValueType(unsigned ResNo) const {
965 return getValueType(ResNo).getSimpleVT();
966 }
967
968 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
969 ///
970 /// If the value type is a scalable vector type, the scalable property will
971 /// be set and the runtime size will be a positive integer multiple of the
972 /// base size.
973 TypeSize getValueSizeInBits(unsigned ResNo) const {
974 return getValueType(ResNo).getSizeInBits();
975 }
976
977 using value_iterator = const EVT *;
978
979 value_iterator value_begin() const { return ValueList; }
980 value_iterator value_end() const { return ValueList+NumValues; }
981 iterator_range<value_iterator> values() const {
982 return llvm::make_range(value_begin(), value_end());
983 }
984
985 /// Return the opcode of this operation for printing.
986 std::string getOperationName(const SelectionDAG *G = nullptr) const;
987 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
988 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
989 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
990 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
991 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
992
993 /// Print a SelectionDAG node and all children down to
994 /// the leaves. The given SelectionDAG allows target-specific nodes
995 /// to be printed in human-readable form. Unlike printr, this will
996 /// print the whole DAG, including children that appear multiple
997 /// times.
998 ///
999 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and children up to
1002 /// depth "depth." The given SelectionDAG allows target-specific
1003 /// nodes to be printed in human-readable form. Unlike printr, this
1004 /// will print children that appear multiple times wherever they are
1005 /// used.
1006 ///
1007 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1008 unsigned depth = 100) const;
1009
1010 /// Dump this node, for debugging.
1011 void dump() const;
1012
1013 /// Dump (recursively) this node and its use-def subgraph.
1014 void dumpr() const;
1015
1016 /// Dump this node, for debugging.
1017 /// The given SelectionDAG allows target-specific nodes to be printed
1018 /// in human-readable form.
1019 void dump(const SelectionDAG *G) const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 /// The given SelectionDAG allows target-specific nodes to be printed
1023 /// in human-readable form.
1024 void dumpr(const SelectionDAG *G) const;
1025
1026 /// printrFull to dbgs(). The given SelectionDAG allows
1027 /// target-specific nodes to be printed in human-readable form.
1028 /// Unlike dumpr, this will print the whole DAG, including children
1029 /// that appear multiple times.
1030 void dumprFull(const SelectionDAG *G = nullptr) const;
1031
1032 /// printrWithDepth to dbgs(). The given
1033 /// SelectionDAG allows target-specific nodes to be printed in
1034 /// human-readable form. Unlike dumpr, this will print children
1035 /// that appear multiple times wherever they are used.
1036 ///
1037 void dumprWithDepth(const SelectionDAG *G = nullptr,
1038 unsigned depth = 100) const;
1039
1040 /// Gather unique data for the node.
1041 void Profile(FoldingSetNodeID &ID) const;
1042
1043 /// This method should only be used by the SDUse class.
1044 void addUse(SDUse &U) { U.addToList(&UseList); }
1045
1046protected:
1047 static SDVTList getSDVTList(EVT VT) {
1048 SDVTList Ret = { getValueTypeList(VT), 1 };
1049 return Ret;
1050 }
1051
1052 /// Create an SDNode.
1053 ///
1054 /// SDNodes are created without any operands, and never own the operand
1055 /// storage. To add operands, see SelectionDAG::createOperands.
1056 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1057 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1058 IROrder(Order), debugLoc(std::move(dl)) {
1059 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1060 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
"Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1060, __extension__ __PRETTY_FUNCTION__))
;
1061 assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
1062 "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
;
1063 }
1064
1065 /// Release the operands and set this node to have zero operands.
1066 void DropOperands();
1067};
1068
1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
1080 DebugLoc DL;
1081 int IROrder = 0;
1082
1083public:
1084 SDLoc() = default;
1085 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1086 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1087 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1088 assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1088, __extension__ __PRETTY_FUNCTION__))
;
1089 if (I)
1090 DL = I->getDebugLoc();
1091 }
1092
1093 unsigned getIROrder() const { return IROrder; }
1094 const DebugLoc &getDebugLoc() const { return DL; }
1095};
1096
1097// Define inline functions from the SDValue class.
1098
1099inline SDValue::SDValue(SDNode *node, unsigned resno)
1100 : Node(node), ResNo(resno) {
1101 // Explicitly check for !ResNo to avoid use-after-free, because there are
1102 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1103 // combines.
1104 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
1105 "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
;
1106 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1106, __extension__ __PRETTY_FUNCTION__))
;
1107}
1108
1109inline unsigned SDValue::getOpcode() const {
1110 return Node->getOpcode();
1111}
1112
1113inline EVT SDValue::getValueType() const {
1114 return Node->getValueType(ResNo);
1115}
1116
1117inline unsigned SDValue::getNumOperands() const {
1118 return Node->getNumOperands();
1119}
1120
1121inline const SDValue &SDValue::getOperand(unsigned i) const {
1122 return Node->getOperand(i);
1123}
1124
1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1126 return Node->getConstantOperandVal(i);
1127}
1128
1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1130 return Node->getConstantOperandAPInt(i);
1131}
1132
1133inline bool SDValue::isTargetOpcode() const {
1134 return Node->isTargetOpcode();
1135}
1136
1137inline bool SDValue::isTargetMemoryOpcode() const {
1138 return Node->isTargetMemoryOpcode();
1139}
1140
1141inline bool SDValue::isMachineOpcode() const {
1142 return Node->isMachineOpcode();
1143}
1144
1145inline unsigned SDValue::getMachineOpcode() const {
1146 return Node->getMachineOpcode();
1147}
1148
1149inline bool SDValue::isUndef() const {
1150 return Node->isUndef();
28
Called C++ object pointer is null
1151}
1152
1153inline bool SDValue::use_empty() const {
1154 return !Node->hasAnyUseOfValue(ResNo);
1155}
1156
1157inline bool SDValue::hasOneUse() const {
1158 return Node->hasNUsesOfValue(1, ResNo);
1159}
1160
1161inline const DebugLoc &SDValue::getDebugLoc() const {
1162 return Node->getDebugLoc();
1163}
1164
1165inline void SDValue::dump() const {
1166 return Node->dump();
1167}
1168
1169inline void SDValue::dump(const SelectionDAG *G) const {
1170 return Node->dump(G);
1171}
1172
1173inline void SDValue::dumpr() const {
1174 return Node->dumpr();
1175}
1176
1177inline void SDValue::dumpr(const SelectionDAG *G) const {
1178 return Node->dumpr(G);
1179}
1180
1181// Define inline functions from the SDUse class.
1182
1183inline void SDUse::set(const SDValue &V) {
1184 if (Val.getNode()) removeFromList();
1185 Val = V;
1186 if (V.getNode()) V.getNode()->addUse(*this);
1187}
1188
1189inline void SDUse::setInitial(const SDValue &V) {
1190 Val = V;
1191 V.getNode()->addUse(*this);
1192}
1193
1194inline void SDUse::setNode(SDNode *N) {
1195 if (Val.getNode()) removeFromList();
1196 Val.setNode(N);
1197 if (N) N->addUse(*this);
1198}
1199
1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand. This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
1205 SDUse Op;
1206
1207public:
1208 explicit HandleSDNode(SDValue X)
1209 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1210 // HandleSDNodes are never inserted into the DAG, so they won't be
1211 // auto-numbered. Use ID 65535 as a sentinel.
1212 PersistentId = 0xffff;
1213
1214 // Manually set up the operand list. This node type is special in that it's
1215 // always stack allocated and SelectionDAG does not manage its operands.
1216 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1217 // be so special.
1218 Op.setUser(this);
1219 Op.setInitial(X);
1220 NumOperands = 1;
1221 OperandList = &Op;
1222 }
1223 ~HandleSDNode();
1224
1225 const SDValue &getValue() const { return Op; }
1226};
1227
1228class AddrSpaceCastSDNode : public SDNode {
1229private:
1230 unsigned SrcAddrSpace;
1231 unsigned DestAddrSpace;
1232
1233public:
1234 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1235 unsigned SrcAS, unsigned DestAS);
1236
1237 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1238 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1239
1240 static bool classof(const SDNode *N) {
1241 return N->getOpcode() == ISD::ADDRSPACECAST;
1242 }
1243};
1244
1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
1248 // VT of in-memory value.
1249 EVT MemoryVT;
1250
1251protected:
1252 /// Memory reference information.
1253 MachineMemOperand *MMO;
1254
1255public:
1256 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1257 EVT memvt, MachineMemOperand *MMO);
1258
1259 bool readMem() const { return MMO->isLoad(); }
1260 bool writeMem() const { return MMO->isStore(); }
1261
1262 /// Returns alignment and volatility of the memory access
1263 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1264 Align getAlign() const { return MMO->getAlign(); }
1265 // FIXME: Remove once transition to getAlign is over.
1266 unsigned getAlignment() const { return MMO->getAlign().value(); }
1267
1268 /// Return the SubclassData value, without HasDebugValue. This contains an
1269 /// encoding of the volatile flag, as well as bits used by subclasses. This
1270 /// function should only be used to compute a FoldingSetNodeID value.
1271 /// The HasDebugValue bit is masked out because CSE map needs to match
1272 /// nodes with debug info with nodes without debug info. Same is about
1273 /// isDivergent bit.
1274 unsigned getRawSubclassData() const {
1275 uint16_t Data;
1276 union {
1277 char RawSDNodeBits[sizeof(uint16_t)];
1278 SDNodeBitfields SDNodeBits;
1279 };
1280 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1281 SDNodeBits.HasDebugValue = 0;
1282 SDNodeBits.IsDivergent = false;
1283 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1284 return Data;
1285 }
1286
1287 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1288 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1289 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1290 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1291
1292 // Returns the offset from the location of the access.
1293 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1294
1295 /// Returns the AA info that describes the dereference.
1296 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1297
1298 /// Returns the Ranges that describes the dereference.
1299 const MDNode *getRanges() const { return MMO->getRanges(); }
1300
1301 /// Returns the synchronization scope ID for this memory operation.
1302 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1303
1304 /// Return the atomic ordering requirements for this memory operation. For
1305 /// cmpxchg atomic operations, return the atomic ordering requirements when
1306 /// store occurs.
1307 AtomicOrdering getSuccessOrdering() const {
1308 return MMO->getSuccessOrdering();
1309 }
1310
1311 /// Return a single atomic ordering that is at least as strong as both the
1312 /// success and failure orderings for an atomic operation. (For operations
1313 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1314 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1315
1316 /// Return true if the memory operation ordering is Unordered or higher.
1317 bool isAtomic() const { return MMO->isAtomic(); }
1318
1319 /// Returns true if the memory operation doesn't imply any ordering
1320 /// constraints on surrounding memory operations beyond the normal memory
1321 /// aliasing rules.
1322 bool isUnordered() const { return MMO->isUnordered(); }
1323
1324 /// Returns true if the memory operation is neither atomic or volatile.
1325 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1326
1327 /// Return the type of the in-memory value.
1328 EVT getMemoryVT() const { return MemoryVT; }
1329
1330 /// Return a MachineMemOperand object describing the memory
1331 /// reference performed by operation.
1332 MachineMemOperand *getMemOperand() const { return MMO; }
1333
1334 const MachinePointerInfo &getPointerInfo() const {
1335 return MMO->getPointerInfo();
1336 }
1337
1338 /// Return the address space for the associated pointer
1339 unsigned getAddressSpace() const {
1340 return getPointerInfo().getAddrSpace();
1341 }
1342
1343 /// Update this MemSDNode's MachineMemOperand information
1344 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1345 /// This must only be used when the new alignment applies to all users of
1346 /// this MachineMemOperand.
1347 void refineAlignment(const MachineMemOperand *NewMMO) {
1348 MMO->refineAlignment(NewMMO);
1349 }
1350
1351 const SDValue &getChain() const { return getOperand(0); }
1352
1353 const SDValue &getBasePtr() const {
1354 switch (getOpcode()) {
1355 case ISD::STORE:
1356 case ISD::MSTORE:
1357 return getOperand(2);
1358 case ISD::MGATHER:
1359 case ISD::MSCATTER:
1360 return getOperand(3);
1361 default:
1362 return getOperand(1);
1363 }
1364 }
1365
1366 // Methods to support isa and dyn_cast
1367 static bool classof(const SDNode *N) {
1368 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1369 // with either an intrinsic or a target opcode.
1370 switch (N->getOpcode()) {
1371 case ISD::LOAD:
1372 case ISD::STORE:
1373 case ISD::PREFETCH:
1374 case ISD::ATOMIC_CMP_SWAP:
1375 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1376 case ISD::ATOMIC_SWAP:
1377 case ISD::ATOMIC_LOAD_ADD:
1378 case ISD::ATOMIC_LOAD_SUB:
1379 case ISD::ATOMIC_LOAD_AND:
1380 case ISD::ATOMIC_LOAD_CLR:
1381 case ISD::ATOMIC_LOAD_OR:
1382 case ISD::ATOMIC_LOAD_XOR:
1383 case ISD::ATOMIC_LOAD_NAND:
1384 case ISD::ATOMIC_LOAD_MIN:
1385 case ISD::ATOMIC_LOAD_MAX:
1386 case ISD::ATOMIC_LOAD_UMIN:
1387 case ISD::ATOMIC_LOAD_UMAX:
1388 case ISD::ATOMIC_LOAD_FADD:
1389 case ISD::ATOMIC_LOAD_FSUB:
1390 case ISD::ATOMIC_LOAD:
1391 case ISD::ATOMIC_STORE:
1392 case ISD::MLOAD:
1393 case ISD::MSTORE:
1394 case ISD::MGATHER:
1395 case ISD::MSCATTER:
1396 return true;
1397 default:
1398 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1399 }
1400 }
1401};
1402
1403/// This is an SDNode representing atomic operations.
1404class AtomicSDNode : public MemSDNode {
1405public:
1406 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1407 EVT MemVT, MachineMemOperand *MMO)
1408 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1409 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
1410 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
;
1411 }
1412
1413 const SDValue &getBasePtr() const { return getOperand(1); }
1414 const SDValue &getVal() const { return getOperand(2); }
1415
1416 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1417 /// otherwise.
1418 bool isCompareAndSwap() const {
1419 unsigned Op = getOpcode();
1420 return Op == ISD::ATOMIC_CMP_SWAP ||
1421 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1422 }
1423
1424 /// For cmpxchg atomic operations, return the atomic ordering requirements
1425 /// when store does not occur.
1426 AtomicOrdering getFailureOrdering() const {
1427 assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1427, __extension__ __PRETTY_FUNCTION__))
;
1428 return MMO->getFailureOrdering();
1429 }
1430
1431 // Methods to support isa and dyn_cast
1432 static bool classof(const SDNode *N) {
1433 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1434 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1435 N->getOpcode() == ISD::ATOMIC_SWAP ||
1436 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1437 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1438 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1439 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1440 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1441 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1442 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1443 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1444 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1445 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1446 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1447 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1448 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1449 N->getOpcode() == ISD::ATOMIC_LOAD ||
1450 N->getOpcode() == ISD::ATOMIC_STORE;
1451 }
1452};
1453
1454/// This SDNode is used for target intrinsics that touch
1455/// memory and need an associated MachineMemOperand. Its opcode may be
1456/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1457/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1458class MemIntrinsicSDNode : public MemSDNode {
1459public:
1460 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1461 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1462 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1463 SDNodeBits.IsMemIntrinsic = true;
1464 }
1465
1466 // Methods to support isa and dyn_cast
1467 static bool classof(const SDNode *N) {
1468 // We lower some target intrinsics to their target opcode
1469 // early a node with a target opcode can be of this class
1470 return N->isMemIntrinsic() ||
1471 N->getOpcode() == ISD::PREFETCH ||
1472 N->isTargetMemoryOpcode();
1473 }
1474};
1475
1476/// This SDNode is used to implement the code generator
1477/// support for the llvm IR shufflevector instruction. It combines elements
1478/// from two input vectors into a new input vector, with the selection and
1479/// ordering of elements determined by an array of integers, referred to as
1480/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1481/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1482/// An index of -1 is treated as undef, such that the code generator may put
1483/// any value in the corresponding element of the result.
1484class ShuffleVectorSDNode : public SDNode {
1485 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1486 // is freed when the SelectionDAG object is destroyed.
1487 const int *Mask;
1488
1489protected:
1490 friend class SelectionDAG;
1491
1492 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1493 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1494
1495public:
1496 ArrayRef<int> getMask() const {
1497 EVT VT = getValueType(0);
1498 return makeArrayRef(Mask, VT.getVectorNumElements());
1499 }
1500
1501 int getMaskElt(unsigned Idx) const {
1502 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1502, __extension__ __PRETTY_FUNCTION__))
;
1503 return Mask[Idx];
1504 }
1505
1506 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1507
1508 int getSplatIndex() const {
1509 assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1509, __extension__ __PRETTY_FUNCTION__))
;
1510 EVT VT = getValueType(0);
1511 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1512 if (Mask[i] >= 0)
1513 return Mask[i];
1514
1515 // We can choose any index value here and be correct because all elements
1516 // are undefined. Return 0 for better potential for callers to simplify.
1517 return 0;
1518 }
1519
1520 static bool isSplatMask(const int *Mask, EVT VT);
1521
1522 /// Change values in a shuffle permute mask assuming
1523 /// the two vector operands have swapped position.
1524 static void commuteMask(MutableArrayRef<int> Mask) {
1525 unsigned NumElems = Mask.size();
1526 for (unsigned i = 0; i != NumElems; ++i) {
1527 int idx = Mask[i];
1528 if (idx < 0)
1529 continue;
1530 else if (idx < (int)NumElems)
1531 Mask[i] = idx + NumElems;
1532 else
1533 Mask[i] = idx - NumElems;
1534 }
1535 }
1536
1537 static bool classof(const SDNode *N) {
1538 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1539 }
1540};
1541
1542class ConstantSDNode : public SDNode {
1543 friend class SelectionDAG;
1544
1545 const ConstantInt *Value;
1546
1547 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1548 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1549 getSDVTList(VT)),
1550 Value(val) {
1551 ConstantSDNodeBits.IsOpaque = isOpaque;
1552 }
1553
1554public:
1555 const ConstantInt *getConstantIntValue() const { return Value; }
1556 const APInt &getAPIntValue() const { return Value->getValue(); }
1557 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1558 int64_t getSExtValue() const { return Value->getSExtValue(); }
1559 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1560 return Value->getLimitedValue(Limit);
1561 }
1562 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1563 Align getAlignValue() const { return Value->getAlignValue(); }
1564
1565 bool isOne() const { return Value->isOne(); }
1566 bool isNullValue() const { return Value->isZero(); }
1567 bool isAllOnesValue() const { return Value->isMinusOne(); }
1568 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1569 bool isMinSignedValue() const { return Value->isMinValue(true); }
1570
1571 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1572
1573 static bool classof(const SDNode *N) {
1574 return N->getOpcode() == ISD::Constant ||
1575 N->getOpcode() == ISD::TargetConstant;
1576 }
1577};
1578
1579uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1580 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1581}
1582
1583const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1584 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1585}
1586
1587class ConstantFPSDNode : public SDNode {
1588 friend class SelectionDAG;
1589
1590 const ConstantFP *Value;
1591
1592 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1593 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1594 DebugLoc(), getSDVTList(VT)),
1595 Value(val) {}
1596
1597public:
1598 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1599 const ConstantFP *getConstantFPValue() const { return Value; }
1600
1601 /// Return true if the value is positive or negative zero.
1602 bool isZero() const { return Value->isZero(); }
1603
1604 /// Return true if the value is a NaN.
1605 bool isNaN() const { return Value->isNaN(); }
1606
1607 /// Return true if the value is an infinity
1608 bool isInfinity() const { return Value->isInfinity(); }
1609
1610 /// Return true if the value is negative.
1611 bool isNegative() const { return Value->isNegative(); }
1612
1613 /// We don't rely on operator== working on double values, as
1614 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1615 /// As such, this method can be used to do an exact bit-for-bit comparison of
1616 /// two floating point values.
1617
1618 /// We leave the version with the double argument here because it's just so
1619 /// convenient to write "2.0" and the like. Without this function we'd
1620 /// have to duplicate its logic everywhere it's called.
1621 bool isExactlyValue(double V) const {
1622 return Value->getValueAPF().isExactlyValue(V);
1623 }
1624 bool isExactlyValue(const APFloat& V) const;
1625
1626 static bool isValueValidForType(EVT VT, const APFloat& Val);
1627
1628 static bool classof(const SDNode *N) {
1629 return N->getOpcode() == ISD::ConstantFP ||
1630 N->getOpcode() == ISD::TargetConstantFP;
1631 }
1632};
1633
1634/// Returns true if \p V is a constant integer zero.
1635bool isNullConstant(SDValue V);
1636
1637/// Returns true if \p V is an FP constant with a value of positive zero.
1638bool isNullFPConstant(SDValue V);
1639
1640/// Returns true if \p V is an integer constant with all bits set.
1641bool isAllOnesConstant(SDValue V);
1642
1643/// Returns true if \p V is a constant integer one.
1644bool isOneConstant(SDValue V);
1645
1646/// Return the non-bitcasted source operand of \p V if it exists.
1647/// If \p V is not a bitcasted value, it is returned as-is.
1648SDValue peekThroughBitcasts(SDValue V);
1649
1650/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1651/// If \p V is not a bitcasted one-use value, it is returned as-is.
1652SDValue peekThroughOneUseBitcasts(SDValue V);
1653
1654/// Return the non-extracted vector source operand of \p V if it exists.
1655/// If \p V is not an extracted subvector, it is returned as-is.
1656SDValue peekThroughExtractSubvectors(SDValue V);
1657
1658/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1659/// constant is canonicalized to be operand 1.
1660bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1661
1662/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1663ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1664 bool AllowTruncation = false);
1665
1666/// Returns the SDNode if it is a demanded constant splat BuildVector or
1667/// constant int.
1668ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1669 bool AllowUndefs = false,
1670 bool AllowTruncation = false);
1671
1672/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1673ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1674
1675/// Returns the SDNode if it is a demanded constant splat BuildVector or
1676/// constant float.
1677ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1678 bool AllowUndefs = false);
1679
1680/// Return true if the value is a constant 0 integer or a splatted vector of
1681/// a constant 0 integer (with no undefs by default).
1682/// Build vector implicit truncation is not an issue for null values.
1683bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1684
1685/// Return true if the value is a constant 1 integer or a splatted vector of a
1686/// constant 1 integer (with no undefs).
1687/// Does not permit build vector implicit truncation.
1688bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1689
1690/// Return true if the value is a constant -1 integer or a splatted vector of a
1691/// constant -1 integer (with no undefs).
1692/// Does not permit build vector implicit truncation.
1693bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1694
1695/// Return true if \p V is either a integer or FP constant.
1696inline bool isIntOrFPConstant(SDValue V) {
1697 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1698}
1699
1700class GlobalAddressSDNode : public SDNode {
1701 friend class SelectionDAG;
1702
1703 const GlobalValue *TheGlobal;
1704 int64_t Offset;
1705 unsigned TargetFlags;
1706
1707 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1708 const GlobalValue *GA, EVT VT, int64_t o,
1709 unsigned TF);
1710
1711public:
1712 const GlobalValue *getGlobal() const { return TheGlobal; }
1713 int64_t getOffset() const { return Offset; }
1714 unsigned getTargetFlags() const { return TargetFlags; }
1715 // Return the address space this GlobalAddress belongs to.
1716 unsigned getAddressSpace() const;
1717
1718 static bool classof(const SDNode *N) {
1719 return N->getOpcode() == ISD::GlobalAddress ||
1720 N->getOpcode() == ISD::TargetGlobalAddress ||
1721 N->getOpcode() == ISD::GlobalTLSAddress ||
1722 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1723 }
1724};
1725
1726class FrameIndexSDNode : public SDNode {
1727 friend class SelectionDAG;
1728
1729 int FI;
1730
1731 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1732 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1733 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1734 }
1735
1736public:
1737 int getIndex() const { return FI; }
1738
1739 static bool classof(const SDNode *N) {
1740 return N->getOpcode() == ISD::FrameIndex ||
1741 N->getOpcode() == ISD::TargetFrameIndex;
1742 }
1743};
1744
1745/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1746/// the offet and size that are started/ended in the underlying FrameIndex.
1747class LifetimeSDNode : public SDNode {
1748 friend class SelectionDAG;
1749 int64_t Size;
1750 int64_t Offset; // -1 if offset is unknown.
1751
1752 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1753 SDVTList VTs, int64_t Size, int64_t Offset)
1754 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1755public:
1756 int64_t getFrameIndex() const {
1757 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1758 }
1759
1760 bool hasOffset() const { return Offset >= 0; }
1761 int64_t getOffset() const {
1762 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1762, __extension__ __PRETTY_FUNCTION__))
;
1763 return Offset;
1764 }
1765 int64_t getSize() const {
1766 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1766, __extension__ __PRETTY_FUNCTION__))
;
1767 return Size;
1768 }
1769
1770 // Methods to support isa and dyn_cast
1771 static bool classof(const SDNode *N) {
1772 return N->getOpcode() == ISD::LIFETIME_START ||
1773 N->getOpcode() == ISD::LIFETIME_END;
1774 }
1775};
1776
1777/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1778/// the index of the basic block being probed. A pseudo probe serves as a place
1779/// holder and will be removed at the end of compilation. It does not have any
1780/// operand because we do not want the instruction selection to deal with any.
1781class PseudoProbeSDNode : public SDNode {
1782 friend class SelectionDAG;
1783 uint64_t Guid;
1784 uint64_t Index;
1785 uint32_t Attributes;
1786
1787 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1788 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1789 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1790 Attributes(Attr) {}
1791
1792public:
1793 uint64_t getGuid() const { return Guid; }
1794 uint64_t getIndex() const { return Index; }
1795 uint32_t getAttributes() const { return Attributes; }
1796
1797 // Methods to support isa and dyn_cast
1798 static bool classof(const SDNode *N) {
1799 return N->getOpcode() == ISD::PSEUDO_PROBE;
1800 }
1801};
1802
1803class JumpTableSDNode : public SDNode {
1804 friend class SelectionDAG;
1805
1806 int JTI;
1807 unsigned TargetFlags;
1808
1809 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1810 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1811 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1812 }
1813
1814public:
1815 int getIndex() const { return JTI; }
1816 unsigned getTargetFlags() const { return TargetFlags; }
1817
1818 static bool classof(const SDNode *N) {
1819 return N->getOpcode() == ISD::JumpTable ||
1820 N->getOpcode() == ISD::TargetJumpTable;
1821 }
1822};
1823
1824class ConstantPoolSDNode : public SDNode {
1825 friend class SelectionDAG;
1826
1827 union {
1828 const Constant *ConstVal;
1829 MachineConstantPoolValue *MachineCPVal;
1830 } Val;
1831 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1832 Align Alignment; // Minimum alignment requirement of CP.
1833 unsigned TargetFlags;
1834
1835 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1836 Align Alignment, unsigned TF)
1837 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1838 DebugLoc(), getSDVTList(VT)),
1839 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1840 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1840, __extension__ __PRETTY_FUNCTION__))
;
1841 Val.ConstVal = c;
1842 }
1843
1844 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1845 Align Alignment, unsigned TF)
1846 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1847 DebugLoc(), getSDVTList(VT)),
1848 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1849 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1849, __extension__ __PRETTY_FUNCTION__))
;
1850 Val.MachineCPVal = v;
1851 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1852 }
1853
1854public:
1855 bool isMachineConstantPoolEntry() const {
1856 return Offset < 0;
1857 }
1858
1859 const Constant *getConstVal() const {
1860 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1860, __extension__ __PRETTY_FUNCTION__))
;
1861 return Val.ConstVal;
1862 }
1863
1864 MachineConstantPoolValue *getMachineCPVal() const {
1865 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1865, __extension__ __PRETTY_FUNCTION__))
;
1866 return Val.MachineCPVal;
1867 }
1868
1869 int getOffset() const {
1870 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1871 }
1872
1873 // Return the alignment of this constant pool object, which is either 0 (for
1874 // default alignment) or the desired value.
1875 Align getAlign() const { return Alignment; }
1876 unsigned getTargetFlags() const { return TargetFlags; }
1877
1878 Type *getType() const;
1879
1880 static bool classof(const SDNode *N) {
1881 return N->getOpcode() == ISD::ConstantPool ||
1882 N->getOpcode() == ISD::TargetConstantPool;
1883 }
1884};
1885
1886/// Completely target-dependent object reference.
1887class TargetIndexSDNode : public SDNode {
1888 friend class SelectionDAG;
1889
1890 unsigned TargetFlags;
1891 int Index;
1892 int64_t Offset;
1893
1894public:
1895 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1896 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1897 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1898
1899 unsigned getTargetFlags() const { return TargetFlags; }
1900 int getIndex() const { return Index; }
1901 int64_t getOffset() const { return Offset; }
1902
1903 static bool classof(const SDNode *N) {
1904 return N->getOpcode() == ISD::TargetIndex;
1905 }
1906};
1907
1908class BasicBlockSDNode : public SDNode {
1909 friend class SelectionDAG;
1910
1911 MachineBasicBlock *MBB;
1912
1913 /// Debug info is meaningful and potentially useful here, but we create
1914 /// blocks out of order when they're jumped to, which makes it a bit
1915 /// harder. Let's see if we need it first.
1916 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1917 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1918 {}
1919
1920public:
1921 MachineBasicBlock *getBasicBlock() const { return MBB; }
1922
1923 static bool classof(const SDNode *N) {
1924 return N->getOpcode() == ISD::BasicBlock;
1925 }
1926};
1927
1928/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1929class BuildVectorSDNode : public SDNode {
1930public:
1931 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1932 explicit BuildVectorSDNode() = delete;
1933
1934 /// Check if this is a constant splat, and if so, find the
1935 /// smallest element size that splats the vector. If MinSplatBits is
1936 /// nonzero, the element size must be at least that large. Note that the
1937 /// splat element may be the entire vector (i.e., a one element vector).
1938 /// Returns the splat element value in SplatValue. Any undefined bits in
1939 /// that value are zero, and the corresponding bits in the SplatUndef mask
1940 /// are set. The SplatBitSize value is set to the splat element size in
1941 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1942 /// undefined. isBigEndian describes the endianness of the target.
1943 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1944 unsigned &SplatBitSize, bool &HasAnyUndefs,
1945 unsigned MinSplatBits = 0,
1946 bool isBigEndian = false) const;
1947
1948 /// Returns the demanded splatted value or a null value if this is not a
1949 /// splat.
1950 ///
1951 /// The DemandedElts mask indicates the elements that must be in the splat.
1952 /// If passed a non-null UndefElements bitvector, it will resize it to match
1953 /// the vector width and set the bits where elements are undef.
1954 SDValue getSplatValue(const APInt &DemandedElts,
1955 BitVector *UndefElements = nullptr) const;
1956
1957 /// Returns the splatted value or a null value if this is not a splat.
1958 ///
1959 /// If passed a non-null UndefElements bitvector, it will resize it to match
1960 /// the vector width and set the bits where elements are undef.
1961 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1962
1963 /// Find the shortest repeating sequence of values in the build vector.
1964 ///
1965 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1966 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1967 ///
1968 /// Currently this must be a power-of-2 build vector.
1969 /// The DemandedElts mask indicates the elements that must be present,
1970 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1971 /// non-null UndefElements bitvector, it will resize it to match the original
1972 /// vector width and set the bits where elements are undef. If result is
1973 /// false, Sequence will be empty.
1974 bool getRepeatedSequence(const APInt &DemandedElts,
1975 SmallVectorImpl<SDValue> &Sequence,
1976 BitVector *UndefElements = nullptr) const;
1977
1978 /// Find the shortest repeating sequence of values in the build vector.
1979 ///
1980 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1981 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1982 ///
1983 /// Currently this must be a power-of-2 build vector.
1984 /// If passed a non-null UndefElements bitvector, it will resize it to match
1985 /// the original vector width and set the bits where elements are undef.
1986 /// If result is false, Sequence will be empty.
1987 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
1988 BitVector *UndefElements = nullptr) const;
1989
1990 /// Returns the demanded splatted constant or null if this is not a constant
1991 /// splat.
1992 ///
1993 /// The DemandedElts mask indicates the elements that must be in the splat.
1994 /// If passed a non-null UndefElements bitvector, it will resize it to match
1995 /// the vector width and set the bits where elements are undef.
1996 ConstantSDNode *
1997 getConstantSplatNode(const APInt &DemandedElts,
1998 BitVector *UndefElements = nullptr) const;
1999
2000 /// Returns the splatted constant or null if this is not a constant
2001 /// splat.
2002 ///
2003 /// If passed a non-null UndefElements bitvector, it will resize it to match
2004 /// the vector width and set the bits where elements are undef.
2005 ConstantSDNode *
2006 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant FP or null if this is not a
2009 /// constant FP splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantFPSDNode *
2015 getConstantFPSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant FP or null if this is not a constant
2019 /// FP splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantFPSDNode *
2024 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// If this is a constant FP splat and the splatted constant FP is an
2027 /// exact power or 2, return the log base 2 integer value. Otherwise,
2028 /// return -1.
2029 ///
2030 /// The BitWidth specifies the necessary bit precision.
2031 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2032 uint32_t BitWidth) const;
2033
2034 bool isConstant() const;
2035
2036 static bool classof(const SDNode *N) {
2037 return N->getOpcode() == ISD::BUILD_VECTOR;
2038 }
2039};
2040
2041/// An SDNode that holds an arbitrary LLVM IR Value. This is
2042/// used when the SelectionDAG needs to make a simple reference to something
2043/// in the LLVM IR representation.
2044///
2045class SrcValueSDNode : public SDNode {
2046 friend class SelectionDAG;
2047
2048 const Value *V;
2049
2050 /// Create a SrcValue for a general value.
2051 explicit SrcValueSDNode(const Value *v)
2052 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2053
2054public:
2055 /// Return the contained Value.
2056 const Value *getValue() const { return V; }
2057
2058 static bool classof(const SDNode *N) {
2059 return N->getOpcode() == ISD::SRCVALUE;
2060 }
2061};
2062
2063class MDNodeSDNode : public SDNode {
2064 friend class SelectionDAG;
2065
2066 const MDNode *MD;
2067
2068 explicit MDNodeSDNode(const MDNode *md)
2069 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2070 {}
2071
2072public:
2073 const MDNode *getMD() const { return MD; }
2074
2075 static bool classof(const SDNode *N) {
2076 return N->getOpcode() == ISD::MDNODE_SDNODE;
2077 }
2078};
2079
2080class RegisterSDNode : public SDNode {
2081 friend class SelectionDAG;
2082
2083 Register Reg;
2084
2085 RegisterSDNode(Register reg, EVT VT)
2086 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2087
2088public:
2089 Register getReg() const { return Reg; }
2090
2091 static bool classof(const SDNode *N) {
2092 return N->getOpcode() == ISD::Register;
2093 }
2094};
2095
2096class RegisterMaskSDNode : public SDNode {
2097 friend class SelectionDAG;
2098
2099 // The memory for RegMask is not owned by the node.
2100 const uint32_t *RegMask;
2101
2102 RegisterMaskSDNode(const uint32_t *mask)
2103 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2104 RegMask(mask) {}
2105
2106public:
2107 const uint32_t *getRegMask() const { return RegMask; }
2108
2109 static bool classof(const SDNode *N) {
2110 return N->getOpcode() == ISD::RegisterMask;
2111 }
2112};
2113
2114class BlockAddressSDNode : public SDNode {
2115 friend class SelectionDAG;
2116
2117 const BlockAddress *BA;
2118 int64_t Offset;
2119 unsigned TargetFlags;
2120
2121 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2122 int64_t o, unsigned Flags)
2123 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2124 BA(ba), Offset(o), TargetFlags(Flags) {}
2125
2126public:
2127 const BlockAddress *getBlockAddress() const { return BA; }
2128 int64_t getOffset() const { return Offset; }
2129 unsigned getTargetFlags() const { return TargetFlags; }
2130
2131 static bool classof(const SDNode *N) {
2132 return N->getOpcode() == ISD::BlockAddress ||
2133 N->getOpcode() == ISD::TargetBlockAddress;
2134 }
2135};
2136
2137class LabelSDNode : public SDNode {
2138 friend class SelectionDAG;
2139
2140 MCSymbol *Label;
2141
2142 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2143 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2144 assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
"not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2144, __extension__ __PRETTY_FUNCTION__))
;
2145 }
2146
2147public:
2148 MCSymbol *getLabel() const { return Label; }
2149
2150 static bool classof(const SDNode *N) {
2151 return N->getOpcode() == ISD::EH_LABEL ||
2152 N->getOpcode() == ISD::ANNOTATION_LABEL;
2153 }
2154};
2155
2156class ExternalSymbolSDNode : public SDNode {
2157 friend class SelectionDAG;
2158
2159 const char *Symbol;
2160 unsigned TargetFlags;
2161
2162 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2163 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2164 DebugLoc(), getSDVTList(VT)),
2165 Symbol(Sym), TargetFlags(TF) {}
2166
2167public:
2168 const char *getSymbol() const { return Symbol; }
2169 unsigned getTargetFlags() const { return TargetFlags; }
2170
2171 static bool classof(const SDNode *N) {
2172 return N->getOpcode() == ISD::ExternalSymbol ||
2173 N->getOpcode() == ISD::TargetExternalSymbol;
2174 }
2175};
2176
2177class MCSymbolSDNode : public SDNode {
2178 friend class SelectionDAG;
2179
2180 MCSymbol *Symbol;
2181
2182 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2183 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2184
2185public:
2186 MCSymbol *getMCSymbol() const { return Symbol; }
2187
2188 static bool classof(const SDNode *N) {
2189 return N->getOpcode() == ISD::MCSymbol;
2190 }
2191};
2192
2193class CondCodeSDNode : public SDNode {
2194 friend class SelectionDAG;
2195
2196 ISD::CondCode Condition;
2197
2198 explicit CondCodeSDNode(ISD::CondCode Cond)
2199 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2200 Condition(Cond) {}
2201
2202public:
2203 ISD::CondCode get() const { return Condition; }
2204
2205 static bool classof(const SDNode *N) {
2206 return N->getOpcode() == ISD::CONDCODE;
2207 }
2208};
2209
2210/// This class is used to represent EVT's, which are used
2211/// to parameterize some operations.
2212class VTSDNode : public SDNode {
2213 friend class SelectionDAG;
2214
2215 EVT ValueType;
2216
2217 explicit VTSDNode(EVT VT)
2218 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2219 ValueType(VT) {}
2220
2221public:
2222 EVT getVT() const { return ValueType; }
2223
2224 static bool classof(const SDNode *N) {
2225 return N->getOpcode() == ISD::VALUETYPE;
2226 }
2227};
2228
2229/// Base class for LoadSDNode and StoreSDNode
2230class LSBaseSDNode : public MemSDNode {
2231public:
2232 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2233 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2234 MachineMemOperand *MMO)
2235 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2236 LSBaseSDNodeBits.AddressingMode = AM;
2237 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2237, __extension__ __PRETTY_FUNCTION__))
;
2238 }
2239
2240 const SDValue &getOffset() const {
2241 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2242 }
2243
2244 /// Return the addressing mode for this load or store:
2245 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2246 ISD::MemIndexedMode getAddressingMode() const {
2247 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2248 }
2249
2250 /// Return true if this is a pre/post inc/dec load/store.
2251 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2252
2253 /// Return true if this is NOT a pre/post inc/dec load/store.
2254 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2255
2256 static bool classof(const SDNode *N) {
2257 return N->getOpcode() == ISD::LOAD ||
2258 N->getOpcode() == ISD::STORE;
2259 }
2260};
2261
2262/// This class is used to represent ISD::LOAD nodes.
2263class LoadSDNode : public LSBaseSDNode {
2264 friend class SelectionDAG;
2265
2266 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2267 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2268 MachineMemOperand *MMO)
2269 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2270 LoadSDNodeBits.ExtTy = ETy;
2271 assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2271, __extension__ __PRETTY_FUNCTION__))
;
2272 assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2272, __extension__ __PRETTY_FUNCTION__))
;
2273 }
2274
2275public:
2276 /// Return whether this is a plain node,
2277 /// or one of the varieties of value-extending loads.
2278 ISD::LoadExtType getExtensionType() const {
2279 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2280 }
2281
2282 const SDValue &getBasePtr() const { return getOperand(1); }
2283 const SDValue &getOffset() const { return getOperand(2); }
2284
2285 static bool classof(const SDNode *N) {
2286 return N->getOpcode() == ISD::LOAD;
2287 }
2288};
2289
2290/// This class is used to represent ISD::STORE nodes.
2291class StoreSDNode : public LSBaseSDNode {
2292 friend class SelectionDAG;
2293
2294 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2295 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2296 MachineMemOperand *MMO)
2297 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2298 StoreSDNodeBits.IsTruncating = isTrunc;
2299 assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2299, __extension__ __PRETTY_FUNCTION__))
;
2300 assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2300, __extension__ __PRETTY_FUNCTION__))
;
2301 }
2302
2303public:
2304 /// Return true if the op does a truncation before store.
2305 /// For integers this is the same as doing a TRUNCATE and storing the result.
2306 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2307 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2308 void setTruncatingStore(bool Truncating) {
2309 StoreSDNodeBits.IsTruncating = Truncating;
2310 }
2311
2312 const SDValue &getValue() const { return getOperand(1); }
2313 const SDValue &getBasePtr() const { return getOperand(2); }
2314 const SDValue &getOffset() const { return getOperand(3); }
2315
2316 static bool classof(const SDNode *N) {
2317 return N->getOpcode() == ISD::STORE;
2318 }
2319};
2320
2321/// This base class is used to represent MLOAD and MSTORE nodes
2322class MaskedLoadStoreSDNode : public MemSDNode {
2323public:
2324 friend class SelectionDAG;
2325
2326 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2327 const DebugLoc &dl, SDVTList VTs,
2328 ISD::MemIndexedMode AM, EVT MemVT,
2329 MachineMemOperand *MMO)
2330 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2331 LSBaseSDNodeBits.AddressingMode = AM;
2332 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2332, __extension__ __PRETTY_FUNCTION__))
;
2333 }
2334
2335 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2336 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2337 // Mask is a vector of i1 elements
2338 const SDValue &getOffset() const {
2339 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2340 }
2341 const SDValue &getMask() const {
2342 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2343 }
2344
2345 /// Return the addressing mode for this load or store:
2346 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2347 ISD::MemIndexedMode getAddressingMode() const {
2348 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2349 }
2350
2351 /// Return true if this is a pre/post inc/dec load/store.
2352 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2353
2354 /// Return true if this is NOT a pre/post inc/dec load/store.
2355 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2356
2357 static bool classof(const SDNode *N) {
2358 return N->getOpcode() == ISD::MLOAD ||
2359 N->getOpcode() == ISD::MSTORE;
2360 }
2361};
2362
2363/// This class is used to represent an MLOAD node
2364class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2365public:
2366 friend class SelectionDAG;
2367
2368 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2369 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2370 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2371 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2372 LoadSDNodeBits.ExtTy = ETy;
2373 LoadSDNodeBits.IsExpanding = IsExpanding;
2374 }
2375
2376 ISD::LoadExtType getExtensionType() const {
2377 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2378 }
2379
2380 const SDValue &getBasePtr() const { return getOperand(1); }
2381 const SDValue &getOffset() const { return getOperand(2); }
2382 const SDValue &getMask() const { return getOperand(3); }
2383 const SDValue &getPassThru() const { return getOperand(4); }
2384
2385 static bool classof(const SDNode *N) {
2386 return N->getOpcode() == ISD::MLOAD;
2387 }
2388
2389 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2390};
2391
2392/// This class is used to represent an MSTORE node
2393class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2394public:
2395 friend class SelectionDAG;
2396
2397 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2398 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2399 EVT MemVT, MachineMemOperand *MMO)
2400 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2401 StoreSDNodeBits.IsTruncating = isTrunc;
2402 StoreSDNodeBits.IsCompressing = isCompressing;
2403 }
2404
2405 /// Return true if the op does a truncation before store.
2406 /// For integers this is the same as doing a TRUNCATE and storing the result.
2407 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2408 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2409
2410 /// Returns true if the op does a compression to the vector before storing.
2411 /// The node contiguously stores the active elements (integers or floats)
2412 /// in src (those with their respective bit set in writemask k) to unaligned
2413 /// memory at base_addr.
2414 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2415
2416 const SDValue &getValue() const { return getOperand(1); }
2417 const SDValue &getBasePtr() const { return getOperand(2); }
2418 const SDValue &getOffset() const { return getOperand(3); }
2419 const SDValue &getMask() const { return getOperand(4); }
2420
2421 static bool classof(const SDNode *N) {
2422 return N->getOpcode() == ISD::MSTORE;
2423 }
2424};
2425
2426/// This is a base class used to represent
2427/// MGATHER and MSCATTER nodes
2428///
2429class MaskedGatherScatterSDNode : public MemSDNode {
2430public:
2431 friend class SelectionDAG;
2432
2433 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2434 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2435 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2436 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2437 LSBaseSDNodeBits.AddressingMode = IndexType;
2438 assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
"Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2438, __extension__ __PRETTY_FUNCTION__))
;
2439 }
2440
2441 /// How is Index applied to BasePtr when computing addresses.
2442 ISD::MemIndexType getIndexType() const {
2443 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2444 }
2445 void setIndexType(ISD::MemIndexType IndexType) {
2446 LSBaseSDNodeBits.AddressingMode = IndexType;
2447 }
2448 bool isIndexScaled() const {
2449 return (getIndexType() == ISD::SIGNED_SCALED) ||
2450 (getIndexType() == ISD::UNSIGNED_SCALED);
2451 }
2452 bool isIndexSigned() const {
2453 return (getIndexType() == ISD::SIGNED_SCALED) ||
2454 (getIndexType() == ISD::SIGNED_UNSCALED);
2455 }
2456
2457 // In the both nodes address is Op1, mask is Op2:
2458 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2459 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2460 // Mask is a vector of i1 elements
2461 const SDValue &getBasePtr() const { return getOperand(3); }
2462 const SDValue &getIndex() const { return getOperand(4); }
2463 const SDValue &getMask() const { return getOperand(2); }
2464 const SDValue &getScale() const { return getOperand(5); }
2465
2466 static bool classof(const SDNode *N) {
2467 return N->getOpcode() == ISD::MGATHER ||
2468 N->getOpcode() == ISD::MSCATTER;
2469 }
2470};
2471
2472/// This class is used to represent an MGATHER node
2473///
2474class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2475public:
2476 friend class SelectionDAG;
2477
2478 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2479 EVT MemVT, MachineMemOperand *MMO,
2480 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2481 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2482 IndexType) {
2483 LoadSDNodeBits.ExtTy = ETy;
2484 }
2485
2486 const SDValue &getPassThru() const { return getOperand(1); }
2487
2488 ISD::LoadExtType getExtensionType() const {
2489 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2490 }
2491
2492 static bool classof(const SDNode *N) {
2493 return N->getOpcode() == ISD::MGATHER;
2494 }
2495};
2496
2497/// This class is used to represent an MSCATTER node
2498///
2499class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2500public:
2501 friend class SelectionDAG;
2502
2503 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2504 EVT MemVT, MachineMemOperand *MMO,
2505 ISD::MemIndexType IndexType, bool IsTrunc)
2506 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2507 IndexType) {
2508 StoreSDNodeBits.IsTruncating = IsTrunc;
2509 }
2510
2511 /// Return true if the op does a truncation before store.
2512 /// For integers this is the same as doing a TRUNCATE and storing the result.
2513 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2514 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2515
2516 const SDValue &getValue() const { return getOperand(1); }
2517
2518 static bool classof(const SDNode *N) {
2519 return N->getOpcode() == ISD::MSCATTER;
2520 }
2521};
2522
2523/// An SDNode that represents everything that will be needed
2524/// to construct a MachineInstr. These nodes are created during the
2525/// instruction selection proper phase.
2526///
2527/// Note that the only supported way to set the `memoperands` is by calling the
2528/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2529/// inside the DAG rather than in the node.
2530class MachineSDNode : public SDNode {
2531private:
2532 friend class SelectionDAG;
2533
2534 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2535 : SDNode(Opc, Order, DL, VTs) {}
2536
2537 // We use a pointer union between a single `MachineMemOperand` pointer and
2538 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2539 // the number of these is zero, the single pointer variant used when the
2540 // number is one, and the array is used for larger numbers.
2541 //
2542 // The array is allocated via the `SelectionDAG`'s allocator and so will
2543 // always live until the DAG is cleaned up and doesn't require ownership here.
2544 //
2545 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2546 // subclasses aren't managed in a conforming C++ manner. See the comments on
2547 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2548 // constraint here is that these don't manage memory with their constructor or
2549 // destructor and can be initialized to a good state even if they start off
2550 // uninitialized.
2551 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2552
2553 // Note that this could be folded into the above `MemRefs` member if doing so
2554 // is advantageous at some point. We don't need to store this in most cases.
2555 // However, at the moment this doesn't appear to make the allocation any
2556 // smaller and makes the code somewhat simpler to read.
2557 int NumMemRefs = 0;
2558
2559public:
2560 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2561
2562 ArrayRef<MachineMemOperand *> memoperands() const {
2563 // Special case the common cases.
2564 if (NumMemRefs == 0)
2565 return {};
2566 if (NumMemRefs == 1)
2567 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2568
2569 // Otherwise we have an actual array.
2570 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2571 }
2572 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2573 mmo_iterator memoperands_end() const { return memoperands().end(); }
2574 bool memoperands_empty() const { return memoperands().empty(); }
2575
2576 /// Clear out the memory reference descriptor list.
2577 void clearMemRefs() {
2578 MemRefs = nullptr;
2579 NumMemRefs = 0;
2580 }
2581
2582 static bool classof(const SDNode *N) {
2583 return N->isMachineOpcode();
2584 }
2585};
2586
2587/// An SDNode that records if a register contains a value that is guaranteed to
2588/// be aligned accordingly.
2589class AssertAlignSDNode : public SDNode {
2590 Align Alignment;
2591
2592public:
2593 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2594 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2595
2596 Align getAlign() const { return Alignment; }
2597
2598 static bool classof(const SDNode *N) {
2599 return N->getOpcode() == ISD::AssertAlign;
2600 }
2601};
2602
2603class SDNodeIterator {
2604 const SDNode *Node;
2605 unsigned Operand;
2606
2607 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2608
2609public:
2610 using iterator_category = std::forward_iterator_tag;
2611 using value_type = SDNode;
2612 using difference_type = std::ptrdiff_t;
2613 using pointer = value_type *;
2614 using reference = value_type &;
2615
2616 bool operator==(const SDNodeIterator& x) const {
2617 return Operand == x.Operand;
2618 }
2619 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2620
2621 pointer operator*() const {
2622 return Node->getOperand(Operand).getNode();
2623 }
2624 pointer operator->() const { return operator*(); }
2625
2626 SDNodeIterator& operator++() { // Preincrement
2627 ++Operand;
2628 return *this;
2629 }
2630 SDNodeIterator operator++(int) { // Postincrement
2631 SDNodeIterator tmp = *this; ++*this; return tmp;
2632 }
2633 size_t operator-(SDNodeIterator Other) const {
2634 assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2635, __extension__ __PRETTY_FUNCTION__))
2635 "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2635, __extension__ __PRETTY_FUNCTION__))
;
2636 return Operand - Other.Operand;
2637 }
2638
2639 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2640 static SDNodeIterator end (const SDNode *N) {
2641 return SDNodeIterator(N, N->getNumOperands());
2642 }
2643
2644 unsigned getOperand() const { return Operand; }
2645 const SDNode *getNode() const { return Node; }
2646};
2647
2648template <> struct GraphTraits<SDNode*> {
2649 using NodeRef = SDNode *;
2650 using ChildIteratorType = SDNodeIterator;
2651
2652 static NodeRef getEntryNode(SDNode *N) { return N; }
2653
2654 static ChildIteratorType child_begin(NodeRef N) {
2655 return SDNodeIterator::begin(N);
2656 }
2657
2658 static ChildIteratorType child_end(NodeRef N) {
2659 return SDNodeIterator::end(N);
2660 }
2661};
2662
2663/// A representation of the largest SDNode, for use in sizeof().
2664///
2665/// This needs to be a union because the largest node differs on 32 bit systems
2666/// with 4 and 8 byte pointer alignment, respectively.
2667using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
2668 BlockAddressSDNode,
2669 GlobalAddressSDNode,
2670 PseudoProbeSDNode>;
2671
2672/// The SDNode class with the greatest alignment requirement.
2673using MostAlignedSDNode = GlobalAddressSDNode;
2674
2675namespace ISD {
2676
2677 /// Returns true if the specified node is a non-extending and unindexed load.
2678 inline bool isNormalLoad(const SDNode *N) {
2679 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
2680 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
2681 Ld->getAddressingMode() == ISD::UNINDEXED;
2682 }
2683
2684 /// Returns true if the specified node is a non-extending load.
2685 inline bool isNON_EXTLoad(const SDNode *N) {
2686 return isa<LoadSDNode>(N) &&
2687 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
2688 }
2689
2690 /// Returns true if the specified node is a EXTLOAD.
2691 inline bool isEXTLoad(const SDNode *N) {
2692 return isa<LoadSDNode>(N) &&
2693 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
2694 }
2695
2696 /// Returns true if the specified node is a SEXTLOAD.
2697 inline bool isSEXTLoad(const SDNode *N) {
2698 return isa<LoadSDNode>(N) &&
2699 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
2700 }
2701
2702 /// Returns true if the specified node is a ZEXTLOAD.
2703 inline bool isZEXTLoad(const SDNode *N) {
2704 return isa<LoadSDNode>(N) &&
2705 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
2706 }
2707
2708 /// Returns true if the specified node is an unindexed load.
2709 inline bool isUNINDEXEDLoad(const SDNode *N) {
2710 return isa<LoadSDNode>(N) &&
2711 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2712 }
2713
2714 /// Returns true if the specified node is a non-truncating
2715 /// and unindexed store.
2716 inline bool isNormalStore(const SDNode *N) {
2717 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
2718 return St && !St->isTruncatingStore() &&
2719 St->getAddressingMode() == ISD::UNINDEXED;
2720 }
2721
2722 /// Returns true if the specified node is an unindexed store.
2723 inline bool isUNINDEXEDStore(const SDNode *N) {
2724 return isa<StoreSDNode>(N) &&
2725 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2726 }
2727
2728 /// Attempt to match a unary predicate against a scalar/splat constant or
2729 /// every element of a constant BUILD_VECTOR.
2730 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2731 bool matchUnaryPredicate(SDValue Op,
2732 std::function<bool(ConstantSDNode *)> Match,
2733 bool AllowUndefs = false);
2734
2735 /// Attempt to match a binary predicate against a pair of scalar/splat
2736 /// constants or every element of a pair of constant BUILD_VECTORs.
2737 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2738 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
2739 bool matchBinaryPredicate(
2740 SDValue LHS, SDValue RHS,
2741 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
2742 bool AllowUndefs = false, bool AllowTypeMismatch = false);
2743
2744 /// Returns true if the specified value is the overflow result from one
2745 /// of the overflow intrinsic nodes.
2746 inline bool isOverflowIntrOpRes(SDValue Op) {
2747 unsigned Opc = Op.getOpcode();
2748 return (Op.getResNo() == 1 &&
2749 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2750 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2751 }
2752
2753} // end namespace ISD
2754
2755} // end namespace llvm
2756
2757#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H