LLVM 20.0.0git
SelectionDAG.cpp
Go to the documentation of this file.
1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/Twine.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
64#include "llvm/Support/Debug.h"
68#include "llvm/Support/Mutex.h"
74#include <algorithm>
75#include <cassert>
76#include <cstdint>
77#include <cstdlib>
78#include <deque>
79#include <limits>
80#include <optional>
81#include <set>
82#include <string>
83#include <utility>
84#include <vector>
85
86using namespace llvm;
87using namespace llvm::SDPatternMatch;
88
89/// makeVTList - Return an instance of the SDVTList struct initialized with the
90/// specified members.
91static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
92 SDVTList Res = {VTs, NumVTs};
93 return Res;
94}
95
96// Default null implementations of the callbacks.
100
101void SelectionDAG::DAGNodeDeletedListener::anchor() {}
102void SelectionDAG::DAGNodeInsertedListener::anchor() {}
103
104#define DEBUG_TYPE "selectiondag"
105
106static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
107 cl::Hidden, cl::init(true),
108 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
109
110static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
111 cl::desc("Number limit for gluing ld/st of memcpy."),
112 cl::Hidden, cl::init(0));
113
115 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(8192),
116 cl::desc("DAG combiner limit number of steps when searching DAG "
117 "for predecessor nodes"));
118
120 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
121}
122
124
125//===----------------------------------------------------------------------===//
126// ConstantFPSDNode Class
127//===----------------------------------------------------------------------===//
128
129/// isExactlyValue - We don't rely on operator== working on double values, as
130/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
131/// As such, this method can be used to do an exact bit-for-bit comparison of
132/// two floating point values.
134 return getValueAPF().bitwiseIsEqual(V);
135}
136
138 const APFloat& Val) {
139 assert(VT.isFloatingPoint() && "Can only convert between FP types");
140
141 // convert modifies in place, so make a copy.
142 APFloat Val2 = APFloat(Val);
143 bool losesInfo;
145 &losesInfo);
146 return !losesInfo;
147}
148
149//===----------------------------------------------------------------------===//
150// ISD Namespace
151//===----------------------------------------------------------------------===//
152
153bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
154 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
155 unsigned EltSize =
156 N->getValueType(0).getVectorElementType().getSizeInBits();
157 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
158 SplatVal = Op0->getAPIntValue().trunc(EltSize);
159 return true;
160 }
161 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
162 SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize);
163 return true;
164 }
165 }
166
167 auto *BV = dyn_cast<BuildVectorSDNode>(N);
168 if (!BV)
169 return false;
170
171 APInt SplatUndef;
172 unsigned SplatBitSize;
173 bool HasUndefs;
174 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
175 // Endianness does not matter here. We are checking for a splat given the
176 // element size of the vector, and if we find such a splat for little endian
177 // layout, then that should be valid also for big endian (as the full vector
178 // size is known to be a multiple of the element size).
179 const bool IsBigEndian = false;
180 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
181 EltSize, IsBigEndian) &&
182 EltSize == SplatBitSize;
183}
184
185// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
186// specializations of the more general isConstantSplatVector()?
187
188bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
189 // Look through a bit convert.
190 while (N->getOpcode() == ISD::BITCAST)
191 N = N->getOperand(0).getNode();
192
193 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
194 APInt SplatVal;
195 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
196 }
197
198 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
199
200 unsigned i = 0, e = N->getNumOperands();
201
202 // Skip over all of the undef values.
203 while (i != e && N->getOperand(i).isUndef())
204 ++i;
205
206 // Do not accept an all-undef vector.
207 if (i == e) return false;
208
209 // Do not accept build_vectors that aren't all constants or which have non-~0
210 // elements. We have to be a bit careful here, as the type of the constant
211 // may not be the same as the type of the vector elements due to type
212 // legalization (the elements are promoted to a legal type for the target and
213 // a vector of a type may be legal when the base element type is not).
214 // We only want to check enough bits to cover the vector elements, because
215 // we care if the resultant vector is all ones, not whether the individual
216 // constants are.
217 SDValue NotZero = N->getOperand(i);
218 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
219 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
220 if (CN->getAPIntValue().countr_one() < EltSize)
221 return false;
222 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
223 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
224 return false;
225 } else
226 return false;
227
228 // Okay, we have at least one ~0 value, check to see if the rest match or are
229 // undefs. Even with the above element type twiddling, this should be OK, as
230 // the same type legalization should have applied to all the elements.
231 for (++i; i != e; ++i)
232 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
233 return false;
234 return true;
235}
236
237bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
238 // Look through a bit convert.
239 while (N->getOpcode() == ISD::BITCAST)
240 N = N->getOperand(0).getNode();
241
242 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
243 APInt SplatVal;
244 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
245 }
246
247 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
248
249 bool IsAllUndef = true;
250 for (const SDValue &Op : N->op_values()) {
251 if (Op.isUndef())
252 continue;
253 IsAllUndef = false;
254 // Do not accept build_vectors that aren't all constants or which have non-0
255 // elements. We have to be a bit careful here, as the type of the constant
256 // may not be the same as the type of the vector elements due to type
257 // legalization (the elements are promoted to a legal type for the target
258 // and a vector of a type may be legal when the base element type is not).
259 // We only want to check enough bits to cover the vector elements, because
260 // we care if the resultant vector is all zeros, not whether the individual
261 // constants are.
262 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
263 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
264 if (CN->getAPIntValue().countr_zero() < EltSize)
265 return false;
266 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
267 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
268 return false;
269 } else
270 return false;
271 }
272
273 // Do not accept an all-undef vector.
274 if (IsAllUndef)
275 return false;
276 return true;
277}
278
280 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
281}
282
284 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
285}
286
288 if (N->getOpcode() != ISD::BUILD_VECTOR)
289 return false;
290
291 for (const SDValue &Op : N->op_values()) {
292 if (Op.isUndef())
293 continue;
294 if (!isa<ConstantSDNode>(Op))
295 return false;
296 }
297 return true;
298}
299
301 if (N->getOpcode() != ISD::BUILD_VECTOR)
302 return false;
303
304 for (const SDValue &Op : N->op_values()) {
305 if (Op.isUndef())
306 continue;
307 if (!isa<ConstantFPSDNode>(Op))
308 return false;
309 }
310 return true;
311}
312
313bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
314 bool Signed) {
315 assert(N->getValueType(0).isVector() && "Expected a vector!");
316
317 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
318 if (EltSize <= NewEltSize)
319 return false;
320
321 if (N->getOpcode() == ISD::ZERO_EXTEND) {
322 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
323 NewEltSize) &&
324 !Signed;
325 }
326 if (N->getOpcode() == ISD::SIGN_EXTEND) {
327 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
328 NewEltSize) &&
329 Signed;
330 }
331 if (N->getOpcode() != ISD::BUILD_VECTOR)
332 return false;
333
334 for (const SDValue &Op : N->op_values()) {
335 if (Op.isUndef())
336 continue;
337 if (!isa<ConstantSDNode>(Op))
338 return false;
339
340 APInt C = Op->getAsAPIntVal().trunc(EltSize);
341 if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)
342 return false;
343 if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)
344 return false;
345 }
346
347 return true;
348}
349
351 // Return false if the node has no operands.
352 // This is "logically inconsistent" with the definition of "all" but
353 // is probably the desired behavior.
354 if (N->getNumOperands() == 0)
355 return false;
356 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
357}
358
360 return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();
361}
362
363template <typename ConstNodeType>
365 std::function<bool(ConstNodeType *)> Match,
366 bool AllowUndefs) {
367 // FIXME: Add support for scalar UNDEF cases?
368 if (auto *C = dyn_cast<ConstNodeType>(Op))
369 return Match(C);
370
371 // FIXME: Add support for vector UNDEF cases?
372 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
373 ISD::SPLAT_VECTOR != Op.getOpcode())
374 return false;
375
376 EVT SVT = Op.getValueType().getScalarType();
377 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
378 if (AllowUndefs && Op.getOperand(i).isUndef()) {
379 if (!Match(nullptr))
380 return false;
381 continue;
382 }
383
384 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
385 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
386 return false;
387 }
388 return true;
389}
390// Build used template types.
391template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
392 SDValue, std::function<bool(ConstantSDNode *)>, bool);
393template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
394 SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
395
397 SDValue LHS, SDValue RHS,
398 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
399 bool AllowUndefs, bool AllowTypeMismatch) {
400 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
401 return false;
402
403 // TODO: Add support for scalar UNDEF cases?
404 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
405 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
406 return Match(LHSCst, RHSCst);
407
408 // TODO: Add support for vector UNDEF cases?
409 if (LHS.getOpcode() != RHS.getOpcode() ||
410 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
411 LHS.getOpcode() != ISD::SPLAT_VECTOR))
412 return false;
413
414 EVT SVT = LHS.getValueType().getScalarType();
415 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
416 SDValue LHSOp = LHS.getOperand(i);
417 SDValue RHSOp = RHS.getOperand(i);
418 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
419 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
420 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
421 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
422 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
423 return false;
424 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
425 LHSOp.getValueType() != RHSOp.getValueType()))
426 return false;
427 if (!Match(LHSCst, RHSCst))
428 return false;
429 }
430 return true;
431}
432
434 switch (MinMaxOpc) {
435 default:
436 llvm_unreachable("unrecognized opcode");
437 case ISD::UMIN:
438 return ISD::UMAX;
439 case ISD::UMAX:
440 return ISD::UMIN;
441 case ISD::SMIN:
442 return ISD::SMAX;
443 case ISD::SMAX:
444 return ISD::SMIN;
445 }
446}
447
449 switch (VecReduceOpcode) {
450 default:
451 llvm_unreachable("Expected VECREDUCE opcode");
454 case ISD::VP_REDUCE_FADD:
455 case ISD::VP_REDUCE_SEQ_FADD:
456 return ISD::FADD;
459 case ISD::VP_REDUCE_FMUL:
460 case ISD::VP_REDUCE_SEQ_FMUL:
461 return ISD::FMUL;
463 case ISD::VP_REDUCE_ADD:
464 return ISD::ADD;
466 case ISD::VP_REDUCE_MUL:
467 return ISD::MUL;
469 case ISD::VP_REDUCE_AND:
470 return ISD::AND;
472 case ISD::VP_REDUCE_OR:
473 return ISD::OR;
475 case ISD::VP_REDUCE_XOR:
476 return ISD::XOR;
478 case ISD::VP_REDUCE_SMAX:
479 return ISD::SMAX;
481 case ISD::VP_REDUCE_SMIN:
482 return ISD::SMIN;
484 case ISD::VP_REDUCE_UMAX:
485 return ISD::UMAX;
487 case ISD::VP_REDUCE_UMIN:
488 return ISD::UMIN;
490 case ISD::VP_REDUCE_FMAX:
491 return ISD::FMAXNUM;
493 case ISD::VP_REDUCE_FMIN:
494 return ISD::FMINNUM;
496 case ISD::VP_REDUCE_FMAXIMUM:
497 return ISD::FMAXIMUM;
499 case ISD::VP_REDUCE_FMINIMUM:
500 return ISD::FMINIMUM;
501 }
502}
503
504bool ISD::isVPOpcode(unsigned Opcode) {
505 switch (Opcode) {
506 default:
507 return false;
508#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
509 case ISD::VPSD: \
510 return true;
511#include "llvm/IR/VPIntrinsics.def"
512 }
513}
514
515bool ISD::isVPBinaryOp(unsigned Opcode) {
516 switch (Opcode) {
517 default:
518 break;
519#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
520#define VP_PROPERTY_BINARYOP return true;
521#define END_REGISTER_VP_SDNODE(VPSD) break;
522#include "llvm/IR/VPIntrinsics.def"
523 }
524 return false;
525}
526
527bool ISD::isVPReduction(unsigned Opcode) {
528 switch (Opcode) {
529 default:
530 return false;
531 case ISD::VP_REDUCE_ADD:
532 case ISD::VP_REDUCE_MUL:
533 case ISD::VP_REDUCE_AND:
534 case ISD::VP_REDUCE_OR:
535 case ISD::VP_REDUCE_XOR:
536 case ISD::VP_REDUCE_SMAX:
537 case ISD::VP_REDUCE_SMIN:
538 case ISD::VP_REDUCE_UMAX:
539 case ISD::VP_REDUCE_UMIN:
540 case ISD::VP_REDUCE_FMAX:
541 case ISD::VP_REDUCE_FMIN:
542 case ISD::VP_REDUCE_FMAXIMUM:
543 case ISD::VP_REDUCE_FMINIMUM:
544 case ISD::VP_REDUCE_FADD:
545 case ISD::VP_REDUCE_FMUL:
546 case ISD::VP_REDUCE_SEQ_FADD:
547 case ISD::VP_REDUCE_SEQ_FMUL:
548 return true;
549 }
550}
551
552/// The operand position of the vector mask.
553std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
554 switch (Opcode) {
555 default:
556 return std::nullopt;
557#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
558 case ISD::VPSD: \
559 return MASKPOS;
560#include "llvm/IR/VPIntrinsics.def"
561 }
562}
563
564/// The operand position of the explicit vector length parameter.
565std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
566 switch (Opcode) {
567 default:
568 return std::nullopt;
569#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
570 case ISD::VPSD: \
571 return EVLPOS;
572#include "llvm/IR/VPIntrinsics.def"
573 }
574}
575
576std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
577 bool hasFPExcept) {
578 // FIXME: Return strict opcodes in case of fp exceptions.
579 switch (VPOpcode) {
580 default:
581 return std::nullopt;
582#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
583#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
584#define END_REGISTER_VP_SDNODE(VPOPC) break;
585#include "llvm/IR/VPIntrinsics.def"
586 }
587 return std::nullopt;
588}
589
590std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
591 switch (Opcode) {
592 default:
593 return std::nullopt;
594#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
595#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
596#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
597#include "llvm/IR/VPIntrinsics.def"
598 }
599}
600
602 switch (ExtType) {
603 case ISD::EXTLOAD:
604 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
605 case ISD::SEXTLOAD:
606 return ISD::SIGN_EXTEND;
607 case ISD::ZEXTLOAD:
608 return ISD::ZERO_EXTEND;
609 default:
610 break;
611 }
612
613 llvm_unreachable("Invalid LoadExtType");
614}
615
617 // To perform this operation, we just need to swap the L and G bits of the
618 // operation.
619 unsigned OldL = (Operation >> 2) & 1;
620 unsigned OldG = (Operation >> 1) & 1;
621 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
622 (OldL << 1) | // New G bit
623 (OldG << 2)); // New L bit.
624}
625
627 unsigned Operation = Op;
628 if (isIntegerLike)
629 Operation ^= 7; // Flip L, G, E bits, but not U.
630 else
631 Operation ^= 15; // Flip all of the condition bits.
632
634 Operation &= ~8; // Don't let N and U bits get set.
635
636 return ISD::CondCode(Operation);
637}
638
640 return getSetCCInverseImpl(Op, Type.isInteger());
641}
642
644 bool isIntegerLike) {
645 return getSetCCInverseImpl(Op, isIntegerLike);
646}
647
648/// For an integer comparison, return 1 if the comparison is a signed operation
649/// and 2 if the result is an unsigned comparison. Return zero if the operation
650/// does not depend on the sign of the input (setne and seteq).
651static int isSignedOp(ISD::CondCode Opcode) {
652 switch (Opcode) {
653 default: llvm_unreachable("Illegal integer setcc operation!");
654 case ISD::SETEQ:
655 case ISD::SETNE: return 0;
656 case ISD::SETLT:
657 case ISD::SETLE:
658 case ISD::SETGT:
659 case ISD::SETGE: return 1;
660 case ISD::SETULT:
661 case ISD::SETULE:
662 case ISD::SETUGT:
663 case ISD::SETUGE: return 2;
664 }
665}
666
668 EVT Type) {
669 bool IsInteger = Type.isInteger();
670 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
671 // Cannot fold a signed integer setcc with an unsigned integer setcc.
672 return ISD::SETCC_INVALID;
673
674 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
675
676 // If the N and U bits get set, then the resultant comparison DOES suddenly
677 // care about orderedness, and it is true when ordered.
678 if (Op > ISD::SETTRUE2)
679 Op &= ~16; // Clear the U bit if the N bit is set.
680
681 // Canonicalize illegal integer setcc's.
682 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
683 Op = ISD::SETNE;
684
685 return ISD::CondCode(Op);
686}
687
689 EVT Type) {
690 bool IsInteger = Type.isInteger();
691 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
692 // Cannot fold a signed setcc with an unsigned setcc.
693 return ISD::SETCC_INVALID;
694
695 // Combine all of the condition bits.
696 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
697
698 // Canonicalize illegal integer setcc's.
699 if (IsInteger) {
700 switch (Result) {
701 default: break;
702 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
703 case ISD::SETOEQ: // SETEQ & SETU[LG]E
704 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
705 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
706 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
707 }
708 }
709
710 return Result;
711}
712
713//===----------------------------------------------------------------------===//
714// SDNode Profile Support
715//===----------------------------------------------------------------------===//
716
717/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
718static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
719 ID.AddInteger(OpC);
720}
721
722/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
723/// solely with their pointer.
725 ID.AddPointer(VTList.VTs);
726}
727
728/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
730 ArrayRef<SDValue> Ops) {
731 for (const auto &Op : Ops) {
732 ID.AddPointer(Op.getNode());
733 ID.AddInteger(Op.getResNo());
734 }
735}
736
737/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
739 ArrayRef<SDUse> Ops) {
740 for (const auto &Op : Ops) {
741 ID.AddPointer(Op.getNode());
742 ID.AddInteger(Op.getResNo());
743 }
744}
745
746static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
747 SDVTList VTList, ArrayRef<SDValue> OpList) {
748 AddNodeIDOpcode(ID, OpC);
749 AddNodeIDValueTypes(ID, VTList);
750 AddNodeIDOperands(ID, OpList);
751}
752
753/// If this is an SDNode with special info, add this info to the NodeID data.
755 switch (N->getOpcode()) {
758 case ISD::MCSymbol:
759 llvm_unreachable("Should only be used on nodes with operands");
760 default: break; // Normal nodes don't need extra info.
762 case ISD::Constant: {
763 const ConstantSDNode *C = cast<ConstantSDNode>(N);
764 ID.AddPointer(C->getConstantIntValue());
765 ID.AddBoolean(C->isOpaque());
766 break;
767 }
769 case ISD::ConstantFP:
770 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
771 break;
776 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
777 ID.AddPointer(GA->getGlobal());
778 ID.AddInteger(GA->getOffset());
779 ID.AddInteger(GA->getTargetFlags());
780 break;
781 }
782 case ISD::BasicBlock:
783 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
784 break;
785 case ISD::Register:
786 ID.AddInteger(cast<RegisterSDNode>(N)->getReg().id());
787 break;
789 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
790 break;
791 case ISD::SRCVALUE:
792 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
793 break;
794 case ISD::FrameIndex:
796 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
797 break;
800 if (cast<LifetimeSDNode>(N)->hasOffset()) {
801 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
802 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
803 }
804 break;
806 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
807 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
808 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
809 break;
810 case ISD::JumpTable:
812 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
813 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
814 break;
817 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
818 ID.AddInteger(CP->getAlign().value());
819 ID.AddInteger(CP->getOffset());
820 if (CP->isMachineConstantPoolEntry())
821 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
822 else
823 ID.AddPointer(CP->getConstVal());
824 ID.AddInteger(CP->getTargetFlags());
825 break;
826 }
827 case ISD::TargetIndex: {
828 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
829 ID.AddInteger(TI->getIndex());
830 ID.AddInteger(TI->getOffset());
831 ID.AddInteger(TI->getTargetFlags());
832 break;
833 }
834 case ISD::LOAD: {
835 const LoadSDNode *LD = cast<LoadSDNode>(N);
836 ID.AddInteger(LD->getMemoryVT().getRawBits());
837 ID.AddInteger(LD->getRawSubclassData());
838 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
839 ID.AddInteger(LD->getMemOperand()->getFlags());
840 break;
841 }
842 case ISD::STORE: {
843 const StoreSDNode *ST = cast<StoreSDNode>(N);
844 ID.AddInteger(ST->getMemoryVT().getRawBits());
845 ID.AddInteger(ST->getRawSubclassData());
846 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
847 ID.AddInteger(ST->getMemOperand()->getFlags());
848 break;
849 }
850 case ISD::VP_LOAD: {
851 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
852 ID.AddInteger(ELD->getMemoryVT().getRawBits());
853 ID.AddInteger(ELD->getRawSubclassData());
854 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
855 ID.AddInteger(ELD->getMemOperand()->getFlags());
856 break;
857 }
858 case ISD::VP_STORE: {
859 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
860 ID.AddInteger(EST->getMemoryVT().getRawBits());
861 ID.AddInteger(EST->getRawSubclassData());
862 ID.AddInteger(EST->getPointerInfo().getAddrSpace());
863 ID.AddInteger(EST->getMemOperand()->getFlags());
864 break;
865 }
866 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
867 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(N);
868 ID.AddInteger(SLD->getMemoryVT().getRawBits());
869 ID.AddInteger(SLD->getRawSubclassData());
870 ID.AddInteger(SLD->getPointerInfo().getAddrSpace());
871 break;
872 }
873 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
874 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
875 ID.AddInteger(SST->getMemoryVT().getRawBits());
876 ID.AddInteger(SST->getRawSubclassData());
877 ID.AddInteger(SST->getPointerInfo().getAddrSpace());
878 break;
879 }
880 case ISD::VP_GATHER: {
881 const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);
882 ID.AddInteger(EG->getMemoryVT().getRawBits());
883 ID.AddInteger(EG->getRawSubclassData());
884 ID.AddInteger(EG->getPointerInfo().getAddrSpace());
885 ID.AddInteger(EG->getMemOperand()->getFlags());
886 break;
887 }
888 case ISD::VP_SCATTER: {
889 const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);
890 ID.AddInteger(ES->getMemoryVT().getRawBits());
891 ID.AddInteger(ES->getRawSubclassData());
892 ID.AddInteger(ES->getPointerInfo().getAddrSpace());
893 ID.AddInteger(ES->getMemOperand()->getFlags());
894 break;
895 }
896 case ISD::MLOAD: {
897 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
898 ID.AddInteger(MLD->getMemoryVT().getRawBits());
899 ID.AddInteger(MLD->getRawSubclassData());
900 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
901 ID.AddInteger(MLD->getMemOperand()->getFlags());
902 break;
903 }
904 case ISD::MSTORE: {
905 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
906 ID.AddInteger(MST->getMemoryVT().getRawBits());
907 ID.AddInteger(MST->getRawSubclassData());
908 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
909 ID.AddInteger(MST->getMemOperand()->getFlags());
910 break;
911 }
912 case ISD::MGATHER: {
913 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
914 ID.AddInteger(MG->getMemoryVT().getRawBits());
915 ID.AddInteger(MG->getRawSubclassData());
916 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
917 ID.AddInteger(MG->getMemOperand()->getFlags());
918 break;
919 }
920 case ISD::MSCATTER: {
921 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
922 ID.AddInteger(MS->getMemoryVT().getRawBits());
923 ID.AddInteger(MS->getRawSubclassData());
924 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
925 ID.AddInteger(MS->getMemOperand()->getFlags());
926 break;
927 }
930 case ISD::ATOMIC_SWAP:
942 case ISD::ATOMIC_LOAD:
943 case ISD::ATOMIC_STORE: {
944 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
945 ID.AddInteger(AT->getMemoryVT().getRawBits());
946 ID.AddInteger(AT->getRawSubclassData());
947 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
948 ID.AddInteger(AT->getMemOperand()->getFlags());
949 break;
950 }
951 case ISD::VECTOR_SHUFFLE: {
952 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
953 for (int M : Mask)
954 ID.AddInteger(M);
955 break;
956 }
958 case ISD::BlockAddress: {
959 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
960 ID.AddPointer(BA->getBlockAddress());
961 ID.AddInteger(BA->getOffset());
962 ID.AddInteger(BA->getTargetFlags());
963 break;
964 }
965 case ISD::AssertAlign:
966 ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());
967 break;
968 case ISD::PREFETCH:
971 // Handled by MemIntrinsicSDNode check after the switch.
972 break;
973 } // end switch (N->getOpcode())
974
975 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
976 // to check.
977 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {
978 ID.AddInteger(MN->getRawSubclassData());
979 ID.AddInteger(MN->getPointerInfo().getAddrSpace());
980 ID.AddInteger(MN->getMemOperand()->getFlags());
981 ID.AddInteger(MN->getMemoryVT().getRawBits());
982 }
983}
984
985/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
986/// data.
987static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
988 AddNodeIDOpcode(ID, N->getOpcode());
989 // Add the return value info.
990 AddNodeIDValueTypes(ID, N->getVTList());
991 // Add the operand info.
992 AddNodeIDOperands(ID, N->ops());
993
994 // Handle SDNode leafs with special info.
996}
997
998//===----------------------------------------------------------------------===//
999// SelectionDAG Class
1000//===----------------------------------------------------------------------===//
1001
1002/// doNotCSE - Return true if CSE should not be performed for this node.
1003static bool doNotCSE(SDNode *N) {
1004 if (N->getValueType(0) == MVT::Glue)
1005 return true; // Never CSE anything that produces a glue result.
1006
1007 switch (N->getOpcode()) {
1008 default: break;
1009 case ISD::HANDLENODE:
1010 case ISD::EH_LABEL:
1011 return true; // Never CSE these nodes.
1012 }
1013
1014 // Check that remaining values produced are not flags.
1015 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1016 if (N->getValueType(i) == MVT::Glue)
1017 return true; // Never CSE anything that produces a glue result.
1018
1019 return false;
1020}
1021
1022/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1023/// SelectionDAG.
1025 // Create a dummy node (which is not added to allnodes), that adds a reference
1026 // to the root node, preventing it from being deleted.
1027 HandleSDNode Dummy(getRoot());
1028
1029 SmallVector<SDNode*, 128> DeadNodes;
1030
1031 // Add all obviously-dead nodes to the DeadNodes worklist.
1032 for (SDNode &Node : allnodes())
1033 if (Node.use_empty())
1034 DeadNodes.push_back(&Node);
1035
1036 RemoveDeadNodes(DeadNodes);
1037
1038 // If the root changed (e.g. it was a dead load, update the root).
1039 setRoot(Dummy.getValue());
1040}
1041
1042/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1043/// given list, and any nodes that become unreachable as a result.
1045
1046 // Process the worklist, deleting the nodes and adding their uses to the
1047 // worklist.
1048 while (!DeadNodes.empty()) {
1049 SDNode *N = DeadNodes.pop_back_val();
1050 // Skip to next node if we've already managed to delete the node. This could
1051 // happen if replacing a node causes a node previously added to the node to
1052 // be deleted.
1053 if (N->getOpcode() == ISD::DELETED_NODE)
1054 continue;
1055
1056 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1057 DUL->NodeDeleted(N, nullptr);
1058
1059 // Take the node out of the appropriate CSE map.
1060 RemoveNodeFromCSEMaps(N);
1061
1062 // Next, brutally remove the operand list. This is safe to do, as there are
1063 // no cycles in the graph.
1064 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1065 SDUse &Use = *I++;
1066 SDNode *Operand = Use.getNode();
1067 Use.set(SDValue());
1068
1069 // Now that we removed this operand, see if there are no uses of it left.
1070 if (Operand->use_empty())
1071 DeadNodes.push_back(Operand);
1072 }
1073
1074 DeallocateNode(N);
1075 }
1076}
1077
1079 SmallVector<SDNode*, 16> DeadNodes(1, N);
1080
1081 // Create a dummy node that adds a reference to the root node, preventing
1082 // it from being deleted. (This matters if the root is an operand of the
1083 // dead node.)
1084 HandleSDNode Dummy(getRoot());
1085
1086 RemoveDeadNodes(DeadNodes);
1087}
1088
1090 // First take this out of the appropriate CSE map.
1091 RemoveNodeFromCSEMaps(N);
1092
1093 // Finally, remove uses due to operands of this node, remove from the
1094 // AllNodes list, and delete the node.
1095 DeleteNodeNotInCSEMaps(N);
1096}
1097
1098void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1099 assert(N->getIterator() != AllNodes.begin() &&
1100 "Cannot delete the entry node!");
1101 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1102
1103 // Drop all of the operands and decrement used node's use counts.
1104 N->DropOperands();
1105
1106 DeallocateNode(N);
1107}
1108
1109void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1110 assert(!(V->isVariadic() && isParameter));
1111 if (isParameter)
1112 ByvalParmDbgValues.push_back(V);
1113 else
1114 DbgValues.push_back(V);
1115 for (const SDNode *Node : V->getSDNodes())
1116 if (Node)
1117 DbgValMap[Node].push_back(V);
1118}
1119
1120void SDDbgInfo::erase(const SDNode *Node) {
1121 DbgValMapType::iterator I = DbgValMap.find(Node);
1122 if (I == DbgValMap.end())
1123 return;
1124 for (auto &Val: I->second)
1125 Val->setIsInvalidated();
1126 DbgValMap.erase(I);
1127}
1128
1129void SelectionDAG::DeallocateNode(SDNode *N) {
1130 // If we have operands, deallocate them.
1131 removeOperands(N);
1132
1133 NodeAllocator.Deallocate(AllNodes.remove(N));
1134
1135 // Set the opcode to DELETED_NODE to help catch bugs when node
1136 // memory is reallocated.
1137 // FIXME: There are places in SDag that have grown a dependency on the opcode
1138 // value in the released node.
1139 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1140 N->NodeType = ISD::DELETED_NODE;
1141
1142 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1143 // them and forget about that node.
1144 DbgInfo->erase(N);
1145
1146 // Invalidate extra info.
1147 SDEI.erase(N);
1148}
1149
1150#ifndef NDEBUG
1151/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1152static void VerifySDNode(SDNode *N, const TargetLowering *TLI) {
1153 switch (N->getOpcode()) {
1154 default:
1155 if (N->getOpcode() > ISD::BUILTIN_OP_END)
1156 TLI->verifyTargetSDNode(N);
1157 break;
1158 case ISD::BUILD_PAIR: {
1159 EVT VT = N->getValueType(0);
1160 assert(N->getNumValues() == 1 && "Too many results!");
1161 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1162 "Wrong return type!");
1163 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1164 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1165 "Mismatched operand types!");
1166 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1167 "Wrong operand type!");
1168 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1169 "Wrong return type size");
1170 break;
1171 }
1172 case ISD::BUILD_VECTOR: {
1173 assert(N->getNumValues() == 1 && "Too many results!");
1174 assert(N->getValueType(0).isVector() && "Wrong return type!");
1175 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1176 "Wrong number of operands!");
1177 EVT EltVT = N->getValueType(0).getVectorElementType();
1178 for (const SDUse &Op : N->ops()) {
1179 assert((Op.getValueType() == EltVT ||
1180 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1181 EltVT.bitsLE(Op.getValueType()))) &&
1182 "Wrong operand type!");
1183 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1184 "Operands must all have the same type");
1185 }
1186 break;
1187 }
1188 }
1189}
1190#endif // NDEBUG
1191
1192/// Insert a newly allocated node into the DAG.
1193///
1194/// Handles insertion into the all nodes list and CSE map, as well as
1195/// verification and other common operations when a new node is allocated.
1196void SelectionDAG::InsertNode(SDNode *N) {
1197 AllNodes.push_back(N);
1198#ifndef NDEBUG
1199 N->PersistentId = NextPersistentId++;
1200 VerifySDNode(N, TLI);
1201#endif
1202 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1203 DUL->NodeInserted(N);
1204}
1205
1206/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1207/// correspond to it. This is useful when we're about to delete or repurpose
1208/// the node. We don't want future request for structurally identical nodes
1209/// to return N anymore.
1210bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1211 bool Erased = false;
1212 switch (N->getOpcode()) {
1213 case ISD::HANDLENODE: return false; // noop.
1214 case ISD::CONDCODE:
1215 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1216 "Cond code doesn't exist!");
1217 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1218 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1219 break;
1221 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1222 break;
1224 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1225 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1226 ESN->getSymbol(), ESN->getTargetFlags()));
1227 break;
1228 }
1229 case ISD::MCSymbol: {
1230 auto *MCSN = cast<MCSymbolSDNode>(N);
1231 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1232 break;
1233 }
1234 case ISD::VALUETYPE: {
1235 EVT VT = cast<VTSDNode>(N)->getVT();
1236 if (VT.isExtended()) {
1237 Erased = ExtendedValueTypeNodes.erase(VT);
1238 } else {
1239 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1240 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1241 }
1242 break;
1243 }
1244 default:
1245 // Remove it from the CSE Map.
1246 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1247 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1248 Erased = CSEMap.RemoveNode(N);
1249 break;
1250 }
1251#ifndef NDEBUG
1252 // Verify that the node was actually in one of the CSE maps, unless it has a
1253 // glue result (which cannot be CSE'd) or is one of the special cases that are
1254 // not subject to CSE.
1255 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1256 !N->isMachineOpcode() && !doNotCSE(N)) {
1257 N->dump(this);
1258 dbgs() << "\n";
1259 llvm_unreachable("Node is not in map!");
1260 }
1261#endif
1262 return Erased;
1263}
1264
1265/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1266/// maps and modified in place. Add it back to the CSE maps, unless an identical
1267/// node already exists, in which case transfer all its users to the existing
1268/// node. This transfer can potentially trigger recursive merging.
1269void
1270SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1271 // For node types that aren't CSE'd, just act as if no identical node
1272 // already exists.
1273 if (!doNotCSE(N)) {
1274 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1275 if (Existing != N) {
1276 // If there was already an existing matching node, use ReplaceAllUsesWith
1277 // to replace the dead one with the existing one. This can cause
1278 // recursive merging of other unrelated nodes down the line.
1279 Existing->intersectFlagsWith(N->getFlags());
1280 ReplaceAllUsesWith(N, Existing);
1281
1282 // N is now dead. Inform the listeners and delete it.
1283 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1284 DUL->NodeDeleted(N, Existing);
1285 DeleteNodeNotInCSEMaps(N);
1286 return;
1287 }
1288 }
1289
1290 // If the node doesn't already exist, we updated it. Inform listeners.
1291 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1292 DUL->NodeUpdated(N);
1293}
1294
1295/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1296/// were replaced with those specified. If this node is never memoized,
1297/// return null, otherwise return a pointer to the slot it would take. If a
1298/// node already exists with these operands, the slot will be non-null.
1299SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1300 void *&InsertPos) {
1301 if (doNotCSE(N))
1302 return nullptr;
1303
1304 SDValue Ops[] = { Op };
1306 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1308 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1309 if (Node)
1310 Node->intersectFlagsWith(N->getFlags());
1311 return Node;
1312}
1313
1314/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1315/// were replaced with those specified. If this node is never memoized,
1316/// return null, otherwise return a pointer to the slot it would take. If a
1317/// node already exists with these operands, the slot will be non-null.
1318SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1319 SDValue Op1, SDValue Op2,
1320 void *&InsertPos) {
1321 if (doNotCSE(N))
1322 return nullptr;
1323
1324 SDValue Ops[] = { Op1, Op2 };
1326 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1328 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1329 if (Node)
1330 Node->intersectFlagsWith(N->getFlags());
1331 return Node;
1332}
1333
1334/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1335/// were replaced with those specified. If this node is never memoized,
1336/// return null, otherwise return a pointer to the slot it would take. If a
1337/// node already exists with these operands, the slot will be non-null.
1338SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1339 void *&InsertPos) {
1340 if (doNotCSE(N))
1341 return nullptr;
1342
1344 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1346 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1347 if (Node)
1348 Node->intersectFlagsWith(N->getFlags());
1349 return Node;
1350}
1351
1353 Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)
1354 : VT.getTypeForEVT(*getContext());
1355
1356 return getDataLayout().getABITypeAlign(Ty);
1357}
1358
1359// EntryNode could meaningfully have debug info if we can find it...
1361 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1362 getVTList(MVT::Other, MVT::Glue)),
1363 Root(getEntryNode()) {
1364 InsertNode(&EntryNode);
1365 DbgInfo = new SDDbgInfo();
1366}
1367
1369 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1370 const TargetLibraryInfo *LibraryInfo,
1371 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1373 FunctionVarLocs const *VarLocs) {
1374 MF = &NewMF;
1375 SDAGISelPass = PassPtr;
1376 ORE = &NewORE;
1379 LibInfo = LibraryInfo;
1380 Context = &MF->getFunction().getContext();
1381 UA = NewUA;
1382 PSI = PSIin;
1383 BFI = BFIin;
1384 MMI = &MMIin;
1385 FnVarLocs = VarLocs;
1386}
1387
1389 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1390 allnodes_clear();
1391 OperandRecycler.clear(OperandAllocator);
1392 delete DbgInfo;
1393}
1394
1396 return llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1397}
1398
1399void SelectionDAG::allnodes_clear() {
1400 assert(&*AllNodes.begin() == &EntryNode);
1401 AllNodes.remove(AllNodes.begin());
1402 while (!AllNodes.empty())
1403 DeallocateNode(&AllNodes.front());
1404#ifndef NDEBUG
1405 NextPersistentId = 0;
1406#endif
1407}
1408
1409SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1410 void *&InsertPos) {
1411 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1412 if (N) {
1413 switch (N->getOpcode()) {
1414 default: break;
1415 case ISD::Constant:
1416 case ISD::ConstantFP:
1417 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1418 "debug location. Use another overload.");
1419 }
1420 }
1421 return N;
1422}
1423
1424SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1425 const SDLoc &DL, void *&InsertPos) {
1426 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1427 if (N) {
1428 switch (N->getOpcode()) {
1429 case ISD::Constant:
1430 case ISD::ConstantFP:
1431 // Erase debug location from the node if the node is used at several
1432 // different places. Do not propagate one location to all uses as it
1433 // will cause a worse single stepping debugging experience.
1434 if (N->getDebugLoc() != DL.getDebugLoc())
1435 N->setDebugLoc(DebugLoc());
1436 break;
1437 default:
1438 // When the node's point of use is located earlier in the instruction
1439 // sequence than its prior point of use, update its debug info to the
1440 // earlier location.
1441 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1442 N->setDebugLoc(DL.getDebugLoc());
1443 break;
1444 }
1445 }
1446 return N;
1447}
1448
1450 allnodes_clear();
1451 OperandRecycler.clear(OperandAllocator);
1452 OperandAllocator.Reset();
1453 CSEMap.clear();
1454
1455 ExtendedValueTypeNodes.clear();
1456 ExternalSymbols.clear();
1457 TargetExternalSymbols.clear();
1458 MCSymbols.clear();
1459 SDEI.clear();
1460 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), nullptr);
1461 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), nullptr);
1462
1463 EntryNode.UseList = nullptr;
1464 InsertNode(&EntryNode);
1465 Root = getEntryNode();
1466 DbgInfo->clear();
1467}
1468
1470 return VT.bitsGT(Op.getValueType())
1471 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1472 : getNode(ISD::FP_ROUND, DL, VT, Op,
1473 getIntPtrConstant(0, DL, /*isTarget=*/true));
1474}
1475
1476std::pair<SDValue, SDValue>
1478 const SDLoc &DL, EVT VT) {
1479 assert(!VT.bitsEq(Op.getValueType()) &&
1480 "Strict no-op FP extend/round not allowed.");
1481 SDValue Res =
1482 VT.bitsGT(Op.getValueType())
1483 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1484 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1485 {Chain, Op, getIntPtrConstant(0, DL, /*isTarget=*/true)});
1486
1487 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1488}
1489
1491 return VT.bitsGT(Op.getValueType()) ?
1492 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1493 getNode(ISD::TRUNCATE, DL, VT, Op);
1494}
1495
1497 return VT.bitsGT(Op.getValueType()) ?
1498 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1499 getNode(ISD::TRUNCATE, DL, VT, Op);
1500}
1501
1503 return VT.bitsGT(Op.getValueType()) ?
1504 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1505 getNode(ISD::TRUNCATE, DL, VT, Op);
1506}
1507
1509 EVT VT) {
1510 assert(!VT.isVector());
1511 auto Type = Op.getValueType();
1512 SDValue DestOp;
1513 if (Type == VT)
1514 return Op;
1515 auto Size = Op.getValueSizeInBits();
1516 DestOp = getBitcast(EVT::getIntegerVT(*Context, Size), Op);
1517 if (DestOp.getValueType() == VT)
1518 return DestOp;
1519
1520 return getAnyExtOrTrunc(DestOp, DL, VT);
1521}
1522
1524 EVT VT) {
1525 assert(!VT.isVector());
1526 auto Type = Op.getValueType();
1527 SDValue DestOp;
1528 if (Type == VT)
1529 return Op;
1530 auto Size = Op.getValueSizeInBits();
1531 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1532 if (DestOp.getValueType() == VT)
1533 return DestOp;
1534
1535 return getSExtOrTrunc(DestOp, DL, VT);
1536}
1537
1539 EVT VT) {
1540 assert(!VT.isVector());
1541 auto Type = Op.getValueType();
1542 SDValue DestOp;
1543 if (Type == VT)
1544 return Op;
1545 auto Size = Op.getValueSizeInBits();
1546 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1547 if (DestOp.getValueType() == VT)
1548 return DestOp;
1549
1550 return getZExtOrTrunc(DestOp, DL, VT);
1551}
1552
1554 EVT OpVT) {
1555 if (VT.bitsLE(Op.getValueType()))
1556 return getNode(ISD::TRUNCATE, SL, VT, Op);
1557
1559 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1560}
1561
1563 EVT OpVT = Op.getValueType();
1564 assert(VT.isInteger() && OpVT.isInteger() &&
1565 "Cannot getZeroExtendInReg FP types");
1566 assert(VT.isVector() == OpVT.isVector() &&
1567 "getZeroExtendInReg type should be vector iff the operand "
1568 "type is vector!");
1569 assert((!VT.isVector() ||
1571 "Vector element counts must match in getZeroExtendInReg");
1572 assert(VT.bitsLE(OpVT) && "Not extending!");
1573 if (OpVT == VT)
1574 return Op;
1576 VT.getScalarSizeInBits());
1577 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1578}
1579
1581 SDValue EVL, const SDLoc &DL,
1582 EVT VT) {
1583 EVT OpVT = Op.getValueType();
1584 assert(VT.isInteger() && OpVT.isInteger() &&
1585 "Cannot getVPZeroExtendInReg FP types");
1586 assert(VT.isVector() && OpVT.isVector() &&
1587 "getVPZeroExtendInReg type and operand type should be vector!");
1589 "Vector element counts must match in getZeroExtendInReg");
1590 assert(VT.bitsLE(OpVT) && "Not extending!");
1591 if (OpVT == VT)
1592 return Op;
1594 VT.getScalarSizeInBits());
1595 return getNode(ISD::VP_AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT), Mask,
1596 EVL);
1597}
1598
1600 // Only unsigned pointer semantics are supported right now. In the future this
1601 // might delegate to TLI to check pointer signedness.
1602 return getZExtOrTrunc(Op, DL, VT);
1603}
1604
1606 // Only unsigned pointer semantics are supported right now. In the future this
1607 // might delegate to TLI to check pointer signedness.
1608 return getZeroExtendInReg(Op, DL, VT);
1609}
1610
1612 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);
1613}
1614
1615/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1617 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1618}
1619
1621 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1622 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1623}
1624
1626 SDValue Mask, SDValue EVL, EVT VT) {
1627 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1628 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL);
1629}
1630
1632 SDValue Mask, SDValue EVL) {
1633 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1634}
1635
1637 SDValue Mask, SDValue EVL) {
1638 if (VT.bitsGT(Op.getValueType()))
1639 return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL);
1640 if (VT.bitsLT(Op.getValueType()))
1641 return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL);
1642 return Op;
1643}
1644
1646 EVT OpVT) {
1647 if (!V)
1648 return getConstant(0, DL, VT);
1649
1650 switch (TLI->getBooleanContents(OpVT)) {
1653 return getConstant(1, DL, VT);
1655 return getAllOnesConstant(DL, VT);
1656 }
1657 llvm_unreachable("Unexpected boolean content enum!");
1658}
1659
1661 bool isT, bool isO) {
1662 return getConstant(APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1663 DL, VT, isT, isO);
1664}
1665
1667 bool isT, bool isO) {
1668 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1669}
1670
1672 EVT VT, bool isT, bool isO) {
1673 assert(VT.isInteger() && "Cannot create FP integer constant!");
1674
1675 EVT EltVT = VT.getScalarType();
1676 const ConstantInt *Elt = &Val;
1677
1678 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1679 // to-be-splatted scalar ConstantInt.
1680 if (isa<VectorType>(Elt->getType()))
1681 Elt = ConstantInt::get(*getContext(), Elt->getValue());
1682
1683 // In some cases the vector type is legal but the element type is illegal and
1684 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1685 // inserted value (the type does not need to match the vector element type).
1686 // Any extra bits introduced will be truncated away.
1687 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1689 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1690 APInt NewVal;
1691 if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))
1692 NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());
1693 else
1694 NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1695 Elt = ConstantInt::get(*getContext(), NewVal);
1696 }
1697 // In other cases the element type is illegal and needs to be expanded, for
1698 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1699 // the value into n parts and use a vector type with n-times the elements.
1700 // Then bitcast to the type requested.
1701 // Legalizing constants too early makes the DAGCombiner's job harder so we
1702 // only legalize if the DAG tells us we must produce legal types.
1703 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1704 TLI->getTypeAction(*getContext(), EltVT) ==
1706 const APInt &NewVal = Elt->getValue();
1707 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1708 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1709
1710 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1711 if (VT.isScalableVector() ||
1713 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1714 "Can only handle an even split!");
1715 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1716
1717 SmallVector<SDValue, 2> ScalarParts;
1718 for (unsigned i = 0; i != Parts; ++i)
1719 ScalarParts.push_back(getConstant(
1720 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1721 ViaEltVT, isT, isO));
1722
1723 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1724 }
1725
1726 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1727 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1728
1729 // Check the temporary vector is the correct size. If this fails then
1730 // getTypeToTransformTo() probably returned a type whose size (in bits)
1731 // isn't a power-of-2 factor of the requested type size.
1732 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1733
1734 SmallVector<SDValue, 2> EltParts;
1735 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1736 EltParts.push_back(getConstant(
1737 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1738 ViaEltVT, isT, isO));
1739
1740 // EltParts is currently in little endian order. If we actually want
1741 // big-endian order then reverse it now.
1742 if (getDataLayout().isBigEndian())
1743 std::reverse(EltParts.begin(), EltParts.end());
1744
1745 // The elements must be reversed when the element order is different
1746 // to the endianness of the elements (because the BITCAST is itself a
1747 // vector shuffle in this situation). However, we do not need any code to
1748 // perform this reversal because getConstant() is producing a vector
1749 // splat.
1750 // This situation occurs in MIPS MSA.
1751
1753 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1754 llvm::append_range(Ops, EltParts);
1755
1756 SDValue V =
1757 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1758 return V;
1759 }
1760
1761 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1762 "APInt size does not match type size!");
1763 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1764 SDVTList VTs = getVTList(EltVT);
1766 AddNodeIDNode(ID, Opc, VTs, {});
1767 ID.AddPointer(Elt);
1768 ID.AddBoolean(isO);
1769 void *IP = nullptr;
1770 SDNode *N = nullptr;
1771 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1772 if (!VT.isVector())
1773 return SDValue(N, 0);
1774
1775 if (!N) {
1776 N = newSDNode<ConstantSDNode>(isT, isO, Elt, VTs);
1777 CSEMap.InsertNode(N, IP);
1778 InsertNode(N);
1779 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1780 }
1781
1782 SDValue Result(N, 0);
1783 if (VT.isVector())
1784 Result = getSplat(VT, DL, Result);
1785 return Result;
1786}
1787
1789 bool isT, bool isO) {
1790 unsigned Size = VT.getScalarSizeInBits();
1791 return getConstant(APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1792}
1793
1795 bool IsOpaque) {
1797 IsTarget, IsOpaque);
1798}
1799
1801 bool isTarget) {
1802 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1803}
1804
1806 const SDLoc &DL) {
1807 assert(VT.isInteger() && "Shift amount is not an integer type!");
1808 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout());
1809 return getConstant(Val, DL, ShiftVT);
1810}
1811
1813 const SDLoc &DL) {
1814 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1815 return getShiftAmountConstant(Val.getZExtValue(), VT, DL);
1816}
1817
1819 bool isTarget) {
1820 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1821}
1822
1824 bool isTarget) {
1825 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1826}
1827
1829 EVT VT, bool isTarget) {
1830 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1831
1832 EVT EltVT = VT.getScalarType();
1833 const ConstantFP *Elt = &V;
1834
1835 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1836 // the to-be-splatted scalar ConstantFP.
1837 if (isa<VectorType>(Elt->getType()))
1838 Elt = ConstantFP::get(*getContext(), Elt->getValue());
1839
1840 // Do the map lookup using the actual bit pattern for the floating point
1841 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1842 // we don't have issues with SNANs.
1843 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1844 SDVTList VTs = getVTList(EltVT);
1846 AddNodeIDNode(ID, Opc, VTs, {});
1847 ID.AddPointer(Elt);
1848 void *IP = nullptr;
1849 SDNode *N = nullptr;
1850 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1851 if (!VT.isVector())
1852 return SDValue(N, 0);
1853
1854 if (!N) {
1855 N = newSDNode<ConstantFPSDNode>(isTarget, Elt, VTs);
1856 CSEMap.InsertNode(N, IP);
1857 InsertNode(N);
1858 }
1859
1860 SDValue Result(N, 0);
1861 if (VT.isVector())
1862 Result = getSplat(VT, DL, Result);
1863 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1864 return Result;
1865}
1866
1868 bool isTarget) {
1869 EVT EltVT = VT.getScalarType();
1870 if (EltVT == MVT::f32)
1871 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1872 if (EltVT == MVT::f64)
1873 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1874 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1875 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1876 bool Ignored;
1877 APFloat APF = APFloat(Val);
1879 &Ignored);
1880 return getConstantFP(APF, DL, VT, isTarget);
1881 }
1882 llvm_unreachable("Unsupported type in getConstantFP");
1883}
1884
1886 EVT VT, int64_t Offset, bool isTargetGA,
1887 unsigned TargetFlags) {
1888 assert((TargetFlags == 0 || isTargetGA) &&
1889 "Cannot set target flags on target-independent globals");
1890
1891 // Truncate (with sign-extension) the offset value to the pointer size.
1893 if (BitWidth < 64)
1895
1896 unsigned Opc;
1897 if (GV->isThreadLocal())
1899 else
1900 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1901
1902 SDVTList VTs = getVTList(VT);
1904 AddNodeIDNode(ID, Opc, VTs, {});
1905 ID.AddPointer(GV);
1906 ID.AddInteger(Offset);
1907 ID.AddInteger(TargetFlags);
1908 void *IP = nullptr;
1909 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1910 return SDValue(E, 0);
1911
1912 auto *N = newSDNode<GlobalAddressSDNode>(
1913 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VTs, Offset, TargetFlags);
1914 CSEMap.InsertNode(N, IP);
1915 InsertNode(N);
1916 return SDValue(N, 0);
1917}
1918
1919SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1920 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1921 SDVTList VTs = getVTList(VT);
1923 AddNodeIDNode(ID, Opc, VTs, {});
1924 ID.AddInteger(FI);
1925 void *IP = nullptr;
1926 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1927 return SDValue(E, 0);
1928
1929 auto *N = newSDNode<FrameIndexSDNode>(FI, VTs, isTarget);
1930 CSEMap.InsertNode(N, IP);
1931 InsertNode(N);
1932 return SDValue(N, 0);
1933}
1934
1935SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1936 unsigned TargetFlags) {
1937 assert((TargetFlags == 0 || isTarget) &&
1938 "Cannot set target flags on target-independent jump tables");
1939 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1940 SDVTList VTs = getVTList(VT);
1942 AddNodeIDNode(ID, Opc, VTs, {});
1943 ID.AddInteger(JTI);
1944 ID.AddInteger(TargetFlags);
1945 void *IP = nullptr;
1946 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1947 return SDValue(E, 0);
1948
1949 auto *N = newSDNode<JumpTableSDNode>(JTI, VTs, isTarget, TargetFlags);
1950 CSEMap.InsertNode(N, IP);
1951 InsertNode(N);
1952 return SDValue(N, 0);
1953}
1954
1956 const SDLoc &DL) {
1958 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1959 getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));
1960}
1961
1963 MaybeAlign Alignment, int Offset,
1964 bool isTarget, unsigned TargetFlags) {
1965 assert((TargetFlags == 0 || isTarget) &&
1966 "Cannot set target flags on target-independent globals");
1967 if (!Alignment)
1968 Alignment = shouldOptForSize()
1969 ? getDataLayout().getABITypeAlign(C->getType())
1970 : getDataLayout().getPrefTypeAlign(C->getType());
1971 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1972 SDVTList VTs = getVTList(VT);
1974 AddNodeIDNode(ID, Opc, VTs, {});
1975 ID.AddInteger(Alignment->value());
1976 ID.AddInteger(Offset);
1977 ID.AddPointer(C);
1978 ID.AddInteger(TargetFlags);
1979 void *IP = nullptr;
1980 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1981 return SDValue(E, 0);
1982
1983 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VTs, Offset, *Alignment,
1984 TargetFlags);
1985 CSEMap.InsertNode(N, IP);
1986 InsertNode(N);
1987 SDValue V = SDValue(N, 0);
1988 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1989 return V;
1990}
1991
1993 MaybeAlign Alignment, int Offset,
1994 bool isTarget, unsigned TargetFlags) {
1995 assert((TargetFlags == 0 || isTarget) &&
1996 "Cannot set target flags on target-independent globals");
1997 if (!Alignment)
1998 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1999 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2000 SDVTList VTs = getVTList(VT);
2002 AddNodeIDNode(ID, Opc, VTs, {});
2003 ID.AddInteger(Alignment->value());
2004 ID.AddInteger(Offset);
2005 C->addSelectionDAGCSEId(ID);
2006 ID.AddInteger(TargetFlags);
2007 void *IP = nullptr;
2008 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2009 return SDValue(E, 0);
2010
2011 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VTs, Offset, *Alignment,
2012 TargetFlags);
2013 CSEMap.InsertNode(N, IP);
2014 InsertNode(N);
2015 return SDValue(N, 0);
2016}
2017
2020 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), {});
2021 ID.AddPointer(MBB);
2022 void *IP = nullptr;
2023 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2024 return SDValue(E, 0);
2025
2026 auto *N = newSDNode<BasicBlockSDNode>(MBB);
2027 CSEMap.InsertNode(N, IP);
2028 InsertNode(N);
2029 return SDValue(N, 0);
2030}
2031
2033 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2034 ValueTypeNodes.size())
2035 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
2036
2037 SDNode *&N = VT.isExtended() ?
2038 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2039
2040 if (N) return SDValue(N, 0);
2041 N = newSDNode<VTSDNode>(VT);
2042 InsertNode(N);
2043 return SDValue(N, 0);
2044}
2045
2047 SDNode *&N = ExternalSymbols[Sym];
2048 if (N) return SDValue(N, 0);
2049 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, getVTList(VT));
2050 InsertNode(N);
2051 return SDValue(N, 0);
2052}
2053
2055 SDNode *&N = MCSymbols[Sym];
2056 if (N)
2057 return SDValue(N, 0);
2058 N = newSDNode<MCSymbolSDNode>(Sym, getVTList(VT));
2059 InsertNode(N);
2060 return SDValue(N, 0);
2061}
2062
2064 unsigned TargetFlags) {
2065 SDNode *&N =
2066 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2067 if (N) return SDValue(N, 0);
2068 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, getVTList(VT));
2069 InsertNode(N);
2070 return SDValue(N, 0);
2071}
2072
2074 if ((unsigned)Cond >= CondCodeNodes.size())
2075 CondCodeNodes.resize(Cond+1);
2076
2077 if (!CondCodeNodes[Cond]) {
2078 auto *N = newSDNode<CondCodeSDNode>(Cond);
2079 CondCodeNodes[Cond] = N;
2080 InsertNode(N);
2081 }
2082
2083 return SDValue(CondCodeNodes[Cond], 0);
2084}
2085
2087 bool ConstantFold) {
2088 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2089 "APInt size does not match type size!");
2090
2091 if (MulImm == 0)
2092 return getConstant(0, DL, VT);
2093
2094 if (ConstantFold) {
2095 const MachineFunction &MF = getMachineFunction();
2096 const Function &F = MF.getFunction();
2097 ConstantRange CR = getVScaleRange(&F, 64);
2098 if (const APInt *C = CR.getSingleElement())
2099 return getConstant(MulImm * C->getZExtValue(), DL, VT);
2100 }
2101
2102 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
2103}
2104
2106 bool ConstantFold) {
2107 if (EC.isScalable())
2108 return getVScale(DL, VT,
2109 APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2110
2111 return getConstant(EC.getKnownMinValue(), DL, VT);
2112}
2113
2115 APInt One(ResVT.getScalarSizeInBits(), 1);
2116 return getStepVector(DL, ResVT, One);
2117}
2118
2120 const APInt &StepVal) {
2121 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2122 if (ResVT.isScalableVector())
2123 return getNode(
2124 ISD::STEP_VECTOR, DL, ResVT,
2125 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
2126
2127 SmallVector<SDValue, 16> OpsStepConstants;
2128 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2129 OpsStepConstants.push_back(
2130 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
2131 return getBuildVector(ResVT, DL, OpsStepConstants);
2132}
2133
2134/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2135/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2137 std::swap(N1, N2);
2139}
2140
2142 SDValue N2, ArrayRef<int> Mask) {
2143 assert(VT.getVectorNumElements() == Mask.size() &&
2144 "Must have the same number of vector elements as mask elements!");
2145 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2146 "Invalid VECTOR_SHUFFLE");
2147
2148 // Canonicalize shuffle undef, undef -> undef
2149 if (N1.isUndef() && N2.isUndef())
2150 return getUNDEF(VT);
2151
2152 // Validate that all indices in Mask are within the range of the elements
2153 // input to the shuffle.
2154 int NElts = Mask.size();
2155 assert(llvm::all_of(Mask,
2156 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2157 "Index out of range");
2158
2159 // Copy the mask so we can do any needed cleanup.
2160 SmallVector<int, 8> MaskVec(Mask);
2161
2162 // Canonicalize shuffle v, v -> v, undef
2163 if (N1 == N2) {
2164 N2 = getUNDEF(VT);
2165 for (int i = 0; i != NElts; ++i)
2166 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2167 }
2168
2169 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2170 if (N1.isUndef())
2171 commuteShuffle(N1, N2, MaskVec);
2172
2173 if (TLI->hasVectorBlend()) {
2174 // If shuffling a splat, try to blend the splat instead. We do this here so
2175 // that even when this arises during lowering we don't have to re-handle it.
2176 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2177 BitVector UndefElements;
2178 SDValue Splat = BV->getSplatValue(&UndefElements);
2179 if (!Splat)
2180 return;
2181
2182 for (int i = 0; i < NElts; ++i) {
2183 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2184 continue;
2185
2186 // If this input comes from undef, mark it as such.
2187 if (UndefElements[MaskVec[i] - Offset]) {
2188 MaskVec[i] = -1;
2189 continue;
2190 }
2191
2192 // If we can blend a non-undef lane, use that instead.
2193 if (!UndefElements[i])
2194 MaskVec[i] = i + Offset;
2195 }
2196 };
2197 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2198 BlendSplat(N1BV, 0);
2199 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2200 BlendSplat(N2BV, NElts);
2201 }
2202
2203 // Canonicalize all index into lhs, -> shuffle lhs, undef
2204 // Canonicalize all index into rhs, -> shuffle rhs, undef
2205 bool AllLHS = true, AllRHS = true;
2206 bool N2Undef = N2.isUndef();
2207 for (int i = 0; i != NElts; ++i) {
2208 if (MaskVec[i] >= NElts) {
2209 if (N2Undef)
2210 MaskVec[i] = -1;
2211 else
2212 AllLHS = false;
2213 } else if (MaskVec[i] >= 0) {
2214 AllRHS = false;
2215 }
2216 }
2217 if (AllLHS && AllRHS)
2218 return getUNDEF(VT);
2219 if (AllLHS && !N2Undef)
2220 N2 = getUNDEF(VT);
2221 if (AllRHS) {
2222 N1 = getUNDEF(VT);
2223 commuteShuffle(N1, N2, MaskVec);
2224 }
2225 // Reset our undef status after accounting for the mask.
2226 N2Undef = N2.isUndef();
2227 // Re-check whether both sides ended up undef.
2228 if (N1.isUndef() && N2Undef)
2229 return getUNDEF(VT);
2230
2231 // If Identity shuffle return that node.
2232 bool Identity = true, AllSame = true;
2233 for (int i = 0; i != NElts; ++i) {
2234 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2235 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2236 }
2237 if (Identity && NElts)
2238 return N1;
2239
2240 // Shuffling a constant splat doesn't change the result.
2241 if (N2Undef) {
2242 SDValue V = N1;
2243
2244 // Look through any bitcasts. We check that these don't change the number
2245 // (and size) of elements and just changes their types.
2246 while (V.getOpcode() == ISD::BITCAST)
2247 V = V->getOperand(0);
2248
2249 // A splat should always show up as a build vector node.
2250 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2251 BitVector UndefElements;
2252 SDValue Splat = BV->getSplatValue(&UndefElements);
2253 // If this is a splat of an undef, shuffling it is also undef.
2254 if (Splat && Splat.isUndef())
2255 return getUNDEF(VT);
2256
2257 bool SameNumElts =
2258 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2259
2260 // We only have a splat which can skip shuffles if there is a splatted
2261 // value and no undef lanes rearranged by the shuffle.
2262 if (Splat && UndefElements.none()) {
2263 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2264 // number of elements match or the value splatted is a zero constant.
2265 if (SameNumElts || isNullConstant(Splat))
2266 return N1;
2267 }
2268
2269 // If the shuffle itself creates a splat, build the vector directly.
2270 if (AllSame && SameNumElts) {
2271 EVT BuildVT = BV->getValueType(0);
2272 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
2273 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
2274
2275 // We may have jumped through bitcasts, so the type of the
2276 // BUILD_VECTOR may not match the type of the shuffle.
2277 if (BuildVT != VT)
2278 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
2279 return NewBV;
2280 }
2281 }
2282 }
2283
2284 SDVTList VTs = getVTList(VT);
2286 SDValue Ops[2] = { N1, N2 };
2288 for (int i = 0; i != NElts; ++i)
2289 ID.AddInteger(MaskVec[i]);
2290
2291 void* IP = nullptr;
2292 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2293 return SDValue(E, 0);
2294
2295 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2296 // SDNode doesn't have access to it. This memory will be "leaked" when
2297 // the node is deallocated, but recovered when the NodeAllocator is released.
2298 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
2299 llvm::copy(MaskVec, MaskAlloc);
2300
2301 auto *N = newSDNode<ShuffleVectorSDNode>(VTs, dl.getIROrder(),
2302 dl.getDebugLoc(), MaskAlloc);
2303 createOperands(N, Ops);
2304
2305 CSEMap.InsertNode(N, IP);
2306 InsertNode(N);
2307 SDValue V = SDValue(N, 0);
2308 NewSDValueDbgMsg(V, "Creating new node: ", this);
2309 return V;
2310}
2311
2313 EVT VT = SV.getValueType(0);
2314 SmallVector<int, 8> MaskVec(SV.getMask());
2316
2317 SDValue Op0 = SV.getOperand(0);
2318 SDValue Op1 = SV.getOperand(1);
2319 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
2320}
2321
2323 SDVTList VTs = getVTList(VT);
2325 AddNodeIDNode(ID, ISD::Register, VTs, {});
2326 ID.AddInteger(Reg.id());
2327 void *IP = nullptr;
2328 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2329 return SDValue(E, 0);
2330
2331 auto *N = newSDNode<RegisterSDNode>(Reg, VTs);
2332 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2333 CSEMap.InsertNode(N, IP);
2334 InsertNode(N);
2335 return SDValue(N, 0);
2336}
2337
2340 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), {});
2341 ID.AddPointer(RegMask);
2342 void *IP = nullptr;
2343 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2344 return SDValue(E, 0);
2345
2346 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2347 CSEMap.InsertNode(N, IP);
2348 InsertNode(N);
2349 return SDValue(N, 0);
2350}
2351
2353 MCSymbol *Label) {
2354 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2355}
2356
2357SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2358 SDValue Root, MCSymbol *Label) {
2360 SDValue Ops[] = { Root };
2361 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2362 ID.AddPointer(Label);
2363 void *IP = nullptr;
2364 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2365 return SDValue(E, 0);
2366
2367 auto *N =
2368 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2369 createOperands(N, Ops);
2370
2371 CSEMap.InsertNode(N, IP);
2372 InsertNode(N);
2373 return SDValue(N, 0);
2374}
2375
2377 int64_t Offset, bool isTarget,
2378 unsigned TargetFlags) {
2379 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2380 SDVTList VTs = getVTList(VT);
2381
2383 AddNodeIDNode(ID, Opc, VTs, {});
2384 ID.AddPointer(BA);
2385 ID.AddInteger(Offset);
2386 ID.AddInteger(TargetFlags);
2387 void *IP = nullptr;
2388 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2389 return SDValue(E, 0);
2390
2391 auto *N = newSDNode<BlockAddressSDNode>(Opc, VTs, BA, Offset, TargetFlags);
2392 CSEMap.InsertNode(N, IP);
2393 InsertNode(N);
2394 return SDValue(N, 0);
2395}
2396
2399 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), {});
2400 ID.AddPointer(V);
2401
2402 void *IP = nullptr;
2403 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2404 return SDValue(E, 0);
2405
2406 auto *N = newSDNode<SrcValueSDNode>(V);
2407 CSEMap.InsertNode(N, IP);
2408 InsertNode(N);
2409 return SDValue(N, 0);
2410}
2411
2414 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), {});
2415 ID.AddPointer(MD);
2416
2417 void *IP = nullptr;
2418 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2419 return SDValue(E, 0);
2420
2421 auto *N = newSDNode<MDNodeSDNode>(MD);
2422 CSEMap.InsertNode(N, IP);
2423 InsertNode(N);
2424 return SDValue(N, 0);
2425}
2426
2428 if (VT == V.getValueType())
2429 return V;
2430
2431 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2432}
2433
2435 unsigned SrcAS, unsigned DestAS) {
2436 SDVTList VTs = getVTList(VT);
2437 SDValue Ops[] = {Ptr};
2440 ID.AddInteger(SrcAS);
2441 ID.AddInteger(DestAS);
2442
2443 void *IP = nullptr;
2444 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2445 return SDValue(E, 0);
2446
2447 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2448 VTs, SrcAS, DestAS);
2449 createOperands(N, Ops);
2450
2451 CSEMap.InsertNode(N, IP);
2452 InsertNode(N);
2453 return SDValue(N, 0);
2454}
2455
2457 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2458}
2459
2460/// getShiftAmountOperand - Return the specified value casted to
2461/// the target's desired shift amount type.
2463 EVT OpTy = Op.getValueType();
2464 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2465 if (OpTy == ShTy || OpTy.isVector()) return Op;
2466
2467 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2468}
2469
2471 SDValue Op2) {
2472 EVT FullTy = Op2.getValueType();
2473
2474 unsigned Stride = ReducedTy.getVectorMinNumElements();
2475 unsigned ScaleFactor = FullTy.getVectorMinNumElements() / Stride;
2476
2477 // Collect all of the subvectors
2478 std::deque<SDValue> Subvectors = {Op1};
2479 for (unsigned I = 0; I < ScaleFactor; I++) {
2480 auto SourceIndex = getVectorIdxConstant(I * Stride, DL);
2481 Subvectors.push_back(
2482 getNode(ISD::EXTRACT_SUBVECTOR, DL, ReducedTy, {Op2, SourceIndex}));
2483 }
2484
2485 // Flatten the subvector tree
2486 while (Subvectors.size() > 1) {
2487 Subvectors.push_back(
2488 getNode(ISD::ADD, DL, ReducedTy, {Subvectors[0], Subvectors[1]}));
2489 Subvectors.pop_front();
2490 Subvectors.pop_front();
2491 }
2492
2493 assert(Subvectors.size() == 1 &&
2494 "There should only be one subvector after tree flattening");
2495
2496 return Subvectors[0];
2497}
2498
2499/// Given a store node \p StoreNode, return true if it is safe to fold that node
2500/// into \p FPNode, which expands to a library call with output pointers.
2502 SDNode *FPNode) {
2504 SmallVector<const SDNode *, 8> DeferredNodes;
2506
2507 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
2508 for (SDValue Op : StoreNode->ops())
2509 if (Op.getNode() != FPNode)
2510 Worklist.push_back(Op.getNode());
2511
2513 while (!Worklist.empty()) {
2514 const SDNode *Node = Worklist.pop_back_val();
2515 auto [_, Inserted] = Visited.insert(Node);
2516 if (!Inserted)
2517 continue;
2518
2519 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
2520 return false;
2521
2522 // Reached the FPNode (would result in a cycle).
2523 // OR Reached CALLSEQ_START (would result in nested call sequences).
2524 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
2525 return false;
2526
2527 if (Node->getOpcode() == ISD::CALLSEQ_END) {
2528 // Defer looking into call sequences (so we can check we're outside one).
2529 // We still need to look through these for the predecessor check.
2530 DeferredNodes.push_back(Node);
2531 continue;
2532 }
2533
2534 for (SDValue Op : Node->ops())
2535 Worklist.push_back(Op.getNode());
2536 }
2537
2538 // True if we're outside a call sequence and don't have the FPNode as a
2539 // predecessor. No cycles or nested call sequences possible.
2540 return !SDNode::hasPredecessorHelper(FPNode, Visited, DeferredNodes,
2541 MaxSteps);
2542}
2543
2546 std::optional<unsigned> CallRetResNo) {
2547 LLVMContext &Ctx = *getContext();
2548 EVT VT = Node->getValueType(0);
2549 unsigned NumResults = Node->getNumValues();
2550
2551 const char *LCName = TLI->getLibcallName(LC);
2552 if (!LC || !LCName)
2553 return false;
2554
2555 auto getVecDesc = [&]() -> VecDesc const * {
2556 for (bool Masked : {false, true}) {
2557 if (VecDesc const *VD = getLibInfo().getVectorMappingInfo(
2558 LCName, VT.getVectorElementCount(), Masked)) {
2559 return VD;
2560 }
2561 }
2562 return nullptr;
2563 };
2564
2565 // For vector types, we must find a vector mapping for the libcall.
2566 VecDesc const *VD = nullptr;
2567 if (VT.isVector() && !(VD = getVecDesc()))
2568 return false;
2569
2570 // Find users of the node that store the results (and share input chains). The
2571 // destination pointers can be used instead of creating stack allocations.
2572 SDValue StoresInChain;
2573 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
2574 for (SDNode *User : Node->users()) {
2576 continue;
2577 auto *ST = cast<StoreSDNode>(User);
2578 SDValue StoreValue = ST->getValue();
2579 unsigned ResNo = StoreValue.getResNo();
2580 // Ensure the store corresponds to an output pointer.
2581 if (CallRetResNo == ResNo)
2582 continue;
2583 // Ensure the store to the default address space and not atomic or volatile.
2584 if (!ST->isSimple() || ST->getAddressSpace() != 0)
2585 continue;
2586 // Ensure all store chains are the same (so they don't alias).
2587 if (StoresInChain && ST->getChain() != StoresInChain)
2588 continue;
2589 // Ensure the store is properly aligned.
2590 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Ctx);
2591 if (ST->getAlign() <
2592 getDataLayout().getABITypeAlign(StoreType->getScalarType()))
2593 continue;
2594 // Avoid:
2595 // 1. Creating cyclic dependencies.
2596 // 2. Expanding the node to a call within a call sequence.
2598 continue;
2599 ResultStores[ResNo] = ST;
2600 StoresInChain = ST->getChain();
2601 }
2602
2604 auto AddArgListEntry = [&](SDValue Node, Type *Ty) {
2606 Entry.Ty = Ty;
2607 Entry.Node = Node;
2608 Args.push_back(Entry);
2609 };
2610
2611 // Pass the arguments.
2612 for (const SDValue &Op : Node->op_values()) {
2613 EVT ArgVT = Op.getValueType();
2614 Type *ArgTy = ArgVT.getTypeForEVT(Ctx);
2615 AddArgListEntry(Op, ArgTy);
2616 }
2617
2618 // Pass the output pointers.
2619 SmallVector<SDValue, 2> ResultPtrs(NumResults);
2621 for (auto [ResNo, ST] : llvm::enumerate(ResultStores)) {
2622 if (ResNo == CallRetResNo)
2623 continue;
2624 EVT ResVT = Node->getValueType(ResNo);
2625 SDValue ResultPtr = ST ? ST->getBasePtr() : CreateStackTemporary(ResVT);
2626 ResultPtrs[ResNo] = ResultPtr;
2627 AddArgListEntry(ResultPtr, PointerTy);
2628 }
2629
2630 SDLoc DL(Node);
2631
2632 // Pass the vector mask (if required).
2633 if (VD && VD->isMasked()) {
2634 EVT MaskVT = TLI->getSetCCResultType(getDataLayout(), Ctx, VT);
2635 SDValue Mask = getBoolConstant(true, DL, MaskVT, VT);
2636 AddArgListEntry(Mask, MaskVT.getTypeForEVT(Ctx));
2637 }
2638
2639 Type *RetType = CallRetResNo.has_value()
2640 ? Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
2641 : Type::getVoidTy(Ctx);
2642 SDValue InChain = StoresInChain ? StoresInChain : getEntryNode();
2643 SDValue Callee = getExternalSymbol(VD ? VD->getVectorFnName().data() : LCName,
2644 TLI->getPointerTy(getDataLayout()));
2646 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
2647 TLI->getLibcallCallingConv(LC), RetType, Callee, std::move(Args));
2648
2649 auto [Call, CallChain] = TLI->LowerCallTo(CLI);
2650
2651 for (auto [ResNo, ResultPtr] : llvm::enumerate(ResultPtrs)) {
2652 if (ResNo == CallRetResNo) {
2653 Results.push_back(Call);
2654 continue;
2655 }
2656 MachinePointerInfo PtrInfo;
2657 if (StoreSDNode *ST = ResultStores[ResNo]) {
2658 // Replace store with the library call.
2659 ReplaceAllUsesOfValueWith(SDValue(ST, 0), CallChain);
2660 PtrInfo = ST->getPointerInfo();
2661 } else {
2663 getMachineFunction(), cast<FrameIndexSDNode>(ResultPtr)->getIndex());
2664 }
2665 SDValue LoadResult =
2666 getLoad(Node->getValueType(ResNo), DL, CallChain, ResultPtr, PtrInfo);
2667 Results.push_back(LoadResult);
2668 }
2669
2670 if (CallRetResNo && !Node->hasAnyUseOfValue(*CallRetResNo)) {
2671 // FIXME: Find a way to avoid updating the root. This is needed for x86,
2672 // which uses a floating-point stack. If (for example) the node to be
2673 // expanded has two results one floating-point which is returned by the
2674 // call, and one integer result, returned via an output pointer. If only the
2675 // integer result is used then the `CopyFromReg` for the FP result may be
2676 // optimized out. This prevents an FP stack pop from being emitted for it.
2677 // Setting the root like this ensures there will be a use of the
2678 // `CopyFromReg` chain, and ensures the FP pop will be emitted.
2679 SDValue NewRoot =
2680 getNode(ISD::TokenFactor, DL, MVT::Other, getRoot(), CallChain);
2681 setRoot(NewRoot);
2682 // Ensure the new root is reachable from the results.
2683 Results[0] = getMergeValues({Results[0], NewRoot}, DL);
2684 }
2685
2686 return true;
2687}
2688
2690 SDLoc dl(Node);
2692 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2693 EVT VT = Node->getValueType(0);
2694 SDValue Tmp1 = Node->getOperand(0);
2695 SDValue Tmp2 = Node->getOperand(1);
2696 const MaybeAlign MA(Node->getConstantOperandVal(3));
2697
2698 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2699 Tmp2, MachinePointerInfo(V));
2700 SDValue VAList = VAListLoad;
2701
2702 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2703 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2704 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2705
2706 VAList = getNode(
2707 ISD::AND, dl, VAList.getValueType(), VAList,
2708 getSignedConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2709 }
2710
2711 // Increment the pointer, VAList, to the next vaarg
2712 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2713 getConstant(getDataLayout().getTypeAllocSize(
2714 VT.getTypeForEVT(*getContext())),
2715 dl, VAList.getValueType()));
2716 // Store the incremented VAList to the legalized pointer
2717 Tmp1 =
2718 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2719 // Load the actual argument out of the pointer VAList
2720 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2721}
2722
2724 SDLoc dl(Node);
2726 // This defaults to loading a pointer from the input and storing it to the
2727 // output, returning the chain.
2728 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2729 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2730 SDValue Tmp1 =
2731 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2732 Node->getOperand(2), MachinePointerInfo(VS));
2733 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2734 MachinePointerInfo(VD));
2735}
2736
2738 const DataLayout &DL = getDataLayout();
2739 Type *Ty = VT.getTypeForEVT(*getContext());
2740 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2741
2742 if (TLI->isTypeLegal(VT) || !VT.isVector())
2743 return RedAlign;
2744
2746 const Align StackAlign = TFI->getStackAlign();
2747
2748 // See if we can choose a smaller ABI alignment in cases where it's an
2749 // illegal vector type that will get broken down.
2750 if (RedAlign > StackAlign) {
2751 EVT IntermediateVT;
2752 MVT RegisterVT;
2753 unsigned NumIntermediates;
2754 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2755 NumIntermediates, RegisterVT);
2756 Ty = IntermediateVT.getTypeForEVT(*getContext());
2757 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2758 if (RedAlign2 < RedAlign)
2759 RedAlign = RedAlign2;
2760
2761 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2762 // If the stack is not realignable, the alignment should be limited to the
2763 // StackAlignment
2764 RedAlign = std::min(RedAlign, StackAlign);
2765 }
2766
2767 return RedAlign;
2768}
2769
2771 MachineFrameInfo &MFI = MF->getFrameInfo();
2773 int StackID = 0;
2774 if (Bytes.isScalable())
2775 StackID = TFI->getStackIDForScalableVectors();
2776 // The stack id gives an indication of whether the object is scalable or
2777 // not, so it's safe to pass in the minimum size here.
2778 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,
2779 false, nullptr, StackID);
2780 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2781}
2782
2784 Type *Ty = VT.getTypeForEVT(*getContext());
2785 Align StackAlign =
2786 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2787 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2788}
2789
2791 TypeSize VT1Size = VT1.getStoreSize();
2792 TypeSize VT2Size = VT2.getStoreSize();
2793 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2794 "Don't know how to choose the maximum size when creating a stack "
2795 "temporary");
2796 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2797 ? VT1Size
2798 : VT2Size;
2799
2800 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2801 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2802 const DataLayout &DL = getDataLayout();
2803 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2804 return CreateStackTemporary(Bytes, Align);
2805}
2806
2808 ISD::CondCode Cond, const SDLoc &dl) {
2809 EVT OpVT = N1.getValueType();
2810
2811 auto GetUndefBooleanConstant = [&]() {
2812 if (VT.getScalarType() == MVT::i1 ||
2813 TLI->getBooleanContents(OpVT) ==
2815 return getUNDEF(VT);
2816 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2817 // so we cannot use getUNDEF(). Return zero instead.
2818 return getConstant(0, dl, VT);
2819 };
2820
2821 // These setcc operations always fold.
2822 switch (Cond) {
2823 default: break;
2824 case ISD::SETFALSE:
2825 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2826 case ISD::SETTRUE:
2827 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2828
2829 case ISD::SETOEQ:
2830 case ISD::SETOGT:
2831 case ISD::SETOGE:
2832 case ISD::SETOLT:
2833 case ISD::SETOLE:
2834 case ISD::SETONE:
2835 case ISD::SETO:
2836 case ISD::SETUO:
2837 case ISD::SETUEQ:
2838 case ISD::SETUNE:
2839 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2840 break;
2841 }
2842
2843 if (OpVT.isInteger()) {
2844 // For EQ and NE, we can always pick a value for the undef to make the
2845 // predicate pass or fail, so we can return undef.
2846 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2847 // icmp eq/ne X, undef -> undef.
2848 if ((N1.isUndef() || N2.isUndef()) &&
2849 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2850 return GetUndefBooleanConstant();
2851
2852 // If both operands are undef, we can return undef for int comparison.
2853 // icmp undef, undef -> undef.
2854 if (N1.isUndef() && N2.isUndef())
2855 return GetUndefBooleanConstant();
2856
2857 // icmp X, X -> true/false
2858 // icmp X, undef -> true/false because undef could be X.
2859 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2860 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2861 }
2862
2863 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2864 const APInt &C2 = N2C->getAPIntValue();
2865 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2866 const APInt &C1 = N1C->getAPIntValue();
2867
2869 dl, VT, OpVT);
2870 }
2871 }
2872
2873 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2874 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2875
2876 if (N1CFP && N2CFP) {
2877 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2878 switch (Cond) {
2879 default: break;
2880 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2881 return GetUndefBooleanConstant();
2882 [[fallthrough]];
2883 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2884 OpVT);
2885 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2886 return GetUndefBooleanConstant();
2887 [[fallthrough]];
2889 R==APFloat::cmpLessThan, dl, VT,
2890 OpVT);
2891 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2892 return GetUndefBooleanConstant();
2893 [[fallthrough]];
2894 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2895 OpVT);
2896 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2897 return GetUndefBooleanConstant();
2898 [[fallthrough]];
2900 VT, OpVT);
2901 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2902 return GetUndefBooleanConstant();
2903 [[fallthrough]];
2905 R==APFloat::cmpEqual, dl, VT,
2906 OpVT);
2907 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2908 return GetUndefBooleanConstant();
2909 [[fallthrough]];
2911 R==APFloat::cmpEqual, dl, VT, OpVT);
2912 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2913 OpVT);
2914 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2915 OpVT);
2917 R==APFloat::cmpEqual, dl, VT,
2918 OpVT);
2919 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2920 OpVT);
2922 R==APFloat::cmpLessThan, dl, VT,
2923 OpVT);
2925 R==APFloat::cmpUnordered, dl, VT,
2926 OpVT);
2928 VT, OpVT);
2929 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2930 OpVT);
2931 }
2932 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2933 // Ensure that the constant occurs on the RHS.
2935 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2936 return SDValue();
2937 return getSetCC(dl, VT, N2, N1, SwappedCond);
2938 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2939 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2940 // If an operand is known to be a nan (or undef that could be a nan), we can
2941 // fold it.
2942 // Choosing NaN for the undef will always make unordered comparison succeed
2943 // and ordered comparison fails.
2944 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2945 switch (ISD::getUnorderedFlavor(Cond)) {
2946 default:
2947 llvm_unreachable("Unknown flavor!");
2948 case 0: // Known false.
2949 return getBoolConstant(false, dl, VT, OpVT);
2950 case 1: // Known true.
2951 return getBoolConstant(true, dl, VT, OpVT);
2952 case 2: // Undefined.
2953 return GetUndefBooleanConstant();
2954 }
2955 }
2956
2957 // Could not fold it.
2958 return SDValue();
2959}
2960
2961/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2962/// use this predicate to simplify operations downstream.
2964 unsigned BitWidth = Op.getScalarValueSizeInBits();
2966}
2967
2968/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2969/// this predicate to simplify operations downstream. Mask is known to be zero
2970/// for bits that V cannot have.
2972 unsigned Depth) const {
2973 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2974}
2975
2976/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2977/// DemandedElts. We use this predicate to simplify operations downstream.
2978/// Mask is known to be zero for bits that V cannot have.
2980 const APInt &DemandedElts,
2981 unsigned Depth) const {
2982 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2983}
2984
2985/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2986/// DemandedElts. We use this predicate to simplify operations downstream.
2988 unsigned Depth /* = 0 */) const {
2989 return computeKnownBits(V, DemandedElts, Depth).isZero();
2990}
2991
2992/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2994 unsigned Depth) const {
2995 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2996}
2997
2999 const APInt &DemandedElts,
3000 unsigned Depth) const {
3001 EVT VT = Op.getValueType();
3002 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
3003
3004 unsigned NumElts = VT.getVectorNumElements();
3005 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
3006
3007 APInt KnownZeroElements = APInt::getZero(NumElts);
3008 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
3009 if (!DemandedElts[EltIdx])
3010 continue; // Don't query elements that are not demanded.
3011 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
3012 if (MaskedVectorIsZero(Op, Mask, Depth))
3013 KnownZeroElements.setBit(EltIdx);
3014 }
3015 return KnownZeroElements;
3016}
3017
3018/// isSplatValue - Return true if the vector V has the same value
3019/// across all DemandedElts. For scalable vectors, we don't know the
3020/// number of lanes at compile time. Instead, we use a 1 bit APInt
3021/// to represent a conservative value for all lanes; that is, that
3022/// one bit value is implicitly splatted across all lanes.
3023bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
3024 APInt &UndefElts, unsigned Depth) const {
3025 unsigned Opcode = V.getOpcode();
3026 EVT VT = V.getValueType();
3027 assert(VT.isVector() && "Vector type expected");
3028 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
3029 "scalable demanded bits are ignored");
3030
3031 if (!DemandedElts)
3032 return false; // No demanded elts, better to assume we don't know anything.
3033
3034 if (Depth >= MaxRecursionDepth)
3035 return false; // Limit search depth.
3036
3037 // Deal with some common cases here that work for both fixed and scalable
3038 // vector types.
3039 switch (Opcode) {
3040 case ISD::SPLAT_VECTOR:
3041 UndefElts = V.getOperand(0).isUndef()
3042 ? APInt::getAllOnes(DemandedElts.getBitWidth())
3043 : APInt(DemandedElts.getBitWidth(), 0);
3044 return true;
3045 case ISD::ADD:
3046 case ISD::SUB:
3047 case ISD::AND:
3048 case ISD::XOR:
3049 case ISD::OR: {
3050 APInt UndefLHS, UndefRHS;
3051 SDValue LHS = V.getOperand(0);
3052 SDValue RHS = V.getOperand(1);
3053 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
3054 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
3055 UndefElts = UndefLHS | UndefRHS;
3056 return true;
3057 }
3058 return false;
3059 }
3060 case ISD::ABS:
3061 case ISD::TRUNCATE:
3062 case ISD::SIGN_EXTEND:
3063 case ISD::ZERO_EXTEND:
3064 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
3065 default:
3066 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3067 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3068 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,
3069 Depth);
3070 break;
3071}
3072
3073 // We don't support other cases than those above for scalable vectors at
3074 // the moment.
3075 if (VT.isScalableVector())
3076 return false;
3077
3078 unsigned NumElts = VT.getVectorNumElements();
3079 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3080 UndefElts = APInt::getZero(NumElts);
3081
3082 switch (Opcode) {
3083 case ISD::BUILD_VECTOR: {
3084 SDValue Scl;
3085 for (unsigned i = 0; i != NumElts; ++i) {
3086 SDValue Op = V.getOperand(i);
3087 if (Op.isUndef()) {
3088 UndefElts.setBit(i);
3089 continue;
3090 }
3091 if (!DemandedElts[i])
3092 continue;
3093 if (Scl && Scl != Op)
3094 return false;
3095 Scl = Op;
3096 }
3097 return true;
3098 }
3099 case ISD::VECTOR_SHUFFLE: {
3100 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3101 APInt DemandedLHS = APInt::getZero(NumElts);
3102 APInt DemandedRHS = APInt::getZero(NumElts);
3103 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
3104 for (int i = 0; i != (int)NumElts; ++i) {
3105 int M = Mask[i];
3106 if (M < 0) {
3107 UndefElts.setBit(i);
3108 continue;
3109 }
3110 if (!DemandedElts[i])
3111 continue;
3112 if (M < (int)NumElts)
3113 DemandedLHS.setBit(M);
3114 else
3115 DemandedRHS.setBit(M - NumElts);
3116 }
3117
3118 // If we aren't demanding either op, assume there's no splat.
3119 // If we are demanding both ops, assume there's no splat.
3120 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3121 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3122 return false;
3123
3124 // See if the demanded elts of the source op is a splat or we only demand
3125 // one element, which should always be a splat.
3126 // TODO: Handle source ops splats with undefs.
3127 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3128 APInt SrcUndefs;
3129 return (SrcElts.popcount() == 1) ||
3130 (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&
3131 (SrcElts & SrcUndefs).isZero());
3132 };
3133 if (!DemandedLHS.isZero())
3134 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
3135 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
3136 }
3138 // Offset the demanded elts by the subvector index.
3139 SDValue Src = V.getOperand(0);
3140 // We don't support scalable vectors at the moment.
3141 if (Src.getValueType().isScalableVector())
3142 return false;
3143 uint64_t Idx = V.getConstantOperandVal(1);
3144 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3145 APInt UndefSrcElts;
3146 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3147 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
3148 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
3149 return true;
3150 }
3151 break;
3152 }
3156 // Widen the demanded elts by the src element count.
3157 SDValue Src = V.getOperand(0);
3158 // We don't support scalable vectors at the moment.
3159 if (Src.getValueType().isScalableVector())
3160 return false;
3161 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3162 APInt UndefSrcElts;
3163 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
3164 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
3165 UndefElts = UndefSrcElts.trunc(NumElts);
3166 return true;
3167 }
3168 break;
3169 }
3170 case ISD::BITCAST: {
3171 SDValue Src = V.getOperand(0);
3172 EVT SrcVT = Src.getValueType();
3173 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3174 unsigned BitWidth = VT.getScalarSizeInBits();
3175
3176 // Ignore bitcasts from unsupported types.
3177 // TODO: Add fp support?
3178 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3179 break;
3180
3181 // Bitcast 'small element' vector to 'large element' vector.
3182 if ((BitWidth % SrcBitWidth) == 0) {
3183 // See if each sub element is a splat.
3184 unsigned Scale = BitWidth / SrcBitWidth;
3185 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3186 APInt ScaledDemandedElts =
3187 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3188 for (unsigned I = 0; I != Scale; ++I) {
3189 APInt SubUndefElts;
3190 APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);
3191 APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);
3192 SubDemandedElts &= ScaledDemandedElts;
3193 if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))
3194 return false;
3195 // TODO: Add support for merging sub undef elements.
3196 if (!SubUndefElts.isZero())
3197 return false;
3198 }
3199 return true;
3200 }
3201 break;
3202 }
3203 }
3204
3205 return false;
3206}
3207
3208/// Helper wrapper to main isSplatValue function.
3209bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3210 EVT VT = V.getValueType();
3211 assert(VT.isVector() && "Vector type expected");
3212
3213 APInt UndefElts;
3214 // Since the number of lanes in a scalable vector is unknown at compile time,
3215 // we track one bit which is implicitly broadcast to all lanes. This means
3216 // that all lanes in a scalable vector are considered demanded.
3217 APInt DemandedElts
3219 return isSplatValue(V, DemandedElts, UndefElts) &&
3220 (AllowUndefs || !UndefElts);
3221}
3222
3225
3226 EVT VT = V.getValueType();
3227 unsigned Opcode = V.getOpcode();
3228 switch (Opcode) {
3229 default: {
3230 APInt UndefElts;
3231 // Since the number of lanes in a scalable vector is unknown at compile time,
3232 // we track one bit which is implicitly broadcast to all lanes. This means
3233 // that all lanes in a scalable vector are considered demanded.
3234 APInt DemandedElts
3236
3237 if (isSplatValue(V, DemandedElts, UndefElts)) {
3238 if (VT.isScalableVector()) {
3239 // DemandedElts and UndefElts are ignored for scalable vectors, since
3240 // the only supported cases are SPLAT_VECTOR nodes.
3241 SplatIdx = 0;
3242 } else {
3243 // Handle case where all demanded elements are UNDEF.
3244 if (DemandedElts.isSubsetOf(UndefElts)) {
3245 SplatIdx = 0;
3246 return getUNDEF(VT);
3247 }
3248 SplatIdx = (UndefElts & DemandedElts).countr_one();
3249 }
3250 return V;
3251 }
3252 break;
3253 }
3254 case ISD::SPLAT_VECTOR:
3255 SplatIdx = 0;
3256 return V;
3257 case ISD::VECTOR_SHUFFLE: {
3258 assert(!VT.isScalableVector());
3259 // Check if this is a shuffle node doing a splat.
3260 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3261 // getTargetVShiftNode currently struggles without the splat source.
3262 auto *SVN = cast<ShuffleVectorSDNode>(V);
3263 if (!SVN->isSplat())
3264 break;
3265 int Idx = SVN->getSplatIndex();
3266 int NumElts = V.getValueType().getVectorNumElements();
3267 SplatIdx = Idx % NumElts;
3268 return V.getOperand(Idx / NumElts);
3269 }
3270 }
3271
3272 return SDValue();
3273}
3274
3276 int SplatIdx;
3277 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3278 EVT SVT = SrcVector.getValueType().getScalarType();
3279 EVT LegalSVT = SVT;
3280 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
3281 if (!SVT.isInteger())
3282 return SDValue();
3283 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
3284 if (LegalSVT.bitsLT(SVT))
3285 return SDValue();
3286 }
3287 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
3288 getVectorIdxConstant(SplatIdx, SDLoc(V)));
3289 }
3290 return SDValue();
3291}
3292
3293std::optional<ConstantRange>
3295 unsigned Depth) const {
3296 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3297 V.getOpcode() == ISD::SRA) &&
3298 "Unknown shift node");
3299 // Shifting more than the bitwidth is not valid.
3300 unsigned BitWidth = V.getScalarValueSizeInBits();
3301
3302 if (auto *Cst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
3303 const APInt &ShAmt = Cst->getAPIntValue();
3304 if (ShAmt.uge(BitWidth))
3305 return std::nullopt;
3306 return ConstantRange(ShAmt);
3307 }
3308
3309 if (auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1))) {
3310 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3311 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3312 if (!DemandedElts[i])
3313 continue;
3314 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3315 if (!SA) {
3316 MinAmt = MaxAmt = nullptr;
3317 break;
3318 }
3319 const APInt &ShAmt = SA->getAPIntValue();
3320 if (ShAmt.uge(BitWidth))
3321 return std::nullopt;
3322 if (!MinAmt || MinAmt->ugt(ShAmt))
3323 MinAmt = &ShAmt;
3324 if (!MaxAmt || MaxAmt->ult(ShAmt))
3325 MaxAmt = &ShAmt;
3326 }
3327 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3328 "Failed to find matching min/max shift amounts");
3329 if (MinAmt && MaxAmt)
3330 return ConstantRange(*MinAmt, *MaxAmt + 1);
3331 }
3332
3333 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3334 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3335 KnownBits KnownAmt = computeKnownBits(V.getOperand(1), DemandedElts, Depth);
3336 if (KnownAmt.getMaxValue().ult(BitWidth))
3337 return ConstantRange::fromKnownBits(KnownAmt, /*IsSigned=*/false);
3338
3339 return std::nullopt;
3340}
3341
3342std::optional<uint64_t>
3344 unsigned Depth) const {
3345 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3346 V.getOpcode() == ISD::SRA) &&
3347 "Unknown shift node");
3348 if (std::optional<ConstantRange> AmtRange =
3349 getValidShiftAmountRange(V, DemandedElts, Depth))
3350 if (const APInt *ShAmt = AmtRange->getSingleElement())
3351 return ShAmt->getZExtValue();
3352 return std::nullopt;
3353}
3354
3355std::optional<uint64_t>
3357 EVT VT = V.getValueType();
3358 APInt DemandedElts = VT.isFixedLengthVector()
3360 : APInt(1, 1);
3361 return getValidShiftAmount(V, DemandedElts, Depth);
3362}
3363
3364std::optional<uint64_t>
3366 unsigned Depth) const {
3367 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3368 V.getOpcode() == ISD::SRA) &&
3369 "Unknown shift node");
3370 if (std::optional<ConstantRange> AmtRange =
3371 getValidShiftAmountRange(V, DemandedElts, Depth))
3372 return AmtRange->getUnsignedMin().getZExtValue();
3373 return std::nullopt;
3374}
3375
3376std::optional<uint64_t>
3378 EVT VT = V.getValueType();
3379 APInt DemandedElts = VT.isFixedLengthVector()
3381 : APInt(1, 1);
3382 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3383}
3384
3385std::optional<uint64_t>
3387 unsigned Depth) const {
3388 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3389 V.getOpcode() == ISD::SRA) &&
3390 "Unknown shift node");
3391 if (std::optional<ConstantRange> AmtRange =
3392 getValidShiftAmountRange(V, DemandedElts, Depth))
3393 return AmtRange->getUnsignedMax().getZExtValue();
3394 return std::nullopt;
3395}
3396
3397std::optional<uint64_t>
3399 EVT VT = V.getValueType();
3400 APInt DemandedElts = VT.isFixedLengthVector()
3402 : APInt(1, 1);
3403 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3404}
3405
3406/// Determine which bits of Op are known to be either zero or one and return
3407/// them in Known. For vectors, the known bits are those that are shared by
3408/// every vector element.
3410 EVT VT = Op.getValueType();
3411
3412 // Since the number of lanes in a scalable vector is unknown at compile time,
3413 // we track one bit which is implicitly broadcast to all lanes. This means
3414 // that all lanes in a scalable vector are considered demanded.
3415 APInt DemandedElts = VT.isFixedLengthVector()
3417 : APInt(1, 1);
3418 return computeKnownBits(Op, DemandedElts, Depth);
3419}
3420
3421/// Determine which bits of Op are known to be either zero or one and return
3422/// them in Known. The DemandedElts argument allows us to only collect the known
3423/// bits that are shared by the requested vector elements.
3425 unsigned Depth) const {
3426 unsigned BitWidth = Op.getScalarValueSizeInBits();
3427
3428 KnownBits Known(BitWidth); // Don't know anything.
3429
3430 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3431 // We know all of the bits for a constant!
3432 return KnownBits::makeConstant(C->getAPIntValue());
3433 }
3434 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
3435 // We know all of the bits for a constant fp!
3436 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
3437 }
3438
3439 if (Depth >= MaxRecursionDepth)
3440 return Known; // Limit search depth.
3441
3442 KnownBits Known2;
3443 unsigned NumElts = DemandedElts.getBitWidth();
3444 assert((!Op.getValueType().isFixedLengthVector() ||
3445 NumElts == Op.getValueType().getVectorNumElements()) &&
3446 "Unexpected vector size");
3447
3448 if (!DemandedElts)
3449 return Known; // No demanded elts, better to assume we don't know anything.
3450
3451 unsigned Opcode = Op.getOpcode();
3452 switch (Opcode) {
3453 case ISD::MERGE_VALUES:
3454 return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,
3455 Depth + 1);
3456 case ISD::SPLAT_VECTOR: {
3457 SDValue SrcOp = Op.getOperand(0);
3458 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3459 "Expected SPLAT_VECTOR implicit truncation");
3460 // Implicitly truncate the bits to match the official semantics of
3461 // SPLAT_VECTOR.
3462 Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth);
3463 break;
3464 }
3466 unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();
3467 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3468 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3469 for (auto [I, SrcOp] : enumerate(Op->ops())) {
3470 Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);
3471 }
3472 break;
3473 }
3474 case ISD::STEP_VECTOR: {
3475 const APInt &Step = Op.getConstantOperandAPInt(0);
3476
3477 if (Step.isPowerOf2())
3478 Known.Zero.setLowBits(Step.logBase2());
3479
3481
3482 if (!isUIntN(BitWidth, Op.getValueType().getVectorMinNumElements()))
3483 break;
3484 const APInt MinNumElts =
3485 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3486
3487 bool Overflow;
3488 const APInt MaxNumElts = getVScaleRange(&F, BitWidth)
3490 .umul_ov(MinNumElts, Overflow);
3491 if (Overflow)
3492 break;
3493
3494 const APInt MaxValue = (MaxNumElts - 1).umul_ov(Step, Overflow);
3495 if (Overflow)
3496 break;
3497
3498 Known.Zero.setHighBits(MaxValue.countl_zero());
3499 break;
3500 }
3501 case ISD::BUILD_VECTOR:
3502 assert(!Op.getValueType().isScalableVector());
3503 // Collect the known bits that are shared by every demanded vector element.
3504 Known.Zero.setAllBits(); Known.One.setAllBits();
3505 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3506 if (!DemandedElts[i])
3507 continue;
3508
3509 SDValue SrcOp = Op.getOperand(i);
3510 Known2 = computeKnownBits(SrcOp, Depth + 1);
3511
3512 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3513 if (SrcOp.getValueSizeInBits() != BitWidth) {
3514 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3515 "Expected BUILD_VECTOR implicit truncation");
3516 Known2 = Known2.trunc(BitWidth);
3517 }
3518
3519 // Known bits are the values that are shared by every demanded element.
3520 Known = Known.intersectWith(Known2);
3521
3522 // If we don't know any bits, early out.
3523 if (Known.isUnknown())
3524 break;
3525 }
3526 break;
3527 case ISD::VECTOR_SHUFFLE: {
3528 assert(!Op.getValueType().isScalableVector());
3529 // Collect the known bits that are shared by every vector element referenced
3530 // by the shuffle.
3531 APInt DemandedLHS, DemandedRHS;
3532 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3533 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3534 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
3535 DemandedLHS, DemandedRHS))
3536 break;
3537
3538 // Known bits are the values that are shared by every demanded element.
3539 Known.Zero.setAllBits(); Known.One.setAllBits();
3540 if (!!DemandedLHS) {
3541 SDValue LHS = Op.getOperand(0);
3542 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
3543 Known = Known.intersectWith(Known2);
3544 }
3545 // If we don't know any bits, early out.
3546 if (Known.isUnknown())
3547 break;
3548 if (!!DemandedRHS) {
3549 SDValue RHS = Op.getOperand(1);
3550 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
3551 Known = Known.intersectWith(Known2);
3552 }
3553 break;
3554 }
3555 case ISD::VSCALE: {
3557 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
3558 Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits();
3559 break;
3560 }
3561 case ISD::CONCAT_VECTORS: {
3562 if (Op.getValueType().isScalableVector())
3563 break;
3564 // Split DemandedElts and test each of the demanded subvectors.
3565 Known.Zero.setAllBits(); Known.One.setAllBits();
3566 EVT SubVectorVT = Op.getOperand(0).getValueType();
3567 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3568 unsigned NumSubVectors = Op.getNumOperands();
3569 for (unsigned i = 0; i != NumSubVectors; ++i) {
3570 APInt DemandedSub =
3571 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
3572 if (!!DemandedSub) {
3573 SDValue Sub = Op.getOperand(i);
3574 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
3575 Known = Known.intersectWith(Known2);
3576 }
3577 // If we don't know any bits, early out.
3578 if (Known.isUnknown())
3579 break;
3580 }
3581 break;
3582 }
3583 case ISD::INSERT_SUBVECTOR: {
3584 if (Op.getValueType().isScalableVector())
3585 break;
3586 // Demand any elements from the subvector and the remainder from the src its
3587 // inserted into.
3588 SDValue Src = Op.getOperand(0);
3589 SDValue Sub = Op.getOperand(1);
3590 uint64_t Idx = Op.getConstantOperandVal(2);
3591 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3592 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3593 APInt DemandedSrcElts = DemandedElts;
3594 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
3595
3596 Known.One.setAllBits();
3597 Known.Zero.setAllBits();
3598 if (!!DemandedSubElts) {
3599 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
3600 if (Known.isUnknown())
3601 break; // early-out.
3602 }
3603 if (!!DemandedSrcElts) {
3604 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3605 Known = Known.intersectWith(Known2);
3606 }
3607 break;
3608 }
3610 // Offset the demanded elts by the subvector index.
3611 SDValue Src = Op.getOperand(0);
3612 // Bail until we can represent demanded elements for scalable vectors.
3613 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3614 break;
3615 uint64_t Idx = Op.getConstantOperandVal(1);
3616 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3617 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3618 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3619 break;
3620 }
3621 case ISD::SCALAR_TO_VECTOR: {
3622 if (Op.getValueType().isScalableVector())
3623 break;
3624 // We know about scalar_to_vector as much as we know about it source,
3625 // which becomes the first element of otherwise unknown vector.
3626 if (DemandedElts != 1)
3627 break;
3628
3629 SDValue N0 = Op.getOperand(0);
3630 Known = computeKnownBits(N0, Depth + 1);
3631 if (N0.getValueSizeInBits() != BitWidth)
3632 Known = Known.trunc(BitWidth);
3633
3634 break;
3635 }
3636 case ISD::BITCAST: {
3637 if (Op.getValueType().isScalableVector())
3638 break;
3639
3640 SDValue N0 = Op.getOperand(0);
3641 EVT SubVT = N0.getValueType();
3642 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3643
3644 // Ignore bitcasts from unsupported types.
3645 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3646 break;
3647
3648 // Fast handling of 'identity' bitcasts.
3649 if (BitWidth == SubBitWidth) {
3650 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
3651 break;
3652 }
3653
3654 bool IsLE = getDataLayout().isLittleEndian();
3655
3656 // Bitcast 'small element' vector to 'large element' scalar/vector.
3657 if ((BitWidth % SubBitWidth) == 0) {
3658 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3659
3660 // Collect known bits for the (larger) output by collecting the known
3661 // bits from each set of sub elements and shift these into place.
3662 // We need to separately call computeKnownBits for each set of
3663 // sub elements as the knownbits for each is likely to be different.
3664 unsigned SubScale = BitWidth / SubBitWidth;
3665 APInt SubDemandedElts(NumElts * SubScale, 0);
3666 for (unsigned i = 0; i != NumElts; ++i)
3667 if (DemandedElts[i])
3668 SubDemandedElts.setBit(i * SubScale);
3669
3670 for (unsigned i = 0; i != SubScale; ++i) {
3671 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
3672 Depth + 1);
3673 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3674 Known.insertBits(Known2, SubBitWidth * Shifts);
3675 }
3676 }
3677
3678 // Bitcast 'large element' scalar/vector to 'small element' vector.
3679 if ((SubBitWidth % BitWidth) == 0) {
3680 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3681
3682 // Collect known bits for the (smaller) output by collecting the known
3683 // bits from the overlapping larger input elements and extracting the
3684 // sub sections we actually care about.
3685 unsigned SubScale = SubBitWidth / BitWidth;
3686 APInt SubDemandedElts =
3687 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
3688 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
3689
3690 Known.Zero.setAllBits(); Known.One.setAllBits();
3691 for (unsigned i = 0; i != NumElts; ++i)
3692 if (DemandedElts[i]) {
3693 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3694 unsigned Offset = (Shifts % SubScale) * BitWidth;
3695 Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));
3696 // If we don't know any bits, early out.
3697 if (Known.isUnknown())
3698 break;
3699 }
3700 }
3701 break;
3702 }
3703 case ISD::AND:
3704 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3705 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3706
3707 Known &= Known2;
3708 break;
3709 case ISD::OR:
3710 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3711 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3712
3713 Known |= Known2;
3714 break;
3715 case ISD::XOR:
3716 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3717 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3718
3719 Known ^= Known2;
3720 break;
3721 case ISD::MUL: {
3722 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3723 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3724 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3725 // TODO: SelfMultiply can be poison, but not undef.
3726 if (SelfMultiply)
3727 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3728 Op.getOperand(0), DemandedElts, false, Depth + 1);
3729 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3730
3731 // If the multiplication is known not to overflow, the product of a number
3732 // with itself is non-negative. Only do this if we didn't already computed
3733 // the opposite value for the sign bit.
3734 if (Op->getFlags().hasNoSignedWrap() &&
3735 Op.getOperand(0) == Op.getOperand(1) &&
3736 !Known.isNegative())
3737 Known.makeNonNegative();
3738 break;
3739 }
3740 case ISD::MULHU: {
3741 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3742 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3743 Known = KnownBits::mulhu(Known, Known2);
3744 break;
3745 }
3746 case ISD::MULHS: {
3747 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3748 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3749 Known = KnownBits::mulhs(Known, Known2);
3750 break;
3751 }
3752 case ISD::ABDU: {
3753 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3754 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3755 Known = KnownBits::abdu(Known, Known2);
3756 break;
3757 }
3758 case ISD::ABDS: {
3759 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3760 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3761 Known = KnownBits::abds(Known, Known2);
3762 unsigned SignBits1 =
3763 ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3764 if (SignBits1 == 1)
3765 break;
3766 unsigned SignBits0 =
3767 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3768 Known.Zero.setHighBits(std::min(SignBits0, SignBits1) - 1);
3769 break;
3770 }
3771 case ISD::UMUL_LOHI: {
3772 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3773 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3774 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3775 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3776 if (Op.getResNo() == 0)
3777 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3778 else
3779 Known = KnownBits::mulhu(Known, Known2);
3780 break;
3781 }
3782 case ISD::SMUL_LOHI: {
3783 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3784 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3785 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3786 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3787 if (Op.getResNo() == 0)
3788 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3789 else
3790 Known = KnownBits::mulhs(Known, Known2);
3791 break;
3792 }
3793 case ISD::AVGFLOORU: {
3794 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3795 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3796 Known = KnownBits::avgFloorU(Known, Known2);
3797 break;
3798 }
3799 case ISD::AVGCEILU: {
3800 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3801 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3802 Known = KnownBits::avgCeilU(Known, Known2);
3803 break;
3804 }
3805 case ISD::AVGFLOORS: {
3806 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3807 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3808 Known = KnownBits::avgFloorS(Known, Known2);
3809 break;
3810 }
3811 case ISD::AVGCEILS: {
3812 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3813 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3814 Known = KnownBits::avgCeilS(Known, Known2);
3815 break;
3816 }
3817 case ISD::SELECT:
3818 case ISD::VSELECT:
3819 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3820 // If we don't know any bits, early out.
3821 if (Known.isUnknown())
3822 break;
3823 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3824
3825 // Only known if known in both the LHS and RHS.
3826 Known = Known.intersectWith(Known2);
3827 break;
3828 case ISD::SELECT_CC:
3829 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3830 // If we don't know any bits, early out.
3831 if (Known.isUnknown())
3832 break;
3833 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3834
3835 // Only known if known in both the LHS and RHS.
3836 Known = Known.intersectWith(Known2);
3837 break;
3838 case ISD::SMULO:
3839 case ISD::UMULO:
3840 if (Op.getResNo() != 1)
3841 break;
3842 // The boolean result conforms to getBooleanContents.
3843 // If we know the result of a setcc has the top bits zero, use this info.
3844 // We know that we have an integer-based boolean since these operations
3845 // are only available for integer.
3846 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3848 BitWidth > 1)
3849 Known.Zero.setBitsFrom(1);
3850 break;
3851 case ISD::SETCC:
3852 case ISD::SETCCCARRY:
3853 case ISD::STRICT_FSETCC:
3854 case ISD::STRICT_FSETCCS: {
3855 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3856 // If we know the result of a setcc has the top bits zero, use this info.
3857 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3859 BitWidth > 1)
3860 Known.Zero.setBitsFrom(1);
3861 break;
3862 }
3863 case ISD::SHL: {
3864 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3865 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3866
3867 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3868 bool NSW = Op->getFlags().hasNoSignedWrap();
3869
3870 bool ShAmtNonZero = Known2.isNonZero();
3871
3872 Known = KnownBits::shl(Known, Known2, NUW, NSW, ShAmtNonZero);
3873
3874 // Minimum shift low bits are known zero.
3875 if (std::optional<uint64_t> ShMinAmt =
3876 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))
3877 Known.Zero.setLowBits(*ShMinAmt);
3878 break;
3879 }
3880 case ISD::SRL:
3881 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3882 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3883 Known = KnownBits::lshr(Known, Known2, /*ShAmtNonZero=*/false,
3884 Op->getFlags().hasExact());
3885
3886 // Minimum shift high bits are known zero.
3887 if (std::optional<uint64_t> ShMinAmt =
3888 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))
3889 Known.Zero.setHighBits(*ShMinAmt);
3890 break;
3891 case ISD::SRA:
3892 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3893 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3894 Known = KnownBits::ashr(Known, Known2, /*ShAmtNonZero=*/false,
3895 Op->getFlags().hasExact());
3896 break;
3897 case ISD::FSHL:
3898 case ISD::FSHR:
3899 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3900 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3901
3902 // For fshl, 0-shift returns the 1st arg.
3903 // For fshr, 0-shift returns the 2nd arg.
3904 if (Amt == 0) {
3905 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3906 DemandedElts, Depth + 1);
3907 break;
3908 }
3909
3910 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3911 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3912 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3913 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3914 if (Opcode == ISD::FSHL) {
3915 Known.One <<= Amt;
3916 Known.Zero <<= Amt;
3917 Known2.One.lshrInPlace(BitWidth - Amt);
3918 Known2.Zero.lshrInPlace(BitWidth - Amt);
3919 } else {
3920 Known.One <<= BitWidth - Amt;
3921 Known.Zero <<= BitWidth - Amt;
3922 Known2.One.lshrInPlace(Amt);
3923 Known2.Zero.lshrInPlace(Amt);
3924 }
3925 Known = Known.unionWith(Known2);
3926 }
3927 break;
3928 case ISD::SHL_PARTS:
3929 case ISD::SRA_PARTS:
3930 case ISD::SRL_PARTS: {
3931 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3932
3933 // Collect lo/hi source values and concatenate.
3934 unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();
3935 unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();
3936 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3937 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3938 Known = Known2.concat(Known);
3939
3940 // Collect shift amount.
3941 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3942
3943 if (Opcode == ISD::SHL_PARTS)
3944 Known = KnownBits::shl(Known, Known2);
3945 else if (Opcode == ISD::SRA_PARTS)
3946 Known = KnownBits::ashr(Known, Known2);
3947 else // if (Opcode == ISD::SRL_PARTS)
3948 Known = KnownBits::lshr(Known, Known2);
3949
3950 // TODO: Minimum shift low/high bits are known zero.
3951
3952 if (Op.getResNo() == 0)
3953 Known = Known.extractBits(LoBits, 0);
3954 else
3955 Known = Known.extractBits(HiBits, LoBits);
3956 break;
3957 }
3959 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3960 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3961 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3962 break;
3963 }
3964 case ISD::CTTZ:
3965 case ISD::CTTZ_ZERO_UNDEF: {
3966 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3967 // If we have a known 1, its position is our upper bound.
3968 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3969 unsigned LowBits = llvm::bit_width(PossibleTZ);
3970 Known.Zero.setBitsFrom(LowBits);
3971 break;
3972 }
3973 case ISD::CTLZ:
3974 case ISD::CTLZ_ZERO_UNDEF: {
3975 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3976 // If we have a known 1, its position is our upper bound.
3977 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3978 unsigned LowBits = llvm::bit_width(PossibleLZ);
3979 Known.Zero.setBitsFrom(LowBits);
3980 break;
3981 }
3982 case ISD::CTPOP: {
3983 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3984 // If we know some of the bits are zero, they can't be one.
3985 unsigned PossibleOnes = Known2.countMaxPopulation();
3986 Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));
3987 break;
3988 }
3989 case ISD::PARITY: {
3990 // Parity returns 0 everywhere but the LSB.
3991 Known.Zero.setBitsFrom(1);
3992 break;
3993 }
3994 case ISD::MGATHER:
3995 case ISD::MLOAD: {
3996 ISD::LoadExtType ETy =
3997 (Opcode == ISD::MGATHER)
3998 ? cast<MaskedGatherSDNode>(Op)->getExtensionType()
3999 : cast<MaskedLoadSDNode>(Op)->getExtensionType();
4000 if (ETy == ISD::ZEXTLOAD) {
4001 EVT MemVT = cast<MemSDNode>(Op)->getMemoryVT();
4002 KnownBits Known0(MemVT.getScalarSizeInBits());
4003 return Known0.zext(BitWidth);
4004 }
4005 break;
4006 }
4007 case ISD::LOAD: {
4008 LoadSDNode *LD = cast<LoadSDNode>(Op);
4009 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
4010 if (ISD::isNON_EXTLoad(LD) && Cst) {
4011 // Determine any common known bits from the loaded constant pool value.
4012 Type *CstTy = Cst->getType();
4013 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
4014 !Op.getValueType().isScalableVector()) {
4015 // If its a vector splat, then we can (quickly) reuse the scalar path.
4016 // NOTE: We assume all elements match and none are UNDEF.
4017 if (CstTy->isVectorTy()) {
4018 if (const Constant *Splat = Cst->getSplatValue()) {
4019 Cst = Splat;
4020 CstTy = Cst->getType();
4021 }
4022 }
4023 // TODO - do we need to handle different bitwidths?
4024 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4025 // Iterate across all vector elements finding common known bits.
4026 Known.One.setAllBits();
4027 Known.Zero.setAllBits();
4028 for (unsigned i = 0; i != NumElts; ++i) {
4029 if (!DemandedElts[i])
4030 continue;
4031 if (Constant *Elt = Cst->getAggregateElement(i)) {
4032 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4033 const APInt &Value = CInt->getValue();
4034 Known.One &= Value;
4035 Known.Zero &= ~Value;
4036 continue;
4037 }
4038 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4039 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4040 Known.One &= Value;
4041 Known.Zero &= ~Value;
4042 continue;
4043 }
4044 }
4045 Known.One.clearAllBits();
4046 Known.Zero.clearAllBits();
4047 break;
4048 }
4049 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4050 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
4051 Known = KnownBits::makeConstant(CInt->getValue());
4052 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
4053 Known =
4054 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
4055 }
4056 }
4057 }
4058 } else if (Op.getResNo() == 0) {
4059 KnownBits Known0(!LD->getMemoryVT().isScalableVT()
4060 ? LD->getMemoryVT().getFixedSizeInBits()
4061 : BitWidth);
4062 EVT VT = Op.getValueType();
4063 // Fill in any known bits from range information. There are 3 types being
4064 // used. The results VT (same vector elt size as BitWidth), the loaded
4065 // MemoryVT (which may or may not be vector) and the range VTs original
4066 // type. The range matadata needs the full range (i.e
4067 // MemoryVT().getSizeInBits()), which is truncated to the correct elt size
4068 // if it is know. These are then extended to the original VT sizes below.
4069 if (const MDNode *MD = LD->getRanges()) {
4071 if (VT.isVector()) {
4072 // Handle truncation to the first demanded element.
4073 // TODO: Figure out which demanded elements are covered
4074 if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
4075 break;
4076 Known0 = Known0.trunc(BitWidth);
4077 }
4078 }
4079
4080 if (LD->getMemoryVT().isVector())
4081 Known0 = Known0.trunc(LD->getMemoryVT().getScalarSizeInBits());
4082
4083 // Extend the Known bits from memory to the size of the result.
4084 if (ISD::isZEXTLoad(Op.getNode()))
4085 Known = Known0.zext(BitWidth);
4086 else if (ISD::isSEXTLoad(Op.getNode()))
4087 Known = Known0.sext(BitWidth);
4088 else if (ISD::isEXTLoad(Op.getNode()))
4089 Known = Known0.anyext(BitWidth);
4090 else
4091 Known = Known0;
4092 assert(Known.getBitWidth() == BitWidth);
4093 return Known;
4094 }
4095 break;
4096 }
4098 if (Op.getValueType().isScalableVector())
4099 break;
4100 EVT InVT = Op.getOperand(0).getValueType();
4101 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
4102 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
4103 Known = Known.zext(BitWidth);
4104 break;
4105 }
4106 case ISD::ZERO_EXTEND: {
4107 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4108 Known = Known.zext(BitWidth);
4109 break;
4110 }
4112 if (Op.getValueType().isScalableVector())
4113 break;
4114 EVT InVT = Op.getOperand(0).getValueType();
4115 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
4116 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
4117 // If the sign bit is known to be zero or one, then sext will extend
4118 // it to the top bits, else it will just zext.
4119 Known = Known.sext(BitWidth);
4120 break;
4121 }
4122 case ISD::SIGN_EXTEND: {
4123 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4124 // If the sign bit is known to be zero or one, then sext will extend
4125 // it to the top bits, else it will just zext.
4126 Known = Known.sext(BitWidth);
4127 break;
4128 }
4130 if (Op.getValueType().isScalableVector())
4131 break;
4132 EVT InVT = Op.getOperand(0).getValueType();
4133 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
4134 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
4135 Known = Known.anyext(BitWidth);
4136 break;
4137 }
4138 case ISD::ANY_EXTEND: {
4139 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4140 Known = Known.anyext(BitWidth);
4141 break;
4142 }
4143 case ISD::TRUNCATE: {
4144 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4145 Known = Known.trunc(BitWidth);
4146 break;
4147 }
4148 case ISD::AssertZext: {
4149 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4151 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4152 Known.Zero |= (~InMask);
4153 Known.One &= (~Known.Zero);
4154 break;
4155 }
4156 case ISD::AssertAlign: {
4157 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
4158 assert(LogOfAlign != 0);
4159
4160 // TODO: Should use maximum with source
4161 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4162 // well as clearing one bits.
4163 Known.Zero.setLowBits(LogOfAlign);
4164 Known.One.clearLowBits(LogOfAlign);
4165 break;
4166 }
4167 case ISD::FGETSIGN:
4168 // All bits are zero except the low bit.
4169 Known.Zero.setBitsFrom(1);
4170 break;
4171 case ISD::ADD:
4172 case ISD::SUB: {
4173 SDNodeFlags Flags = Op.getNode()->getFlags();
4174 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4175 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4177 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(),
4178 Flags.hasNoUnsignedWrap(), Known, Known2);
4179 break;
4180 }
4181 case ISD::USUBO:
4182 case ISD::SSUBO:
4183 case ISD::USUBO_CARRY:
4184 case ISD::SSUBO_CARRY:
4185 if (Op.getResNo() == 1) {
4186 // If we know the result of a setcc has the top bits zero, use this info.
4187 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
4189 BitWidth > 1)
4190 Known.Zero.setBitsFrom(1);
4191 break;
4192 }
4193 [[fallthrough]];
4194 case ISD::SUBC: {
4195 assert(Op.getResNo() == 0 &&
4196 "We only compute knownbits for the difference here.");
4197
4198 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4199 KnownBits Borrow(1);
4200 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4201 Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
4202 // Borrow has bit width 1
4203 Borrow = Borrow.trunc(1);
4204 } else {
4205 Borrow.setAllZero();
4206 }
4207
4208 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4209 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4210 Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
4211 break;
4212 }
4213 case ISD::UADDO:
4214 case ISD::SADDO:
4215 case ISD::UADDO_CARRY:
4216 case ISD::SADDO_CARRY:
4217 if (Op.getResNo() == 1) {
4218 // If we know the result of a setcc has the top bits zero, use this info.
4219 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
4221 BitWidth > 1)
4222 Known.Zero.setBitsFrom(1);
4223 break;
4224 }
4225 [[fallthrough]];
4226 case ISD::ADDC:
4227 case ISD::ADDE: {
4228 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4229
4230 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4231 KnownBits Carry(1);
4232 if (Opcode == ISD::ADDE)
4233 // Can't track carry from glue, set carry to unknown.
4234 Carry.resetAll();
4235 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4236 Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
4237 // Carry has bit width 1
4238 Carry = Carry.trunc(1);
4239 } else {
4240 Carry.setAllZero();
4241 }
4242
4243 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4244 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4245 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
4246 break;
4247 }
4248 case ISD::UDIV: {
4249 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4250 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4251 Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());
4252 break;
4253 }
4254 case ISD::SDIV: {
4255 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4256 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4257 Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());
4258 break;
4259 }
4260 case ISD::SREM: {
4261 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4262 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4263 Known = KnownBits::srem(Known, Known2);
4264 break;
4265 }
4266 case ISD::UREM: {
4267 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4268 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4269 Known = KnownBits::urem(Known, Known2);
4270 break;
4271 }
4272 case ISD::EXTRACT_ELEMENT: {
4273 Known = computeKnownBits(Op.getOperand(0), Depth+1);
4274 const unsigned Index = Op.getConstantOperandVal(1);
4275 const unsigned EltBitWidth = Op.getValueSizeInBits();
4276
4277 // Remove low part of known bits mask
4278 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
4279 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
4280
4281 // Remove high part of known bit mask
4282 Known = Known.trunc(EltBitWidth);
4283 break;
4284 }
4286 SDValue InVec = Op.getOperand(0);
4287 SDValue EltNo = Op.getOperand(1);
4288 EVT VecVT = InVec.getValueType();
4289 // computeKnownBits not yet implemented for scalable vectors.
4290 if (VecVT.isScalableVector())
4291 break;
4292 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4293 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4294
4295 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4296 // anything about the extended bits.
4297 if (BitWidth > EltBitWidth)
4298 Known = Known.trunc(EltBitWidth);
4299
4300 // If we know the element index, just demand that vector element, else for
4301 // an unknown element index, ignore DemandedElts and demand them all.
4302 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4303 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4304 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4305 DemandedSrcElts =
4306 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4307
4308 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
4309 if (BitWidth > EltBitWidth)
4310 Known = Known.anyext(BitWidth);
4311 break;
4312 }
4314 if (Op.getValueType().isScalableVector())
4315 break;
4316
4317 // If we know the element index, split the demand between the
4318 // source vector and the inserted element, otherwise assume we need
4319 // the original demanded vector elements and the value.
4320 SDValue InVec = Op.getOperand(0);
4321 SDValue InVal = Op.getOperand(1);
4322 SDValue EltNo = Op.getOperand(2);
4323 bool DemandedVal = true;
4324 APInt DemandedVecElts = DemandedElts;
4325 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4326 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4327 unsigned EltIdx = CEltNo->getZExtValue();
4328 DemandedVal = !!DemandedElts[EltIdx];
4329 DemandedVecElts.clearBit(EltIdx);
4330 }
4331 Known.One.setAllBits();
4332 Known.Zero.setAllBits();
4333 if (DemandedVal) {
4334 Known2 = computeKnownBits(InVal, Depth + 1);
4335 Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));
4336 }
4337 if (!!DemandedVecElts) {
4338 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
4339 Known = Known.intersectWith(Known2);
4340 }
4341 break;
4342 }
4343 case ISD::BITREVERSE: {
4344 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4345 Known = Known2.reverseBits();
4346 break;
4347 }
4348 case ISD::BSWAP: {
4349 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4350 Known = Known2.byteSwap();
4351 break;
4352 }
4353 case ISD::ABS: {
4354 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4355 Known = Known2.abs();
4356 Known.Zero.setHighBits(
4357 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1) - 1);
4358 break;
4359 }
4360 case ISD::USUBSAT: {
4361 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4362 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4363 Known = KnownBits::usub_sat(Known, Known2);
4364 break;
4365 }
4366 case ISD::UMIN: {
4367 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4368 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4369 Known = KnownBits::umin(Known, Known2);
4370 break;
4371 }
4372 case ISD::UMAX: {
4373 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4374 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4375 Known = KnownBits::umax(Known, Known2);
4376 break;
4377 }
4378 case ISD::SMIN:
4379 case ISD::SMAX: {
4380 // If we have a clamp pattern, we know that the number of sign bits will be
4381 // the minimum of the clamp min/max range.
4382 bool IsMax = (Opcode == ISD::SMAX);
4383 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4384 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4385 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4386 CstHigh =
4387 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4388 if (CstLow && CstHigh) {
4389 if (!IsMax)
4390 std::swap(CstLow, CstHigh);
4391
4392 const APInt &ValueLow = CstLow->getAPIntValue();
4393 const APInt &ValueHigh = CstHigh->getAPIntValue();
4394 if (ValueLow.sle(ValueHigh)) {
4395 unsigned LowSignBits = ValueLow.getNumSignBits();
4396 unsigned HighSignBits = ValueHigh.getNumSignBits();
4397 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
4398 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4399 Known.One.setHighBits(MinSignBits);
4400 break;
4401 }
4402 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4403 Known.Zero.setHighBits(MinSignBits);
4404 break;
4405 }
4406 }
4407 }
4408
4409 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4410 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4411 if (IsMax)
4412 Known = KnownBits::smax(Known, Known2);
4413 else
4414 Known = KnownBits::smin(Known, Known2);
4415
4416 // For SMAX, if CstLow is non-negative we know the result will be
4417 // non-negative and thus all sign bits are 0.
4418 // TODO: There's an equivalent of this for smin with negative constant for
4419 // known ones.
4420 if (IsMax && CstLow) {
4421 const APInt &ValueLow = CstLow->getAPIntValue();
4422 if (ValueLow.isNonNegative()) {
4423 unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4424 Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));
4425 }
4426 }
4427
4428 break;
4429 }
4430 case ISD::UINT_TO_FP: {
4431 Known.makeNonNegative();
4432 break;
4433 }
4434 case ISD::SINT_TO_FP: {
4435 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4436 if (Known2.isNonNegative())
4437 Known.makeNonNegative();
4438 else if (Known2.isNegative())
4439 Known.makeNegative();
4440 break;
4441 }
4442 case ISD::FP_TO_UINT_SAT: {
4443 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4444 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4446 break;
4447 }
4449 if (Op.getResNo() == 1) {
4450 // The boolean result conforms to getBooleanContents.
4451 // If we know the result of a setcc has the top bits zero, use this info.
4452 // We know that we have an integer-based boolean since these operations
4453 // are only available for integer.
4454 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
4456 BitWidth > 1)
4457 Known.Zero.setBitsFrom(1);
4458 break;
4459 }
4460 [[fallthrough]];
4462 case ISD::ATOMIC_SWAP:
4474 case ISD::ATOMIC_LOAD: {
4475 unsigned MemBits =
4476 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4477 // If we are looking at the loaded value.
4478 if (Op.getResNo() == 0) {
4480 Known.Zero.setBitsFrom(MemBits);
4481 else if (Op->getOpcode() == ISD::ATOMIC_LOAD &&
4482 cast<AtomicSDNode>(Op)->getExtensionType() == ISD::ZEXTLOAD)
4483 Known.Zero.setBitsFrom(MemBits);
4484 }
4485 break;
4486 }
4487 case ISD::FrameIndex:
4489 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
4490 Known, getMachineFunction());
4491 break;
4492
4493 default:
4494 if (Opcode < ISD::BUILTIN_OP_END)
4495 break;
4496 [[fallthrough]];
4500 // TODO: Probably okay to remove after audit; here to reduce change size
4501 // in initial enablement patch for scalable vectors
4502 if (Op.getValueType().isScalableVector())
4503 break;
4504
4505 // Allow the target to implement this method for its nodes.
4506 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
4507 break;
4508 }
4509
4510 return Known;
4511}
4512
4513/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4515 switch (OR) {
4523 }
4524 llvm_unreachable("Unknown OverflowResult");
4525}
4526
4529 // X + 0 never overflow
4530 if (isNullConstant(N1))
4531 return OFK_Never;
4532
4533 // If both operands each have at least two sign bits, the addition
4534 // cannot overflow.
4535 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4536 return OFK_Never;
4537
4538 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4539 return OFK_Sometime;
4540}
4541
4544 // X + 0 never overflow
4545 if (isNullConstant(N1))
4546 return OFK_Never;
4547
4548 // mulhi + 1 never overflow
4549 KnownBits N1Known = computeKnownBits(N1);
4550 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4551 N1Known.getMaxValue().ult(2))
4552 return OFK_Never;
4553
4554 KnownBits N0Known = computeKnownBits(N0);
4555 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4556 N0Known.getMaxValue().ult(2))
4557 return OFK_Never;
4558
4559 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4560 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4561 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4562 return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));
4563}
4564
4567 // X - 0 never overflow
4568 if (isNullConstant(N1))
4569 return OFK_Never;
4570
4571 // If both operands each have at least two sign bits, the subtraction
4572 // cannot overflow.
4573 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4574 return OFK_Never;
4575
4576 KnownBits N0Known = computeKnownBits(N0);
4577 KnownBits N1Known = computeKnownBits(N1);
4578 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);
4579 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);
4580 return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));
4581}
4582
4585 // X - 0 never overflow
4586 if (isNullConstant(N1))
4587 return OFK_Never;
4588
4589 KnownBits N0Known = computeKnownBits(N0);
4590 KnownBits N1Known = computeKnownBits(N1);
4591 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4592 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4593 return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));
4594}
4595
4598 // X * 0 and X * 1 never overflow.
4599 if (isNullConstant(N1) || isOneConstant(N1))
4600 return OFK_Never;
4601
4602 KnownBits N0Known = computeKnownBits(N0);
4603 KnownBits N1Known = computeKnownBits(N1);
4604 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4605 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4606 return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));
4607}
4608
4611 // X * 0 and X * 1 never overflow.
4612 if (isNullConstant(N1) || isOneConstant(N1))
4613 return OFK_Never;
4614
4615 // Get the size of the result.
4616 unsigned BitWidth = N0.getScalarValueSizeInBits();
4617
4618 // Sum of the sign bits.
4619 unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);
4620
4621 // If we have enough sign bits, then there's no overflow.
4622 if (SignBits > BitWidth + 1)
4623 return OFK_Never;
4624
4625 if (SignBits == BitWidth + 1) {
4626 // The overflow occurs when the true multiplication of the
4627 // the operands is the minimum negative number.
4628 KnownBits N0Known = computeKnownBits(N0);
4629 KnownBits N1Known = computeKnownBits(N1);
4630 // If one of the operands is non-negative, then there's no
4631 // overflow.
4632 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4633 return OFK_Never;
4634 }
4635
4636 return OFK_Sometime;
4637}
4638
4640 if (Depth >= MaxRecursionDepth)
4641 return false; // Limit search depth.
4642
4643 EVT OpVT = Val.getValueType();
4644 unsigned BitWidth = OpVT.getScalarSizeInBits();
4645
4646 // Is the constant a known power of 2?
4648 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4649 }))
4650 return true;
4651
4652 // A left-shift of a constant one will have exactly one bit set because
4653 // shifting the bit off the end is undefined.
4654 if (Val.getOpcode() == ISD::SHL) {
4655 auto *C = isConstOrConstSplat(Val.getOperand(0));
4656 if (C && C->getAPIntValue() == 1)
4657 return true;
4658 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4659 isKnownNeverZero(Val, Depth);
4660 }
4661
4662 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4663 // one bit set.
4664 if (Val.getOpcode() == ISD::SRL) {
4665 auto *C = isConstOrConstSplat(Val.getOperand(0));
4666 if (C && C->getAPIntValue().isSignMask())
4667 return true;
4668 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4669 isKnownNeverZero(Val, Depth);
4670 }
4671
4672 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4673 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4674
4675 // Are all operands of a build vector constant powers of two?
4676 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4677 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
4678 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4679 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4680 return false;
4681 }))
4682 return true;
4683
4684 // Is the operand of a splat vector a constant power of two?
4685 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4686 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
4687 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
4688 return true;
4689
4690 // vscale(power-of-two) is a power-of-two for some targets
4691 if (Val.getOpcode() == ISD::VSCALE &&
4692 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4694 return true;
4695
4696 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4697 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4698 return isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1) &&
4700
4701 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4702 return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) &&
4704
4705 // Looking for `x & -x` pattern:
4706 // If x == 0:
4707 // x & -x -> 0
4708 // If x != 0:
4709 // x & -x -> non-zero pow2
4710 // so if we find the pattern return whether we know `x` is non-zero.
4711 SDValue X;
4712 if (sd_match(Val, m_And(m_Value(X), m_Neg(m_Deferred(X)))))
4713 return isKnownNeverZero(X, Depth);
4714
4715 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4716 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4717
4718 // More could be done here, though the above checks are enough
4719 // to handle some common cases.
4720 return false;
4721}
4722
4724 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Val, true))
4725 return C1->getValueAPF().getExactLog2Abs() >= 0;
4726
4727 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4728 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4729
4730 return false;
4731}
4732
4734 EVT VT = Op.getValueType();
4735
4736 // Since the number of lanes in a scalable vector is unknown at compile time,
4737 // we track one bit which is implicitly broadcast to all lanes. This means
4738 // that all lanes in a scalable vector are considered demanded.
4739 APInt DemandedElts = VT.isFixedLengthVector()
4741 : APInt(1, 1);
4742 return ComputeNumSignBits(Op, DemandedElts, Depth);
4743}
4744
4745unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4746 unsigned Depth) const {
4747 EVT VT = Op.getValueType();
4748 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4749 unsigned VTBits = VT.getScalarSizeInBits();
4750 unsigned NumElts = DemandedElts.getBitWidth();
4751 unsigned Tmp, Tmp2;
4752 unsigned FirstAnswer = 1;
4753
4754 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4755 const APInt &Val = C->getAPIntValue();
4756 return Val.getNumSignBits();
4757 }
4758
4759 if (Depth >= MaxRecursionDepth)
4760 return 1; // Limit search depth.
4761
4762 if (!DemandedElts)
4763 return 1; // No demanded elts, better to assume we don't know anything.
4764
4765 unsigned Opcode = Op.getOpcode();
4766 switch (Opcode) {
4767 default: break;
4768 case ISD::AssertSext:
4769 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4770 return VTBits-Tmp+1;
4771 case ISD::AssertZext:
4772 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4773 return VTBits-Tmp;
4774 case ISD::MERGE_VALUES:
4775 return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,
4776 Depth + 1);
4777 case ISD::SPLAT_VECTOR: {
4778 // Check if the sign bits of source go down as far as the truncated value.
4779 unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();
4780 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4781 if (NumSrcSignBits > (NumSrcBits - VTBits))
4782 return NumSrcSignBits - (NumSrcBits - VTBits);
4783 break;
4784 }
4785 case ISD::BUILD_VECTOR:
4786 assert(!VT.isScalableVector());
4787 Tmp = VTBits;
4788 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4789 if (!DemandedElts[i])
4790 continue;
4791
4792 SDValue SrcOp = Op.getOperand(i);
4793 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4794 // for constant nodes to ensure we only look at the sign bits.
4795 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SrcOp)) {
4796 APInt T = C->getAPIntValue().trunc(VTBits);
4797 Tmp2 = T.getNumSignBits();
4798 } else {
4799 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
4800
4801 if (SrcOp.getValueSizeInBits() != VTBits) {
4802 assert(SrcOp.getValueSizeInBits() > VTBits &&
4803 "Expected BUILD_VECTOR implicit truncation");
4804 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4805 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4806 }
4807 }
4808 Tmp = std::min(Tmp, Tmp2);
4809 }
4810 return Tmp;
4811
4812 case ISD::VECTOR_SHUFFLE: {
4813 // Collect the minimum number of sign bits that are shared by every vector
4814 // element referenced by the shuffle.
4815 APInt DemandedLHS, DemandedRHS;
4816 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
4817 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4818 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
4819 DemandedLHS, DemandedRHS))
4820 return 1;
4821
4822 Tmp = std::numeric_limits<unsigned>::max();
4823 if (!!DemandedLHS)
4824 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
4825 if (!!DemandedRHS) {
4826 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
4827 Tmp = std::min(Tmp, Tmp2);
4828 }
4829 // If we don't know anything, early out and try computeKnownBits fall-back.
4830 if (Tmp == 1)
4831 break;
4832 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4833 return Tmp;
4834 }
4835
4836 case ISD::BITCAST: {
4837 if (VT.isScalableVector())
4838 break;
4839 SDValue N0 = Op.getOperand(0);
4840 EVT SrcVT = N0.getValueType();
4841 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4842
4843 // Ignore bitcasts from unsupported types..
4844 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4845 break;
4846
4847 // Fast handling of 'identity' bitcasts.
4848 if (VTBits == SrcBits)
4849 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
4850
4851 bool IsLE = getDataLayout().isLittleEndian();
4852
4853 // Bitcast 'large element' scalar/vector to 'small element' vector.
4854 if ((SrcBits % VTBits) == 0) {
4855 assert(VT.isVector() && "Expected bitcast to vector");
4856
4857 unsigned Scale = SrcBits / VTBits;
4858 APInt SrcDemandedElts =
4859 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
4860
4861 // Fast case - sign splat can be simply split across the small elements.
4862 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
4863 if (Tmp == SrcBits)
4864 return VTBits;
4865
4866 // Slow case - determine how far the sign extends into each sub-element.
4867 Tmp2 = VTBits;
4868 for (unsigned i = 0; i != NumElts; ++i)
4869 if (DemandedElts[i]) {
4870 unsigned SubOffset = i % Scale;
4871 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4872 SubOffset = SubOffset * VTBits;
4873 if (Tmp <= SubOffset)
4874 return 1;
4875 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4876 }
4877 return Tmp2;
4878 }
4879 break;
4880 }
4881
4883 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4884 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4885 return VTBits - Tmp + 1;
4886 case ISD::SIGN_EXTEND:
4887 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
4888 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
4890 // Max of the input and what this extends.
4891 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4892 Tmp = VTBits-Tmp+1;
4893 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4894 return std::max(Tmp, Tmp2);
4896 if (VT.isScalableVector())
4897 break;
4898 SDValue Src = Op.getOperand(0);
4899 EVT SrcVT = Src.getValueType();
4900 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
4901 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4902 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
4903 }
4904 case ISD::SRA:
4905 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4906 // SRA X, C -> adds C sign bits.
4907 if (std::optional<uint64_t> ShAmt =
4908 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))
4909 Tmp = std::min<uint64_t>(Tmp + *ShAmt, VTBits);
4910 return Tmp;
4911 case ISD::SHL:
4912 if (std::optional<ConstantRange> ShAmtRange =
4913 getValidShiftAmountRange(Op, DemandedElts, Depth + 1)) {
4914 uint64_t MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
4915 uint64_t MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
4916 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
4917 // shifted out, then we can compute the number of sign bits for the
4918 // operand being extended. A future improvement could be to pass along the
4919 // "shifted left by" information in the recursive calls to
4920 // ComputeKnownSignBits. Allowing us to handle this more generically.
4921 if (ISD::isExtOpcode(Op.getOperand(0).getOpcode())) {
4922 SDValue Ext = Op.getOperand(0);
4923 EVT ExtVT = Ext.getValueType();
4924 SDValue Extendee = Ext.getOperand(0);
4925 EVT ExtendeeVT = Extendee.getValueType();
4926 uint64_t SizeDifference =
4927 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
4928 if (SizeDifference <= MinShAmt) {
4929 Tmp = SizeDifference +
4930 ComputeNumSignBits(Extendee, DemandedElts, Depth + 1);
4931 if (MaxShAmt < Tmp)
4932 return Tmp - MaxShAmt;
4933 }
4934 }
4935 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4936 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4937 if (MaxShAmt < Tmp)
4938 return Tmp - MaxShAmt;
4939 }
4940 break;
4941 case ISD::AND:
4942 case ISD::OR:
4943 case ISD::XOR: // NOT is handled here.
4944 // Logical binary ops preserve the number of sign bits at the worst.
4945 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4946 if (Tmp != 1) {
4947 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4948 FirstAnswer = std::min(Tmp, Tmp2);
4949 // We computed what we know about the sign bits as our first
4950 // answer. Now proceed to the generic code that uses
4951 // computeKnownBits, and pick whichever answer is better.
4952 }
4953 break;
4954
4955 case ISD::SELECT:
4956 case ISD::VSELECT:
4957 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4958 if (Tmp == 1) return 1; // Early out.
4959 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4960 return std::min(Tmp, Tmp2);
4961 case ISD::SELECT_CC:
4962 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4963 if (Tmp == 1) return 1; // Early out.
4964 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
4965 return std::min(Tmp, Tmp2);
4966
4967 case ISD::SMIN:
4968 case ISD::SMAX: {
4969 // If we have a clamp pattern, we know that the number of sign bits will be
4970 // the minimum of the clamp min/max range.
4971 bool IsMax = (Opcode == ISD::SMAX);
4972 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4973 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4974 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4975 CstHigh =
4976 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4977 if (CstLow && CstHigh) {
4978 if (!IsMax)
4979 std::swap(CstLow, CstHigh);
4980 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
4981 Tmp = CstLow->getAPIntValue().getNumSignBits();
4982 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4983 return std::min(Tmp, Tmp2);
4984 }
4985 }
4986
4987 // Fallback - just get the minimum number of sign bits of the operands.
4988 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4989 if (Tmp == 1)
4990 return 1; // Early out.
4991 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4992 return std::min(Tmp, Tmp2);
4993 }
4994 case ISD::UMIN:
4995 case ISD::UMAX:
4996 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4997 if (Tmp == 1)
4998 return 1; // Early out.
4999 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5000 return std::min(Tmp, Tmp2);
5001 case ISD::SSUBO_CARRY:
5002 case ISD::USUBO_CARRY:
5003 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5004 if (Op.getResNo() == 0 && Op.getOperand(0) == Op.getOperand(1))
5005 return VTBits;
5006 [[fallthrough]];
5007 case ISD::SADDO:
5008 case ISD::UADDO:
5009 case ISD::SADDO_CARRY:
5010 case ISD::UADDO_CARRY:
5011 case ISD::SSUBO:
5012 case ISD::USUBO:
5013 case ISD::SMULO:
5014 case ISD::UMULO:
5015 if (Op.getResNo() != 1)
5016 break;
5017 // The boolean result conforms to getBooleanContents. Fall through.
5018 // If setcc returns 0/-1, all bits are sign bits.
5019 // We know that we have an integer-based boolean since these operations
5020 // are only available for integer.
5021 if (TLI->getBooleanContents(VT.isVector(), false) ==
5023 return VTBits;
5024 break;
5025 case ISD::SETCC:
5026 case ISD::SETCCCARRY:
5027 case ISD::STRICT_FSETCC:
5028 case ISD::STRICT_FSETCCS: {
5029 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5030 // If setcc returns 0/-1, all bits are sign bits.
5031 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
5033 return VTBits;
5034 break;
5035 }
5036 case ISD::ROTL:
5037 case ISD::ROTR:
5038 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5039
5040 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5041 if (Tmp == VTBits)
5042 return VTBits;
5043
5044 if (ConstantSDNode *C =
5045 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
5046 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
5047
5048 // Handle rotate right by N like a rotate left by 32-N.
5049 if (Opcode == ISD::ROTR)
5050 RotAmt = (VTBits - RotAmt) % VTBits;
5051
5052 // If we aren't rotating out all of the known-in sign bits, return the
5053 // number that are left. This handles rotl(sext(x), 1) for example.
5054 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5055 }
5056 break;
5057 case ISD::ADD:
5058 case ISD::ADDC:
5059 // Add can have at most one carry bit. Thus we know that the output
5060 // is, at worst, one more bit than the inputs.
5061 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5062 if (Tmp == 1) return 1; // Early out.
5063
5064 // Special case decrementing a value (ADD X, -1):
5065 if (ConstantSDNode *CRHS =
5066 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
5067 if (CRHS->isAllOnes()) {
5068 KnownBits Known =
5069 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
5070
5071 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5072 // sign bits set.
5073 if ((Known.Zero | 1).isAllOnes())
5074 return VTBits;
5075
5076 // If we are subtracting one from a positive number, there is no carry
5077 // out of the result.
5078 if (Known.isNonNegative())
5079 return Tmp;
5080 }
5081
5082 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5083 if (Tmp2 == 1) return 1; // Early out.
5084 return std::min(Tmp, Tmp2) - 1;
5085 case ISD::SUB:
5086 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5087 if (Tmp2 == 1) return 1; // Early out.
5088
5089 // Handle NEG.
5090 if (ConstantSDNode *CLHS =
5091 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
5092 if (CLHS->isZero()) {
5093 KnownBits Known =
5094 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
5095 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5096 // sign bits set.
5097 if ((Known.Zero | 1).isAllOnes())
5098 return VTBits;
5099
5100 // If the input is known to be positive (the sign bit is known clear),
5101 // the output of the NEG has the same number of sign bits as the input.
5102 if (Known.isNonNegative())
5103 return Tmp2;
5104
5105 // Otherwise, we treat this like a SUB.
5106 }
5107
5108 // Sub can have at most one carry bit. Thus we know that the output
5109 // is, at worst, one more bit than the inputs.
5110 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5111 if (Tmp == 1) return 1; // Early out.
5112 return std::min(Tmp, Tmp2) - 1;
5113 case ISD::MUL: {
5114 // The output of the Mul can be at most twice the valid bits in the inputs.
5115 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
5116 if (SignBitsOp0 == 1)
5117 break;
5118 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
5119 if (SignBitsOp1 == 1)
5120 break;
5121 unsigned OutValidBits =
5122 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5123 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5124 }
5125 case ISD::AVGCEILS:
5126 case ISD::AVGFLOORS:
5127 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5128 if (Tmp == 1)
5129 return 1; // Early out.
5130 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5131 return std::min(Tmp, Tmp2);
5132 case ISD::SREM:
5133 // The sign bit is the LHS's sign bit, except when the result of the
5134 // remainder is zero. The magnitude of the result should be less than or
5135 // equal to the magnitude of the LHS. Therefore, the result should have
5136 // at least as many sign bits as the left hand side.
5137 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5138 case ISD::TRUNCATE: {
5139 // Check if the sign bits of source go down as far as the truncated value.
5140 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
5141 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
5142 if (NumSrcSignBits > (NumSrcBits - VTBits))
5143 return NumSrcSignBits - (NumSrcBits - VTBits);
5144 break;
5145 }
5146 case ISD::EXTRACT_ELEMENT: {
5147 if (VT.isScalableVector())
5148 break;
5149 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
5150 const int BitWidth = Op.getValueSizeInBits();
5151 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
5152
5153 // Get reverse index (starting from 1), Op1 value indexes elements from
5154 // little end. Sign starts at big end.
5155 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
5156
5157 // If the sign portion ends in our element the subtraction gives correct
5158 // result. Otherwise it gives either negative or > bitwidth result
5159 return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth);
5160 }
5162 if (VT.isScalableVector())
5163 break;
5164 // If we know the element index, split the demand between the
5165 // source vector and the inserted element, otherwise assume we need
5166 // the original demanded vector elements and the value.
5167 SDValue InVec = Op.getOperand(0);
5168 SDValue InVal = Op.getOperand(1);
5169 SDValue EltNo = Op.getOperand(2);
5170 bool DemandedVal = true;
5171 APInt DemandedVecElts = DemandedElts;
5172 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
5173 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
5174 unsigned EltIdx = CEltNo->getZExtValue();
5175 DemandedVal = !!DemandedElts[EltIdx];
5176 DemandedVecElts.clearBit(EltIdx);
5177 }
5178 Tmp = std::numeric_limits<unsigned>::max();
5179 if (DemandedVal) {
5180 // TODO - handle implicit truncation of inserted elements.
5181 if (InVal.getScalarValueSizeInBits() != VTBits)
5182 break;
5183 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
5184 Tmp = std::min(Tmp, Tmp2);
5185 }
5186 if (!!DemandedVecElts) {
5187 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
5188 Tmp = std::min(Tmp, Tmp2);
5189 }
5190 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5191 return Tmp;
5192 }
5194 assert(!VT.isScalableVector());
5195 SDValue InVec = Op.getOperand(0);
5196 SDValue EltNo = Op.getOperand(1);
5197 EVT VecVT = InVec.getValueType();
5198 // ComputeNumSignBits not yet implemented for scalable vectors.
5199 if (VecVT.isScalableVector())
5200 break;
5201 const unsigned BitWidth = Op.getValueSizeInBits();
5202 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
5203 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5204
5205 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5206 // anything about sign bits. But if the sizes match we can derive knowledge
5207 // about sign bits from the vector operand.
5208 if (BitWidth != EltBitWidth)
5209 break;
5210
5211 // If we know the element index, just demand that vector element, else for
5212 // an unknown element index, ignore DemandedElts and demand them all.
5213 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
5214 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
5215 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
5216 DemandedSrcElts =
5217 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
5218
5219 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
5220 }
5222 // Offset the demanded elts by the subvector index.
5223 SDValue Src = Op.getOperand(0);
5224 // Bail until we can represent demanded elements for scalable vectors.
5225 if (Src.getValueType().isScalableVector())
5226 break;
5227 uint64_t Idx = Op.getConstantOperandVal(1);
5228 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5229 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
5230 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
5231 }
5232 case ISD::CONCAT_VECTORS: {
5233 if (VT.isScalableVector())
5234 break;
5235 // Determine the minimum number of sign bits across all demanded
5236 // elts of the input vectors. Early out if the result is already 1.
5237 Tmp = std::numeric_limits<unsigned>::max();
5238 EVT SubVectorVT = Op.getOperand(0).getValueType();
5239 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5240 unsigned NumSubVectors = Op.getNumOperands();
5241 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5242 APInt DemandedSub =
5243 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
5244 if (!DemandedSub)
5245 continue;
5246 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
5247 Tmp = std::min(Tmp, Tmp2);
5248 }
5249 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5250 return Tmp;
5251 }
5252 case ISD::INSERT_SUBVECTOR: {
5253 if (VT.isScalableVector())
5254 break;
5255 // Demand any elements from the subvector and the remainder from the src its
5256 // inserted into.
5257 SDValue Src = Op.getOperand(0);
5258 SDValue Sub = Op.getOperand(1);
5259 uint64_t Idx = Op.getConstantOperandVal(2);
5260 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5261 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
5262 APInt DemandedSrcElts = DemandedElts;
5263 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
5264
5265 Tmp = std::numeric_limits<unsigned>::max();
5266 if (!!DemandedSubElts) {
5267 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
5268 if (Tmp == 1)
5269 return 1; // early-out
5270 }
5271 if (!!DemandedSrcElts) {
5272 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
5273 Tmp = std::min(Tmp, Tmp2);
5274 }
5275 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5276 return Tmp;
5277 }
5278 case ISD::LOAD: {
5279 LoadSDNode *LD = cast<LoadSDNode>(Op);
5280 if (const MDNode *Ranges = LD->getRanges()) {
5281 if (DemandedElts != 1)
5282 break;
5283
5285 if (VTBits > CR.getBitWidth()) {
5286 switch (LD->getExtensionType()) {
5287 case ISD::SEXTLOAD:
5288 CR = CR.signExtend(VTBits);
5289 break;
5290 case ISD::ZEXTLOAD:
5291 CR = CR.zeroExtend(VTBits);
5292 break;
5293 default:
5294 break;
5295 }
5296 }
5297
5298 if (VTBits != CR.getBitWidth())
5299 break;
5300 return std::min(CR.getSignedMin().getNumSignBits(),
5302 }
5303
5304 break;
5305 }
5308 case ISD::ATOMIC_SWAP:
5320 case ISD::ATOMIC_LOAD: {
5321 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
5322 // If we are looking at the loaded value.
5323 if (Op.getResNo() == 0) {
5324 if (Tmp == VTBits)
5325 return 1; // early-out
5327 return VTBits - Tmp + 1;
5329 return VTBits - Tmp;
5330 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5331 ISD::LoadExtType ETy = cast<AtomicSDNode>(Op)->getExtensionType();
5332 if (ETy == ISD::SEXTLOAD)
5333 return VTBits - Tmp + 1;
5334 if (ETy == ISD::ZEXTLOAD)
5335 return VTBits - Tmp;
5336 }
5337 }
5338 break;
5339 }
5340 }
5341
5342 // If we are looking at the loaded value of the SDNode.
5343 if (Op.getResNo() == 0) {
5344 // Handle LOADX separately here. EXTLOAD case will fallthrough.
5345 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
5346 unsigned ExtType = LD->getExtensionType();
5347 switch (ExtType) {
5348 default: break;
5349 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5350 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5351 return VTBits - Tmp + 1;
5352 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5353 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5354 return VTBits - Tmp;
5355 case ISD::NON_EXTLOAD:
5356 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5357 // We only need to handle vectors - computeKnownBits should handle
5358 // scalar cases.
5359 Type *CstTy = Cst->getType();
5360 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5361 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5362 VTBits == CstTy->getScalarSizeInBits()) {
5363 Tmp = VTBits;
5364 for (unsigned i = 0; i != NumElts; ++i) {
5365 if (!DemandedElts[i])
5366 continue;
5367 if (Constant *Elt = Cst->getAggregateElement(i)) {
5368 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
5369 const APInt &Value = CInt->getValue();
5370 Tmp = std::min(Tmp, Value.getNumSignBits());
5371 continue;
5372 }
5373 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
5374 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5375 Tmp = std::min(Tmp, Value.getNumSignBits());
5376 continue;
5377 }
5378 }
5379 // Unknown type. Conservatively assume no bits match sign bit.
5380 return 1;
5381 }
5382 return Tmp;
5383 }
5384 }
5385 break;
5386 }
5387 }
5388 }
5389
5390 // Allow the target to implement this method for its nodes.
5391 if (Opcode >= ISD::BUILTIN_OP_END ||
5392 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5393 Opcode == ISD::INTRINSIC_W_CHAIN ||
5394 Opcode == ISD::INTRINSIC_VOID) {
5395 // TODO: This can probably be removed once target code is audited. This
5396 // is here purely to reduce patch size and review complexity.
5397 if (!VT.isScalableVector()) {
5398 unsigned NumBits =
5399 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
5400 if (NumBits > 1)
5401 FirstAnswer = std::max(FirstAnswer, NumBits);
5402 }
5403 }
5404
5405 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5406 // use this information.
5407 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5408 return std::max(FirstAnswer, Known.countMinSignBits());
5409}
5410
5412 unsigned Depth) const {
5413 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5414 return Op.getScalarValueSizeInBits() - SignBits + 1;
5415}
5416
5418 const APInt &DemandedElts,
5419 unsigned Depth) const {
5420 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5421 return Op.getScalarValueSizeInBits() - SignBits + 1;
5422}
5423
5425 unsigned Depth) const {
5426 // Early out for FREEZE.
5427 if (Op.getOpcode() == ISD::FREEZE)
5428 return true;
5429
5430 EVT VT = Op.getValueType();
5431 APInt DemandedElts = VT.isFixedLengthVector()
5433 : APInt(1, 1);
5434 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5435}
5436
5438 const APInt &DemandedElts,
5439 bool PoisonOnly,
5440 unsigned Depth) const {
5441 unsigned Opcode = Op.getOpcode();
5442
5443 // Early out for FREEZE.
5444 if (Opcode == ISD::FREEZE)
5445 return true;
5446
5447 if (Depth >= MaxRecursionDepth)
5448 return false; // Limit search depth.
5449
5450 if (isIntOrFPConstant(Op))
5451 return true;
5452
5453 switch (Opcode) {
5454 case ISD::CONDCODE:
5455 case ISD::VALUETYPE:
5456 case ISD::FrameIndex:
5458 case ISD::CopyFromReg:
5459 return true;
5460
5461 case ISD::UNDEF:
5462 return PoisonOnly;
5463
5464 case ISD::BUILD_VECTOR:
5465 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5466 // this shouldn't affect the result.
5467 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5468 if (!DemandedElts[i])
5469 continue;
5471 Depth + 1))
5472 return false;
5473 }
5474 return true;
5475
5476 case ISD::SPLAT_VECTOR:
5477 return isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), PoisonOnly,
5478 Depth + 1);
5479
5480 case ISD::VECTOR_SHUFFLE: {
5481 APInt DemandedLHS, DemandedRHS;
5482 auto *SVN = cast<ShuffleVectorSDNode>(Op);
5483 if (!getShuffleDemandedElts(DemandedElts.getBitWidth(), SVN->getMask(),
5484 DemandedElts, DemandedLHS, DemandedRHS,
5485 /*AllowUndefElts=*/false))
5486 return false;
5487 if (!DemandedLHS.isZero() &&
5488 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedLHS,
5489 PoisonOnly, Depth + 1))
5490 return false;
5491 if (!DemandedRHS.isZero() &&
5492 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedRHS,
5493 PoisonOnly, Depth + 1))
5494 return false;
5495 return true;
5496 }
5497
5498 // TODO: Search for noundef attributes from library functions.
5499
5500 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5501
5502 default:
5503 // Allow the target to implement this method for its nodes.
5504 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5505 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5507 Op, DemandedElts, *this, PoisonOnly, Depth);
5508 break;
5509 }
5510
5511 // If Op can't create undef/poison and none of its operands are undef/poison
5512 // then Op is never undef/poison.
5513 // NOTE: TargetNodes can handle this in themselves in
5514 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5515 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5516 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5517 Depth) &&
5518 all_of(Op->ops(), [&](SDValue V) {
5519 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
5520 });
5521}
5522
5524 bool ConsiderFlags,
5525 unsigned Depth) const {
5526 EVT VT = Op.getValueType();
5527 APInt DemandedElts = VT.isFixedLengthVector()
5529 : APInt(1, 1);
5530 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5531 Depth);
5532}
5533
5535 bool PoisonOnly, bool ConsiderFlags,
5536 unsigned Depth) const {
5537 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
5538 return true;
5539
5540 unsigned Opcode = Op.getOpcode();
5541 switch (Opcode) {
5542 case ISD::FREEZE:
5545 case ISD::SADDSAT:
5546 case ISD::UADDSAT:
5547 case ISD::SSUBSAT:
5548 case ISD::USUBSAT:
5549 case ISD::MULHU:
5550 case ISD::MULHS:
5551 case ISD::SMIN:
5552 case ISD::SMAX:
5553 case ISD::UMIN:
5554 case ISD::UMAX:
5555 case ISD::AND:
5556 case ISD::XOR:
5557 case ISD::ROTL:
5558 case ISD::ROTR:
5559 case ISD::FSHL:
5560 case ISD::FSHR:
5561 case ISD::BSWAP:
5562 case ISD::CTPOP:
5563 case ISD::BITREVERSE:
5564 case ISD::PARITY:
5565 case ISD::SIGN_EXTEND:
5566 case ISD::TRUNCATE:
5570 case ISD::BITCAST:
5571 case ISD::BUILD_VECTOR:
5572 case ISD::BUILD_PAIR:
5573 case ISD::SPLAT_VECTOR:
5574 return false;
5575
5576 case ISD::SELECT_CC:
5577 case ISD::SETCC: {
5578 // Integer setcc cannot create undef or poison.
5579 if (Op.getOperand(0).getValueType().isInteger())
5580 return false;
5581
5582 // FP compares are more complicated. They can create poison for nan/infinity
5583 // based on options and flags. The options and flags also cause special
5584 // nonan condition codes to be used. Those condition codes may be preserved
5585 // even if the nonan flag is dropped somewhere.
5586 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
5587 ISD::CondCode CCCode = cast<CondCodeSDNode>(Op.getOperand(CCOp))->get();
5588 if (((unsigned)CCCode & 0x10U))
5589 return true;
5590
5592 return Options.NoNaNsFPMath || Options.NoInfsFPMath;
5593 }
5594
5595 case ISD::OR:
5596 case ISD::ZERO_EXTEND:
5597 case ISD::ADD:
5598 case ISD::SUB:
5599 case ISD::MUL:
5600 // No poison except from flags (which is handled above)
5601 return false;
5602
5603 case ISD::SHL:
5604 case ISD::SRL:
5605 case ISD::SRA:
5606 // If the max shift amount isn't in range, then the shift can
5607 // create poison.
5608 return !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedElts,
5609 PoisonOnly, Depth + 1) ||
5610 !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);
5611
5613 // Check if we demand any upper (undef) elements.
5614 return !PoisonOnly && DemandedElts.ugt(1);
5615
5618 // Ensure that the element index is in bounds.
5619 EVT VecVT = Op.getOperand(0).getValueType();
5620 SDValue Idx = Op.getOperand(Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
5622 Depth + 1)) {
5623 KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);
5624 return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
5625 }
5626 return true;
5627 }
5628
5629 case ISD::VECTOR_SHUFFLE: {
5630 // Check for any demanded shuffle element that is undef.
5631 auto *SVN = cast<ShuffleVectorSDNode>(Op);
5632 for (auto [Idx, Elt] : enumerate(SVN->getMask()))
5633 if (Elt < 0 && DemandedElts[Idx])
5634 return true;
5635 return false;
5636 }
5637
5638 default:
5639 // Allow the target to implement this method for its nodes.
5640 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5641 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5643 Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth);
5644 break;
5645 }
5646
5647 // Be conservative and return true.
5648 return true;
5649}
5650
5651bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
5652 unsigned Opcode = Op.getOpcode();
5653 if (Opcode == ISD::OR)
5654 return Op->getFlags().hasDisjoint() ||
5655 haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));
5656 if (Opcode == ISD::XOR)
5657 return !NoWrap && isMinSignedConstant(Op.getOperand(1));
5658 return false;
5659}
5660
5662 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Op.getOperand(1)) &&
5663 (Op.getOpcode() == ISD::ADD || isADDLike(Op));
5664}
5665
5666bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5667 // If we're told that NaNs won't happen, assume they won't.
5668 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5669 return true;
5670
5671 if (Depth >= MaxRecursionDepth)
5672 return false; // Limit search depth.
5673
5674 // If the value is a constant, we can obviously see if it is a NaN or not.
5675 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
5676 return !C->getValueAPF().isNaN() ||
5677 (SNaN && !C->getValueAPF().isSignaling());
5678 }
5679
5680 unsigned Opcode = Op.getOpcode();
5681 switch (Opcode) {
5682 case ISD::FADD:
5683 case ISD::FSUB:
5684 case ISD::FMUL:
5685 case ISD::FDIV:
5686 case ISD::FREM:
5687 case ISD::FSIN:
5688 case ISD::FCOS:
5689 case ISD::FTAN:
5690 case ISD::FASIN:
5691 case ISD::FACOS:
5692 case ISD::FATAN:
5693 case ISD::FATAN2:
5694 case ISD::FSINH:
5695 case ISD::FCOSH:
5696 case ISD::FTANH:
5697 case ISD::FMA:
5698 case ISD::FMAD: {
5699 if (SNaN)
5700 return true;
5701 // TODO: Need isKnownNeverInfinity
5702 return false;
5703 }
5704 case ISD::FCANONICALIZE:
5705 case ISD::FEXP:
5706 case ISD::FEXP2:
5707 case ISD::FEXP10:
5708 case ISD::FTRUNC:
5709 case ISD::FFLOOR:
5710 case ISD::FCEIL:
5711 case ISD::FROUND:
5712 case ISD::FROUNDEVEN:
5713 case ISD::LROUND:
5714 case ISD::LLROUND:
5715 case ISD::FRINT:
5716 case ISD::LRINT:
5717 case ISD::LLRINT:
5718 case ISD::FNEARBYINT:
5719 case ISD::FLDEXP: {
5720 if (SNaN)
5721 return true;
5722 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5723 }
5724 case ISD::FABS:
5725 case ISD::FNEG:
5726 case ISD::FCOPYSIGN: {
5727 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5728 }
5729 case ISD::SELECT:
5730 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5731 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5732 case ISD::FP_EXTEND:
5733 case ISD::FP_ROUND: {
5734 if (SNaN)
5735 return true;
5736 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5737 }
5738 case ISD::SINT_TO_FP:
5739 case ISD::UINT_TO_FP:
5740 return true;
5741 case ISD::FSQRT: // Need is known positive
5742 case ISD::FLOG:
5743 case ISD::FLOG2:
5744 case ISD::FLOG10:
5745 case ISD::FPOWI:
5746 case ISD::FPOW: {
5747 if (SNaN)
5748 return true;
5749 // TODO: Refine on operand
5750 return false;
5751 }
5752 case ISD::FMINNUM:
5753 case ISD::FMAXNUM:
5754 case ISD::FMINIMUMNUM:
5755 case ISD::FMAXIMUMNUM: {
5756 // Only one needs to be known not-nan, since it will be returned if the
5757 // other ends up being one.
5758 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
5759 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5760 }
5761 case ISD::FMINNUM_IEEE:
5762 case ISD::FMAXNUM_IEEE: {
5763 if (SNaN)
5764 return true;
5765 // This can return a NaN if either operand is an sNaN, or if both operands
5766 // are NaN.
5767 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
5768 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
5769 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
5770 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
5771 }
5772 case ISD::FMINIMUM:
5773 case ISD::FMAXIMUM: {
5774 // TODO: Does this quiet or return the origina NaN as-is?
5775 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5776 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5777 }
5779 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5780 }
5781 case ISD::BUILD_VECTOR: {
5782 for (const SDValue &Opnd : Op->ops())
5783 if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
5784 return false;
5785 return true;
5786 }
5787 default:
5788 if (Opcode >= ISD::BUILTIN_OP_END ||
5789 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5790 Opcode == ISD::INTRINSIC_W_CHAIN ||
5791 Opcode == ISD::INTRINSIC_VOID) {
5792 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
5793 }
5794
5795 return false;
5796 }
5797}
5798
5800 assert(Op.getValueType().isFloatingPoint() &&
5801 "Floating point type expected");
5802
5803 // If the value is a constant, we can obviously see if it is a zero or not.
5805 Op, [](ConstantFPSDNode *C) { return !C->isZero(); });
5806}
5807
5809 if (Depth >= MaxRecursionDepth)
5810 return false; // Limit search depth.
5811
5812 assert(!Op.getValueType().isFloatingPoint() &&
5813 "Floating point types unsupported - use isKnownNeverZeroFloat");
5814
5815 // If the value is a constant, we can obviously see if it is a zero or not.
5817 [](ConstantSDNode *C) { return !C->isZero(); }))
5818 return true;
5819
5820 // TODO: Recognize more cases here. Most of the cases are also incomplete to
5821 // some degree.
5822 switch (Op.getOpcode()) {
5823 default:
5824 break;
5825
5826 case ISD::OR:
5827 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5828 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5829
5830 case ISD::VSELECT:
5831 case ISD::SELECT:
5832 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5833 isKnownNeverZero(Op.getOperand(2), Depth + 1);
5834
5835 case ISD::SHL: {
5836 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5837 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5838 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5839 // 1 << X is never zero.
5840 if (ValKnown.One[0])
5841 return true;
5842 // If max shift cnt of known ones is non-zero, result is non-zero.
5843 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5844 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5845 !ValKnown.One.shl(MaxCnt).isZero())
5846 return true;
5847 break;
5848 }
5849 case ISD::UADDSAT:
5850 case ISD::UMAX:
5851 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5852 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5853
5854 // For smin/smax: If either operand is known negative/positive
5855 // respectively we don't need the other to be known at all.
5856 case ISD::SMAX: {
5857 KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);
5858 if (Op1.isStrictlyPositive())
5859 return true;
5860
5861 KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);
5862 if (Op0.isStrictlyPositive())
5863 return true;
5864
5865 if (Op1.isNonZero() && Op0.isNonZero())
5866 return true;
5867
5868 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5869 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5870 }
5871 case ISD::SMIN: {
5872 KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);
5873 if (Op1.isNegative())
5874 return true;
5875
5876 KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);
5877 if (Op0.isNegative())
5878 return true;
5879
5880 if (Op1.isNonZero() && Op0.isNonZero())
5881 return true;
5882
5883 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5884 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5885 }
5886 case ISD::UMIN:
5887 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5888 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5889
5890 case ISD::ROTL:
5891 case ISD::ROTR:
5892 case ISD::BITREVERSE:
5893 case ISD::BSWAP:
5894 case ISD::CTPOP:
5895 case ISD::ABS:
5896 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5897
5898 case ISD::SRA:
5899 case ISD::SRL: {
5900 if (Op->getFlags().hasExact())
5901 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5902 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5903 if (ValKnown.isNegative())
5904 return true;
5905 // If max shift cnt of known ones is non-zero, result is non-zero.
5906 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5907 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5908 !ValKnown.One.lshr(MaxCnt).isZero())
5909 return true;
5910 break;
5911 }
5912 case ISD::UDIV:
5913 case ISD::SDIV:
5914 // div exact can only produce a zero if the dividend is zero.
5915 // TODO: For udiv this is also true if Op1 u<= Op0
5916 if (Op->getFlags().hasExact())
5917 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5918 break;
5919
5920 case ISD::ADD:
5921 if (Op->getFlags().hasNoUnsignedWrap())
5922 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5923 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5924 return true;
5925 // TODO: There are a lot more cases we can prove for add.
5926 break;
5927
5928 case ISD::SUB: {
5929 if (isNullConstant(Op.getOperand(0)))
5930 return isKnownNeverZero(Op.getOperand(1), Depth + 1);
5931
5932 std::optional<bool> ne =
5933 KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1),
5934 computeKnownBits(Op.getOperand(1), Depth + 1));
5935 return ne && *ne;
5936 }
5937
5938 case ISD::MUL:
5939 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5940 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5941 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5942 return true;
5943 break;
5944
5945 case ISD::ZERO_EXTEND:
5946 case ISD::SIGN_EXTEND:
5947 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5948 case ISD::VSCALE: {
5950 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
5951 ConstantRange CR =
5952 getVScaleRange(&F, Op.getScalarValueSizeInBits()).multiply(Multiplier);
5953 if (!CR.contains(APInt(CR.getBitWidth(), 0)))
5954 return true;
5955 break;
5956 }
5957 }
5958
5960}
5961
5963 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Op, true))
5964 return !C1->isNegative();
5965
5966 return Op.getOpcode() == ISD::FABS;
5967}
5968
5970 // Check the obvious case.
5971 if (A == B) return true;
5972
5973 // For negative and positive zero.
5974 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
5975 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
5976 if (CA->isZero() && CB->isZero()) return true;
5977
5978 // Otherwise they may not be equal.
5979 return false;
5980}
5981
5982// Only bits set in Mask must be negated, other bits may be arbitrary.
5984 if (isBitwiseNot(V, AllowUndefs))
5985 return V.getOperand(0);
5986
5987 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
5988 // bits in the non-extended part.
5989 ConstantSDNode *MaskC = isConstOrConstSplat(Mask);
5990 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5991 return SDValue();
5992 SDValue ExtArg = V.getOperand(0);
5993 if (ExtArg.getScalarValueSizeInBits() >=
5994 MaskC->getAPIntValue().getActiveBits() &&
5995 isBitwiseNot(ExtArg, AllowUndefs) &&
5996 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&
5997 ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())
5998 return ExtArg.getOperand(0).getOperand(0);
5999 return SDValue();
6000}
6001
6003 // Match masked merge pattern (X & ~M) op (Y & M)
6004 // Including degenerate case (X & ~M) op M
6005 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6006 SDValue Other) {
6007 if (SDValue NotOperand =
6008 getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {
6009 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6010 NotOperand->getOpcode() == ISD::TRUNCATE)
6011 NotOperand = NotOperand->getOperand(0);
6012
6013 if (Other == NotOperand)
6014 return true;
6015 if (Other->getOpcode() == ISD::AND)
6016 return NotOperand == Other->getOperand(0) ||
6017 NotOperand == Other->getOperand(1);
6018 }
6019 return false;
6020 };
6021
6022 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6023 A = A->getOperand(0);
6024
6025 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6026 B = B->getOperand(0);
6027
6028 if (A->getOpcode() == ISD::AND)
6029 return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||
6030 MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);
6031 return false;
6032}
6033
6034// FIXME: unify with llvm::haveNoCommonBitsSet.
6036 assert(A.getValueType() == B.getValueType() &&
6037 "Values must have the same type");
6040 return true;
6043}
6044
6045static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6046 SelectionDAG &DAG) {
6047 if (cast<ConstantSDNode>(Step)->isZero())
6048 return DAG.getConstant(0, DL, VT);
6049
6050 return SDValue();
6051}
6052
6055 SelectionDAG &DAG) {
6056 int NumOps = Ops.size();
6057 assert(NumOps != 0 && "Can't build an empty vector!");
6058 assert(!VT.isScalableVector() &&
6059 "BUILD_VECTOR cannot be used with scalable types");
6060 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6061 "Incorrect element count in BUILD_VECTOR!");
6062
6063 // BUILD_VECTOR of UNDEFs is UNDEF.
6064 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
6065 return DAG.getUNDEF(VT);
6066
6067 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6068 SDValue IdentitySrc;
6069 bool IsIdentity = true;
6070 for (int i = 0; i != NumOps; ++i) {
6071 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6072 Ops[i].getOperand(0).getValueType() != VT ||
6073 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
6074 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
6075 Ops[i].getConstantOperandAPInt(1) != i) {
6076 IsIdentity = false;
6077 break;
6078 }
6079 IdentitySrc = Ops[i].getOperand(0);
6080 }
6081 if (IsIdentity)
6082 return IdentitySrc;
6083
6084 return SDValue();
6085}
6086
6087/// Try to simplify vector concatenation to an input value, undef, or build
6088/// vector.
6091 SelectionDAG &DAG) {
6092 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6093 assert(llvm::all_of(Ops,
6094 [Ops](SDValue Op) {
6095 return Ops[0].getValueType() == Op.getValueType();
6096 }) &&
6097 "Concatenation of vectors with inconsistent value types!");
6098 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6099 VT.getVectorElementCount() &&
6100 "Incorrect element count in vector concatenation!");
6101
6102 if (Ops.size() == 1)
6103 return Ops[0];
6104
6105 // Concat of UNDEFs is UNDEF.
6106 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
6107 return DAG.getUNDEF(VT);
6108
6109 // Scan the operands and look for extract operations from a single source
6110 // that correspond to insertion at the same location via this concatenation:
6111 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6112 SDValue IdentitySrc;
6113 bool IsIdentity = true;
6114 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6115 SDValue Op = Ops[i];
6116 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6117 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6118 Op.getOperand(0).getValueType() != VT ||
6119 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
6120 Op.getConstantOperandVal(1) != IdentityIndex) {
6121 IsIdentity = false;
6122 break;
6123 }
6124 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6125 "Unexpected identity source vector for concat of extracts");
6126 IdentitySrc = Op.getOperand(0);
6127 }
6128 if (IsIdentity) {
6129 assert(IdentitySrc && "Failed to set source vector of extracts");
6130 return IdentitySrc;
6131 }
6132
6133 // The code below this point is only designed to work for fixed width
6134 // vectors, so we bail out for now.
6135 if (VT.isScalableVector())
6136 return SDValue();
6137
6138 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
6139 // simplified to one big BUILD_VECTOR.
6140 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6141 EVT SVT = VT.getScalarType();
6143 for (SDValue Op : Ops) {
6144 EVT OpVT = Op.getValueType();
6145 if (Op.isUndef())
6146 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
6147 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
6148 Elts.append(Op->op_begin(), Op->op_end());
6149 else
6150 return SDValue();
6151 }
6152
6153 // BUILD_VECTOR requires all inputs to be of the same type, find the
6154 // maximum type and extend them all.
6155 for (SDValue Op : Elts)
6156 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
6157
6158 if (SVT.bitsGT(VT.getScalarType())) {
6159 for (SDValue &Op : Elts) {
6160 if (Op.isUndef())
6161 Op = DAG.getUNDEF(SVT);
6162 else
6163 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
6164 ? DAG.getZExtOrTrunc(Op, DL, SVT)
6165 : DAG.getSExtOrTrunc(Op, DL, SVT);
6166 }
6167 }
6168
6169 SDValue V = DAG.getBuildVector(VT, DL, Elts);
6170 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
6171 return V;
6172}
6173
6174/// Gets or creates the specified node.
6175SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
6176 SDVTList VTs = getVTList(VT);
6178 AddNodeIDNode(ID, Opcode, VTs, {});
6179 void *IP = nullptr;
6180 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6181 return SDValue(E, 0);
6182
6183 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6184 CSEMap.InsertNode(N, IP);
6185
6186 InsertNode(N);
6187 SDValue V = SDValue(N, 0);
6188 NewSDValueDbgMsg(V, "Creating new node: ", this);
6189 return V;
6190}
6191
6192SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6193 SDValue N1) {
6194 SDNodeFlags Flags;
6195 if (Inserter)
6196 Flags = Inserter->getFlags();
6197 return getNode(Opcode, DL, VT, N1, Flags);
6198}
6199
6200SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6201 SDValue N1, const SDNodeFlags Flags) {
6202 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
6203
6204 // Constant fold unary operations with a vector integer or float operand.
6205 switch (Opcode) {
6206 default:
6207 // FIXME: Entirely reasonable to perform folding of other unary
6208 // operations here as the need arises.
6209 break;
6210 case ISD::FNEG:
6211 case ISD::FABS:
6212 case ISD::FCEIL:
6213 case ISD::FTRUNC:
6214 case ISD::FFLOOR:
6215 case ISD::FP_EXTEND:
6216 case ISD::FP_TO_SINT:
6217 case ISD::FP_TO_UINT:
6218 case ISD::FP_TO_FP16:
6219 case ISD::FP_TO_BF16:
6220 case ISD::TRUNCATE:
6221 case ISD::ANY_EXTEND:
6222 case ISD::ZERO_EXTEND:
6223 case ISD::SIGN_EXTEND:
6224 case ISD::UINT_TO_FP:
6225 case ISD::SINT_TO_FP:
6226 case ISD::FP16_TO_FP:
6227 case ISD::BF16_TO_FP:
6228 case ISD::BITCAST:
6229 case ISD::ABS:
6230 case ISD::BITREVERSE:
6231 case ISD::BSWAP:
6232 case ISD::CTLZ:
6234 case ISD::CTTZ:
6236 case ISD::CTPOP:
6237 case ISD::STEP_VECTOR: {
6238 SDValue Ops = {N1};
6239 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
6240 return Fold;
6241 }
6242 }
6243
6244 unsigned OpOpcode = N1.getNode()->getOpcode();
6245 switch (Opcode) {
6246 case ISD::STEP_VECTOR:
6247 assert(VT.isScalableVector() &&
6248 "STEP_VECTOR can only be used with scalable types");
6249 assert(OpOpcode == ISD::TargetConstant &&
6250 VT.getVectorElementType() == N1.getValueType() &&
6251 "Unexpected step operand");
6252 break;
6253 case ISD::FREEZE:
6254 assert(VT == N1.getValueType() && "Unexpected VT!");
6255 if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false,
6256 /*Depth*/ 1))
6257 return N1;
6258 break;
6259 case ISD::TokenFactor:
6260 case ISD::MERGE_VALUES:
6262 return N1; // Factor, merge or concat of one node? No need.
6263 case ISD::BUILD_VECTOR: {
6264 // Attempt to simplify BUILD_VECTOR.
6265 SDValue Ops[] = {N1};
6266 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6267 return V;
6268 break;
6269 }
6270 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
6271 case ISD::FP_EXTEND:
6273 "Invalid FP cast!");
6274 if (N1.getValueType() == VT) return N1; // noop conversion.
6275 assert((!VT.isVector() || VT.getVectorElementCount() ==
6277 "Vector element count mismatch!");
6278 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
6279 if (N1.isUndef())
6280 return getUNDEF(VT);
6281 break;
6282 case ISD::FP_TO_SINT:
6283 case ISD::FP_TO_UINT:
6284 if (N1.isUndef())
6285 return getUNDEF(VT);
6286 break;
6287 case ISD::SINT_TO_FP:
6288 case ISD::UINT_TO_FP:
6289 // [us]itofp(undef) = 0, because the result value is bounded.
6290 if (N1.isUndef())
6291 return getConstantFP(0.0, DL, VT);
6292 break;
6293 case ISD::SIGN_EXTEND:
6294 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6295 "Invalid SIGN_EXTEND!");
6296 assert(VT.isVector() == N1.getValueType().isVector() &&
6297 "SIGN_EXTEND result type type should be vector iff the operand "
6298 "type is vector!");
6299 if (N1.getValueType() == VT) return N1; // noop extension
6300 assert((!VT.isVector() || VT.getVectorElementCount() ==
6302 "Vector element count mismatch!");
6303 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
6304 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
6305 SDNodeFlags Flags;
6306 if (OpOpcode == ISD::ZERO_EXTEND)
6307 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6308 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
6309 }
6310 if (OpOpcode == ISD::UNDEF)
6311 // sext(undef) = 0, because the top bits will all be the same.
6312 return getConstant(0, DL, VT);
6313 break;
6314 case ISD::ZERO_EXTEND:
6315 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6316 "Invalid ZERO_EXTEND!");
6317 assert(VT.isVector() == N1.getValueType().isVector() &&
6318 "ZERO_EXTEND result type type should be vector iff the operand "
6319 "type is vector!");
6320 if (N1.getValueType() == VT) return N1; // noop extension
6321 assert((!VT.isVector() || VT.getVectorElementCount() ==
6323 "Vector element count mismatch!");
6324 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
6325 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
6326 SDNodeFlags Flags;
6327 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6328 return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);
6329 }
6330 if (OpOpcode == ISD::UNDEF)
6331 // zext(undef) = 0, because the top bits will be zero.
6332 return getConstant(0, DL, VT);
6333
6334 // Skip unnecessary zext_inreg pattern:
6335 // (zext (trunc x)) -> x iff the upper bits are known zero.
6336 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
6337 // use to recognise zext_inreg patterns.
6338 if (OpOpcode == ISD::TRUNCATE) {
6339 SDValue OpOp = N1.getOperand(0);
6340 if (OpOp.getValueType() == VT) {
6341 if (OpOp.getOpcode() != ISD::AND) {
6344 if (MaskedValueIsZero(OpOp, HiBits)) {
6345 transferDbgValues(N1, OpOp);
6346 return OpOp;
6347 }
6348 }
6349 }
6350 }
6351 break;
6352 case ISD::ANY_EXTEND:
6353 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6354 "Invalid ANY_EXTEND!");
6355 assert(VT.isVector() == N1.getValueType().isVector() &&
6356 "ANY_EXTEND result type type should be vector iff the operand "
6357 "type is vector!");
6358 if (N1.getValueType() == VT) return N1; // noop extension
6359 assert((!VT.isVector() || VT.getVectorElementCount() ==
6361 "Vector element count mismatch!");
6362 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
6363
6364 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6365 OpOpcode == ISD::ANY_EXTEND) {
6366 SDNodeFlags Flags;
6367 if (OpOpcode == ISD::ZERO_EXTEND)
6368 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6369 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
6370 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
6371 }
6372 if (OpOpcode == ISD::UNDEF)
6373 return getUNDEF(VT);
6374
6375 // (ext (trunc x)) -> x
6376 if (OpOpcode == ISD::TRUNCATE) {
6377 SDValue OpOp = N1.getOperand(0);
6378 if (OpOp.getValueType() == VT) {
6379 transferDbgValues(N1, OpOp);
6380 return OpOp;
6381 }
6382 }
6383 break;
6384 case ISD::TRUNCATE:
6385 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6386 "Invalid TRUNCATE!");
6387 assert(VT.isVector() == N1.getValueType().isVector() &&
6388 "TRUNCATE result type type should be vector iff the operand "
6389 "type is vector!");
6390 if (N1.getValueType() == VT) return N1; // noop truncate
6391 assert((!VT.isVector() || VT.getVectorElementCount() ==
6393 "Vector element count mismatch!");
6394 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
6395 if (OpOpcode == ISD::TRUNCATE)
6396 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
6397 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6398 OpOpcode == ISD::ANY_EXTEND) {
6399 // If the source is smaller than the dest, we still need an extend.
6401 VT.getScalarType()))
6402 return getNode(OpOpcode, DL, VT, N1.getOperand(0));
6403 if (N1.getOperand(0).getValueType().bitsGT(VT))
6404 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
6405 return N1.getOperand(0);
6406 }
6407 if (OpOpcode == ISD::UNDEF)
6408 return getUNDEF(VT);
6409 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
6410 return getVScale(DL, VT,
6412 break;
6416 assert(VT.isVector() && "This DAG node is restricted to vector types.");
6417 assert(N1.getValueType().bitsLE(VT) &&
6418 "The input must be the same size or smaller than the result.");
6421 "The destination vector type must have fewer lanes than the input.");
6422 break;
6423 case ISD::ABS:
6424 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
6425 if (OpOpcode == ISD::UNDEF)
6426 return getConstant(0, DL, VT);
6427 break;
6428 case ISD::BSWAP:
6429 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
6430 assert((VT.getScalarSizeInBits() % 16 == 0) &&
6431 "BSWAP types must be a multiple of 16 bits!");
6432 if (OpOpcode == ISD::UNDEF)
6433 return getUNDEF(VT);
6434 // bswap(bswap(X)) -> X.
6435 if (OpOpcode == ISD::BSWAP)
6436 return N1.getOperand(0);
6437 break;
6438 case ISD::BITREVERSE:
6439 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
6440 if (OpOpcode == ISD::UNDEF)
6441 return getUNDEF(VT);
6442 break;
6443 case ISD::BITCAST:
6445 "Cannot BITCAST between types of different sizes!");
6446 if (VT == N1.getValueType()) return N1; // noop conversion.
6447 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
6448 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));
6449 if (OpOpcode == ISD::UNDEF)
6450 return getUNDEF(VT);
6451 break;
6453 assert(VT.isVector() && !N1.getValueType().isVector() &&
6454 (VT.getVectorElementType() == N1.getValueType() ||
6456 N1.getValueType().isInteger() &&
6458 "Illegal SCALAR_TO_VECTOR node!");
6459 if (OpOpcode == ISD::UNDEF)
6460 return getUNDEF(VT);
6461 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
6462 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
6463 isa<ConstantSDNode>(N1.getOperand(1)) &&
6464 N1.getConstantOperandVal(1) == 0 &&
6465 N1.getOperand(0).getValueType() == VT)
6466 return N1.getOperand(0);
6467 break;
6468 case ISD::FNEG:
6469 // Negation of an unknown bag of bits is still completely undefined.
6470 if (OpOpcode == ISD::UNDEF)
6471 return getUNDEF(VT);
6472
6473 if (OpOpcode == ISD::FNEG) // --X -> X
6474 return N1.getOperand(0);
6475 break;
6476 case ISD::FABS:
6477 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
6478 return getNode(ISD::FABS, DL, VT, N1.getOperand(0));
6479 break;
6480 case ISD::VSCALE:
6481 assert(VT == N1.getValueType() && "Unexpected VT!");
6482 break;
6483 case ISD::CTPOP:
6484 if (N1.getValueType().getScalarType() == MVT::i1)
6485 return N1;
6486 break;
6487 case ISD::CTLZ:
6488 case ISD::CTTZ:
6489 if (N1.getValueType().getScalarType() == MVT::i1)
6490 return getNOT(DL, N1, N1.getValueType());
6491 break;
6492 case ISD::VECREDUCE_ADD:
6493 if (N1.getValueType().getScalarType() == MVT::i1)
6494 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);
6495 break;
6498 if (N1.getValueType().getScalarType() == MVT::i1)
6499 return getNode(ISD::VECREDUCE_OR, DL, VT, N1);
6500 break;
6503 if (N1.getValueType().getScalarType() == MVT::i1)
6504 return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
6505 break;
6506 case ISD::SPLAT_VECTOR:
6507 assert(VT.isVector() && "Wrong return type!");
6508 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
6509 // that for now.
6511 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
6513 N1.getValueType().isInteger() &&
6515 "Wrong operand type!");
6516 break;
6517 }
6518
6519 SDNode *N;
6520 SDVTList VTs = getVTList(VT);
6521 SDValue Ops[] = {N1};
6522 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
6524 AddNodeIDNode(ID, Opcode, VTs, Ops);
6525 void *IP = nullptr;
6526 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
6527 E->intersectFlagsWith(Flags);
6528 return SDValue(E, 0);
6529 }
6530
6531 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6532 N->setFlags(Flags);
6533 createOperands(N, Ops);
6534 CSEMap.InsertNode(N, IP);
6535 } else {
6536 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6537 createOperands(N, Ops);
6538 }
6539
6540 InsertNode(N);
6541 SDValue V = SDValue(N, 0);
6542 NewSDValueDbgMsg(V, "Creating new node: ", this);
6543 return V;
6544}
6545
6546static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
6547 const APInt &C2) {
6548 switch (Opcode) {
6549 case ISD::ADD: return C1 + C2;
6550 case ISD::SUB: return C1 - C2;
6551 case ISD::MUL: return C1 * C2;
6552 case ISD::AND: return C1 & C2;
6553 case ISD::OR: return C1 | C2;
6554 case ISD::XOR: return C1 ^ C2;
6555 case ISD::SHL: return C1 << C2;
6556 case ISD::SRL: return C1.lshr(C2);
6557 case ISD::SRA: return C1.ashr(C2);
6558 case ISD::ROTL: return C1.rotl(C2);
6559 case ISD::ROTR: return C1.rotr(C2);
6560 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
6561 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
6562 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
6563 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
6564 case ISD::SADDSAT: return C1.sadd_sat(C2);
6565 case ISD::UADDSAT: return C1.uadd_sat(C2);
6566 case ISD::SSUBSAT: return C1.ssub_sat(C2);
6567 case ISD::USUBSAT: return C1.usub_sat(C2);
6568 case ISD::SSHLSAT: return C1.sshl_sat(C2);
6569 case ISD::USHLSAT: return C1.ushl_sat(C2);
6570 case ISD::UDIV:
6571 if (!C2.getBoolValue())
6572 break;
6573 return C1.udiv(C2);
6574 case ISD::UREM:
6575 if (!C2.getBoolValue())
6576 break;
6577 return C1.urem(C2);
6578 case ISD::SDIV:
6579 if (!C2.getBoolValue())
6580 break;
6581 return C1.sdiv(C2);
6582 case ISD::SREM:
6583 if (!C2.getBoolValue())
6584 break;
6585 return C1.srem(C2);
6586 case ISD::AVGFLOORS:
6587 return APIntOps::avgFloorS(C1, C2);
6588 case ISD::AVGFLOORU:
6589 return APIntOps::avgFloorU(C1, C2);
6590 case ISD::AVGCEILS:
6591 return APIntOps::avgCeilS(C1, C2);
6592 case ISD::AVGCEILU:
6593 return APIntOps::avgCeilU(C1, C2);
6594 case ISD::ABDS:
6595 return APIntOps::abds(C1, C2);
6596 case ISD::ABDU:
6597 return APIntOps::abdu(C1, C2);
6598 case ISD::MULHS:
6599 return APIntOps::mulhs(C1, C2);
6600 case ISD::MULHU:
6601 return APIntOps::mulhu(C1, C2);
6602 }
6603 return std::nullopt;
6604}
6605// Handle constant folding with UNDEF.
6606// TODO: Handle more cases.
6607static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6608 bool IsUndef1, const APInt &C2,
6609 bool IsUndef2) {
6610 if (!(IsUndef1 || IsUndef2))
6611 return FoldValue(Opcode, C1, C2);
6612
6613 // Fold and(x, undef) -> 0
6614 // Fold mul(x, undef) -> 0
6615 if (Opcode == ISD::AND || Opcode == ISD::MUL)
6616 return APInt::getZero(C1.getBitWidth());
6617
6618 return std::nullopt;
6619}
6620
6622 const GlobalAddressSDNode *GA,
6623 const SDNode *N2) {
6624 if (GA->getOpcode() != ISD::GlobalAddress)
6625 return SDValue();
6626 if (!TLI->isOffsetFoldingLegal(GA))
6627 return SDValue();
6628 auto *C2 = dyn_cast<ConstantSDNode>(N2);
6629 if (!C2)
6630 return SDValue();
6631 int64_t Offset = C2->getSExtValue();
6632 switch (Opcode) {
6633 case ISD::ADD: break;
6634 case ISD::SUB: Offset = -uint64_t(Offset); break;
6635 default: return SDValue();
6636 }
6637 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
6638 GA->getOffset() + uint64_t(Offset));
6639}
6640
6641bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6642 switch (Opcode) {
6643 case ISD::SDIV:
6644 case ISD::UDIV:
6645 case ISD::SREM:
6646 case ISD::UREM: {
6647 // If a divisor is zero/undef or any element of a divisor vector is
6648 // zero/undef, the whole op is undef.
6649 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6650 SDValue Divisor = Ops[1];
6651 if (Divisor.isUndef() || isNullConstant(Divisor))
6652 return true;
6653
6654 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
6655 llvm::any_of(Divisor->op_values(),
6656 [](SDValue V) { return V.isUndef() ||
6657 isNullConstant(V); });
6658 // TODO: Handle signed overflow.
6659 }
6660 // TODO: Handle oversized shifts.
6661 default:
6662 return false;
6663 }
6664}
6665
6667 EVT VT, ArrayRef<SDValue> Ops,
6668 SDNodeFlags Flags) {
6669 // If the opcode is a target-specific ISD node, there's nothing we can
6670 // do here and the operand rules may not line up with the below, so
6671 // bail early.
6672 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6673 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6674 // foldCONCAT_VECTORS in getNode before this is called.
6675 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6676 return SDValue();
6677
6678 unsigned NumOps = Ops.size();
6679 if (NumOps == 0)
6680 return SDValue();
6681
6682 if (isUndef(Opcode, Ops))
6683 return getUNDEF(VT);
6684
6685 // Handle unary special cases.
6686 if (NumOps == 1) {
6687 SDValue N1 = Ops[0];
6688
6689 // Constant fold unary operations with an integer constant operand. Even
6690 // opaque constant will be folded, because the folding of unary operations
6691 // doesn't create new constants with different values. Nevertheless, the
6692 // opaque flag is preserved during folding to prevent future folding with
6693 // other constants.
6694 if (auto *C = dyn_cast<ConstantSDNode>(N1)) {
6695 const APInt &Val = C->getAPIntValue();
6696 switch (Opcode) {
6697 case ISD::SIGN_EXTEND:
6698 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6699 C->isTargetOpcode(), C->isOpaque());
6700 case ISD::TRUNCATE:
6701 if (C->isOpaque())
6702 break;
6703 [[fallthrough]];
6704 case ISD::ZERO_EXTEND:
6705 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6706 C->isTargetOpcode(), C->isOpaque());
6707 case ISD::ANY_EXTEND:
6708 // Some targets like RISCV prefer to sign extend some types.
6709 if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))
6710 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6711 C->isTargetOpcode(), C->isOpaque());
6712 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6713 C->isTargetOpcode(), C->isOpaque());
6714 case ISD::ABS:
6715 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
6716 C->isOpaque());
6717 case ISD::BITREVERSE:
6718 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
6719 C->isOpaque());
6720 case ISD::BSWAP:
6721 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
6722 C->isOpaque());
6723 case ISD::CTPOP:
6724 return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),
6725 C->isOpaque());
6726 case ISD::CTLZ:
6728 return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),
6729 C->isOpaque());
6730 case ISD::CTTZ:
6732 return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
6733 C->isOpaque());
6734 case ISD::UINT_TO_FP:
6735 case ISD::SINT_TO_FP: {
6737 (void)FPV.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,
6739 return getConstantFP(FPV, DL, VT);
6740 }
6741 case ISD::FP16_TO_FP:
6742 case ISD::BF16_TO_FP: {
6743 bool Ignored;
6744 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6745 : APFloat::BFloat(),
6746 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
6747
6748 // This can return overflow, underflow, or inexact; we don't care.
6749 // FIXME need to be more flexible about rounding mode.
6751 &Ignored);
6752 return getConstantFP(FPV, DL, VT);
6753 }
6754 case ISD::STEP_VECTOR:
6755 if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))
6756 return V;
6757 break;
6758 case ISD::BITCAST:
6759 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
6760 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6761 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
6762 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6763 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
6764 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6765 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
6766 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
6767 break;
6768 }
6769 }
6770
6771 // Constant fold unary operations with a floating point constant operand.
6772 if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {
6773 APFloat V = C->getValueAPF(); // make copy
6774 switch (Opcode) {
6775 case ISD::FNEG:
6776 V.changeSign();
6777 return getConstantFP(V, DL, VT);
6778 case ISD::FABS:
6779 V.clearSign();
6780 return getConstantFP(V, DL, VT);
6781 case ISD::FCEIL: {
6782 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
6783 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6784 return getConstantFP(V, DL, VT);
6785 return SDValue();
6786 }
6787 case ISD::FTRUNC: {
6788 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
6789 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6790 return getConstantFP(V, DL, VT);
6791 return SDValue();
6792 }
6793 case ISD::FFLOOR: {
6794 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
6795 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6796 return getConstantFP(V, DL, VT);
6797 return SDValue();
6798 }
6799 case ISD::FP_EXTEND: {
6800 bool ignored;
6801 // This can return overflow, underflow, or inexact; we don't care.
6802 // FIXME need to be more flexible about rounding mode.
6803 (void)V.convert(VT.getFltSemantics(), APFloat::rmNearestTiesToEven,
6804 &ignored);
6805 return getConstantFP(V, DL, VT);
6806 }
6807 case ISD::FP_TO_SINT:
6808 case ISD::FP_TO_UINT: {
6809 bool ignored;
6810 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6811 // FIXME need to be more flexible about rounding mode.
6813 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
6814 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6815 break;
6816 return getConstant(IntVal, DL, VT);
6817 }
6818 case ISD::FP_TO_FP16:
6819 case ISD::FP_TO_BF16: {
6820 bool Ignored;
6821 // This can return overflow, underflow, or inexact; we don't care.
6822 // FIXME need to be more flexible about rounding mode.
6823 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6824 : APFloat::BFloat(),
6826 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6827 }
6828 case ISD::BITCAST:
6829 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
6830 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6831 VT);
6832 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
6833 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6834 VT);
6835 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
6836 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6837 VT);
6838 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
6839 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6840 break;
6841 }
6842 }
6843
6844 // Early-out if we failed to constant fold a bitcast.
6845 if (Opcode == ISD::BITCAST)
6846 return SDValue();
6847 }
6848
6849 // Handle binops special cases.
6850 if (NumOps == 2) {
6851 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6852 return CFP;
6853
6854 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6855 if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6856 if (C1->isOpaque() || C2->isOpaque())
6857 return SDValue();
6858
6859 std::optional<APInt> FoldAttempt =
6860 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6861 if (!FoldAttempt)
6862 return SDValue();
6863
6864 SDValue Folded = getConstant(*FoldAttempt, DL, VT);
6865 assert((!Folded || !VT.isVector()) &&
6866 "Can't fold vectors ops with scalar operands");
6867 return Folded;
6868 }
6869 }
6870
6871 // fold (add Sym, c) -> Sym+c
6872 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[0]))
6873 return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());
6874 if (TLI->isCommutativeBinOp(Opcode))
6875 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[1]))
6876 return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());
6877
6878 // fold (sext_in_reg c1) -> c2
6879 if (Opcode == ISD::SIGN_EXTEND_INREG) {
6880 EVT EVT = cast<VTSDNode>(Ops[1])->getVT();
6881
6882 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6883 unsigned FromBits = EVT.getScalarSizeInBits();
6884 Val <<= Val.getBitWidth() - FromBits;
6885 Val.ashrInPlace(Val.getBitWidth() - FromBits);
6886 return getConstant(Val, DL, ConstantVT);
6887 };
6888
6889 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6890 const APInt &Val = C1->getAPIntValue();
6891 return SignExtendInReg(Val, VT);
6892 }
6893
6895 SmallVector<SDValue, 8> ScalarOps;
6896 llvm::EVT OpVT = Ops[0].getOperand(0).getValueType();
6897 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
6898 SDValue Op = Ops[0].getOperand(I);
6899 if (Op.isUndef()) {
6900 ScalarOps.push_back(getUNDEF(OpVT));
6901 continue;
6902 }
6903 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
6904 ScalarOps.push_back(SignExtendInReg(Val, OpVT));
6905 }
6906 return getBuildVector(VT, DL, ScalarOps);
6907 }
6908
6909 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
6910 isa<ConstantSDNode>(Ops[0].getOperand(0)))
6911 return getNode(ISD::SPLAT_VECTOR, DL, VT,
6912 SignExtendInReg(Ops[0].getConstantOperandAPInt(0),
6913 Ops[0].getOperand(0).getValueType()));
6914 }
6915 }
6916
6917 // This is for vector folding only from here on.
6918 if (!VT.isVector())
6919 return SDValue();
6920
6921 ElementCount NumElts = VT.getVectorElementCount();
6922
6923 // See if we can fold through any bitcasted integer ops.
6924 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6925 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6926 (Ops[0].getOpcode() == ISD::BITCAST ||
6927 Ops[1].getOpcode() == ISD::BITCAST)) {
6928 SDValue N1 = peekThroughBitcasts(Ops[0]);
6929 SDValue N2 = peekThroughBitcasts(Ops[1]);
6930 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6931 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6932 if (BV1 && BV2 && N1.getValueType().isInteger() &&
6933 N2.getValueType().isInteger()) {
6934 bool IsLE = getDataLayout().isLittleEndian();
6935 unsigned EltBits = VT.getScalarSizeInBits();
6936 SmallVector<APInt> RawBits1, RawBits2;
6937 BitVector UndefElts1, UndefElts2;
6938 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6939 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6940 SmallVector<APInt> RawBits;
6941 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6942 std::optional<APInt> Fold = FoldValueWithUndef(
6943 Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);
6944 if (!Fold)
6945 break;
6946 RawBits.push_back(*Fold);
6947 }
6948 if (RawBits.size() == NumElts.getFixedValue()) {
6949 // We have constant folded, but we might need to cast this again back
6950 // to the original (possibly legalized) type.
6951 EVT BVVT, BVEltVT;
6952 if (N1.getValueType() == VT) {
6953 BVVT = N1.getValueType();
6954 BVEltVT = BV1->getOperand(0).getValueType();
6955 } else {
6956 BVVT = N2.getValueType();
6957 BVEltVT = BV2->getOperand(0).getValueType();
6958 }
6959 unsigned BVEltBits = BVEltVT.getSizeInBits();
6960 SmallVector<APInt> DstBits;
6961 BitVector DstUndefs;
6963 DstBits, RawBits, DstUndefs,
6964 BitVector(RawBits.size(), false));
6965 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
6966 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6967 if (DstUndefs[I])
6968 continue;
6969 Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);
6970 }
6971 return getBitcast(VT, getBuildVector(BVVT, DL, Ops));
6972 }
6973 }
6974 }
6975 }
6976
6977 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6978 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6979 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6980 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6981 APInt RHSVal;
6982 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) {
6983 APInt NewStep = Opcode == ISD::MUL
6984 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6985 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6986 return getStepVector(DL, VT, NewStep);
6987 }
6988 }
6989
6990 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6991 return !Op.getValueType().isVector() ||
6992 Op.getValueType().getVectorElementCount() == NumElts;
6993 };
6994
6995 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6996 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6997 Op.getOpcode() == ISD::BUILD_VECTOR ||
6998 Op.getOpcode() == ISD::SPLAT_VECTOR;
6999 };
7000
7001 // All operands must be vector types with the same number of elements as
7002 // the result type and must be either UNDEF or a build/splat vector
7003 // or UNDEF scalars.
7004 if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
7005 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
7006 return SDValue();
7007
7008 // If we are comparing vectors, then the result needs to be a i1 boolean that
7009 // is then extended back to the legal result type depending on how booleans
7010 // are represented.
7011 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7012 ISD::NodeType ExtendCode =
7013 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
7016
7017 // Find legal integer scalar type for constant promotion and
7018 // ensure that its scalar size is at least as large as source.
7019 EVT LegalSVT = VT.getScalarType();
7020 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
7021 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
7022 if (LegalSVT.bitsLT(VT.getScalarType()))
7023 return SDValue();
7024 }
7025
7026 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
7027 // only have one operand to check. For fixed-length vector types we may have
7028 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
7029 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
7030
7031 // Constant fold each scalar lane separately.
7032 SmallVector<SDValue, 4> ScalarResults;
7033 for (unsigned I = 0; I != NumVectorElts; I++) {
7034 SmallVector<SDValue, 4> ScalarOps;
7035 for (SDValue Op : Ops) {
7036 EVT InSVT = Op.getValueType().getScalarType();
7037 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
7038 Op.getOpcode() != ISD::SPLAT_VECTOR) {
7039 if (Op.isUndef())
7040 ScalarOps.push_back(getUNDEF(InSVT));
7041 else
7042 ScalarOps.push_back(Op);
7043 continue;
7044 }
7045
7046 SDValue ScalarOp =
7047 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
7048 EVT ScalarVT = ScalarOp.getValueType();
7049
7050 // Build vector (integer) scalar operands may need implicit
7051 // truncation - do this before constant folding.
7052 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
7053 // Don't create illegally-typed nodes unless they're constants or undef
7054 // - if we fail to constant fold we can't guarantee the (dead) nodes
7055 // we're creating will be cleaned up before being visited for
7056 // legalization.
7057 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
7058 !isa<ConstantSDNode>(ScalarOp) &&
7059 TLI->getTypeAction(*getContext(), InSVT) !=
7061 return SDValue();
7062 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
7063 }
7064
7065 ScalarOps.push_back(ScalarOp);
7066 }
7067
7068 // Constant fold the scalar operands.
7069 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
7070
7071 // Scalar folding only succeeded if the result is a constant or UNDEF.
7072 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
7073 ScalarResult.getOpcode() != ISD::ConstantFP)
7074 return SDValue();
7075
7076 // Legalize the (integer) scalar constant if necessary. We only do
7077 // this once we know the folding succeeded, since otherwise we would
7078 // get a node with illegal type which has a user.
7079 if (LegalSVT != SVT)
7080 ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);
7081
7082 ScalarResults.push_back(ScalarResult);
7083 }
7084
7085 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
7086 : getBuildVector(VT, DL, ScalarResults);
7087 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
7088 return V;
7089}
7090
7092 EVT VT, ArrayRef<SDValue> Ops) {
7093 // TODO: Add support for unary/ternary fp opcodes.
7094 if (Ops.size() != 2)
7095 return SDValue();
7096
7097 // TODO: We don't do any constant folding for strict FP opcodes here, but we
7098 // should. That will require dealing with a potentially non-default
7099 // rounding mode, checking the "opStatus" return value from the APFloat
7100 // math calculations, and possibly other variations.
7101 SDValue N1 = Ops[0];
7102 SDValue N2 = Ops[1];
7103 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
7104 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
7105 if (N1CFP && N2CFP) {
7106 APFloat C1 = N1CFP->getValueAPF(); // make copy
7107 const APFloat &C2 = N2CFP->getValueAPF();
7108 switch (Opcode) {
7109 case ISD::FADD:
7111 return getConstantFP(C1, DL, VT);
7112 case ISD::FSUB:
7114 return getConstantFP(C1, DL, VT);
7115 case ISD::FMUL:
7117 return getConstantFP(C1, DL, VT);
7118 case ISD::FDIV:
7120 return getConstantFP(C1, DL, VT);
7121 case ISD::FREM:
7122 C1.mod(C2);
7123 return getConstantFP(C1, DL, VT);
7124 case ISD::FCOPYSIGN:
7125 C1.copySign(C2);
7126 return getConstantFP(C1, DL, VT);
7127 case ISD::FMINNUM:
7128 return getConstantFP(minnum(C1, C2), DL, VT);
7129 case ISD::FMAXNUM:
7130 return getConstantFP(maxnum(C1, C2), DL, VT);
7131 case ISD::FMINIMUM:
7132 return getConstantFP(minimum(C1, C2), DL, VT);
7133 case ISD::FMAXIMUM:
7134 return getConstantFP(maximum(C1, C2), DL, VT);
7135 case ISD::FMINIMUMNUM:
7136 return getConstantFP(minimumnum(C1, C2), DL, VT);
7137 case ISD::FMAXIMUMNUM:
7138 return getConstantFP(maximumnum(C1, C2), DL, VT);
7139 default: break;
7140 }
7141 }
7142 if (N1CFP && Opcode == ISD::FP_ROUND) {
7143 APFloat C1 = N1CFP->getValueAPF(); // make copy
7144 bool Unused;
7145 // This can return overflow, underflow, or inexact; we don't care.
7146 // FIXME need to be more flexible about rounding mode.
7148 &Unused);
7149 return getConstantFP(C1, DL, VT);
7150 }
7151
7152 switch (Opcode) {
7153 case ISD::FSUB:
7154 // -0.0 - undef --> undef (consistent with "fneg undef")
7155 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
7156 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
7157 return getUNDEF(VT);
7158 [[fallthrough]];
7159
7160 case ISD::FADD:
7161 case ISD::FMUL:
7162 case ISD::FDIV:
7163 case ISD::FREM:
7164 // If both operands are undef, the result is undef. If 1 operand is undef,
7165 // the result is NaN. This should match the behavior of the IR optimizer.
7166 if (N1.isUndef() && N2.isUndef())
7167 return getUNDEF(VT);
7168 if (N1.isUndef() || N2.isUndef())
7170 }
7171 return SDValue();
7172}
7173
7175 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
7176
7177 // There's no need to assert on a byte-aligned pointer. All pointers are at
7178 // least byte aligned.
7179 if (A == Align(1))
7180 return Val;
7181
7182 SDVTList VTs = getVTList(Val.getValueType());
7184 AddNodeIDNode(ID, ISD::AssertAlign, VTs, {Val});
7185 ID.AddInteger(A.value());
7186
7187 void *IP = nullptr;
7188 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
7189 return SDValue(E, 0);
7190
7191 auto *N =
7192 newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, A);
7193 createOperands(N, {Val});
7194
7195 CSEMap.InsertNode(N, IP);
7196 InsertNode(N);
7197
7198 SDValue V(N, 0);
7199 NewSDValueDbgMsg(V, "Creating new node: ", this);
7200 return V;
7201}
7202
7203SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7204 SDValue N1, SDValue N2) {
7205 SDNodeFlags Flags;
7206 if (Inserter)
7207 Flags = Inserter->getFlags();
7208 return getNode(Opcode, DL, VT, N1, N2, Flags);
7209}
7210
7212 SDValue &N2) const {
7213 if (!TLI->isCommutativeBinOp(Opcode))
7214 return;
7215
7216 // Canonicalize:
7217 // binop(const, nonconst) -> binop(nonconst, const)
7220 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N1);
7221 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N2);
7222 if ((N1C && !N2C) || (N1CFP && !N2CFP))
7223 std::swap(N1, N2);
7224
7225 // Canonicalize:
7226 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
7227 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
7229 std::swap(N1, N2);
7230}
7231
7232SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7233 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
7235 N2.getOpcode() != ISD::DELETED_NODE &&
7236 "Operand is DELETED_NODE!");
7237
7238 canonicalizeCommutativeBinop(Opcode, N1, N2);
7239
7240 auto *N1C = dyn_cast<ConstantSDNode>(N1);
7241 auto *N2C = dyn_cast<ConstantSDNode>(N2);
7242
7243 // Don't allow undefs in vector splats - we might be returning N2 when folding
7244 // to zero etc.
7245 ConstantSDNode *N2CV =
7246 isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
7247
7248 switch (Opcode) {
7249 default: break;
7250 case ISD::TokenFactor:
7251 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
7252 N2.getValueType() == MVT::Other && "Invalid token factor!");
7253 // Fold trivial token factors.
7254 if (N1.getOpcode() == ISD::EntryToken) return N2;
7255 if (N2.getOpcode() == ISD::EntryToken) return N1;
7256 if (N1 == N2) return N1;
7257 break;
7258 case ISD::BUILD_VECTOR: {
7259 // Attempt to simplify BUILD_VECTOR.
7260 SDValue Ops[] = {N1, N2};
7261 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7262 return V;
7263 break;
7264 }
7265 case ISD::CONCAT_VECTORS: {
7266 SDValue Ops[] = {N1, N2};
7267 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7268 return V;
7269 break;
7270 }
7271 case ISD::AND:
7272 assert(VT.isInteger() && "This operator does not apply to FP types!");
7273 assert(N1.getValueType() == N2.getValueType() &&
7274 N1.getValueType() == VT && "Binary operator types must match!");
7275 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
7276 // worth handling here.
7277 if (N2CV && N2CV->isZero())
7278 return N2;
7279 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
7280 return N1;
7281 break;
7282 case ISD::OR:
7283 case ISD::XOR:
7284 case ISD::ADD:
7285 case ISD::SUB:
7286 assert(VT.isInteger() && "This operator does not apply to FP types!");
7287 assert(N1.getValueType() == N2.getValueType() &&
7288 N1.getValueType() == VT && "Binary operator types must match!");
7289 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
7290 // it's worth handling here.
7291 if (N2CV && N2CV->isZero())
7292 return N1;
7293 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
7294 VT.getVectorElementType() == MVT::i1)
7295 return getNode(ISD::XOR, DL, VT, N1, N2);
7296 break;
7297 case ISD::MUL:
7298 assert(VT.isInteger() && "This operator does not apply to FP types!");
7299 assert(N1.getValueType() == N2.getValueType() &&
7300 N1.getValueType() == VT && "Binary operator types must match!");
7301 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
7302 return getNode(ISD::AND, DL, VT, N1, N2);
7303 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7304 const APInt &MulImm = N1->getConstantOperandAPInt(0);
7305 const APInt &N2CImm = N2C->getAPIntValue();
7306 return getVScale(DL, VT, MulImm * N2CImm);
7307 }
7308 break;
7309 case ISD::UDIV:
7310 case ISD::UREM:
7311 case ISD::MULHU:
7312 case ISD::MULHS:
7313 case ISD::SDIV:
7314 case ISD::SREM:
7315 case ISD::SADDSAT:
7316 case ISD::SSUBSAT:
7317 case ISD::UADDSAT:
7318 case ISD::USUBSAT:
7319 assert(VT.isInteger() && "This operator does not apply to FP types!");
7320 assert(N1.getValueType() == N2.getValueType() &&
7321 N1.getValueType() == VT && "Binary operator types must match!");
7322 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
7323 // fold (add_sat x, y) -> (or x, y) for bool types.
7324 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
7325 return getNode(ISD::OR, DL, VT, N1, N2);
7326 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
7327 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
7328 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
7329 }
7330 break;
7331 case ISD::SCMP:
7332 case ISD::UCMP:
7333 assert(N1.getValueType() == N2.getValueType() &&
7334 "Types of operands of UCMP/SCMP must match");
7335 assert(N1.getValueType().isVector() == VT.isVector() &&
7336 "Operands and return type of must both be scalars or vectors");
7337 if (VT.isVector())
7340 "Result and operands must have the same number of elements");
7341 break;
7342 case ISD::AVGFLOORS:
7343 case ISD::AVGFLOORU:
7344 case ISD::AVGCEILS:
7345 case ISD::AVGCEILU:
7346 assert(VT.isInteger() && "This operator does not apply to FP types!");
7347 assert(N1.getValueType() == N2.getValueType() &&
7348 N1.getValueType() == VT && "Binary operator types must match!");
7349 break;
7350 case ISD::ABDS:
7351 case ISD::ABDU:
7352 assert(VT.isInteger() && "This operator does not apply to FP types!");
7353 assert(N1.getValueType() == N2.getValueType() &&
7354 N1.getValueType() == VT && "Binary operator types must match!");
7355 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
7356 return getNode(ISD::XOR, DL, VT, N1, N2);
7357 break;
7358 case ISD::SMIN:
7359 case ISD::UMAX:
7360 assert(VT.isInteger() && "This operator does not apply to FP types!");
7361 assert(N1.getValueType() == N2.getValueType() &&
7362 N1.getValueType() == VT && "Binary operator types must match!");
7363 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
7364 return getNode(ISD::OR, DL, VT, N1, N2);
7365 break;
7366 case ISD::SMAX:
7367 case ISD::UMIN:
7368 assert(VT.isInteger() && "This operator does not apply to FP types!");
7369 assert(N1.getValueType() == N2.getValueType() &&
7370 N1.getValueType() == VT && "Binary operator types must match!");
7371 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
7372 return getNode(ISD::AND, DL, VT, N1, N2);
7373 break;
7374 case ISD::FADD:
7375 case ISD::FSUB:
7376 case ISD::FMUL:
7377 case ISD::FDIV:
7378 case ISD::FREM:
7379 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7380 assert(N1.getValueType() == N2.getValueType() &&
7381 N1.getValueType() == VT && "Binary operator types must match!");
7382 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
7383 return V;
7384 break;
7385 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
7386 assert(N1.getValueType() == VT &&
7389 "Invalid FCOPYSIGN!");
7390 break;
7391 case ISD::SHL:
7392 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7393 const APInt &MulImm = N1->getConstantOperandAPInt(0);
7394 const APInt &ShiftImm = N2C->getAPIntValue();
7395 return getVScale(DL, VT, MulImm << ShiftImm);
7396 }
7397 [[fallthrough]];
7398 case ISD::SRA:
7399 case ISD::SRL:
7400 if (SDValue V = simplifyShift(N1, N2))
7401 return V;
7402 [[fallthrough]];
7403 case ISD::ROTL:
7404 case ISD::ROTR:
7405 assert(VT == N1.getValueType() &&
7406 "Shift operators return type must be the same as their first arg");
7407 assert(VT.isInteger() && N2.getValueType().isInteger() &&
7408 "Shifts only work on integers");
7409 assert((!VT.isVector() || VT == N2.getValueType()) &&
7410 "Vector shift amounts must be in the same as their first arg");
7411 // Verify that the shift amount VT is big enough to hold valid shift
7412 // amounts. This catches things like trying to shift an i1024 value by an
7413 // i8, which is easy to fall into in generic code that uses
7414 // TLI.getShiftAmount().
7417 "Invalid use of small shift amount with oversized value!");
7418
7419 // Always fold shifts of i1 values so the code generator doesn't need to
7420 // handle them. Since we know the size of the shift has to be less than the
7421 // size of the value, the shift/rotate count is guaranteed to be zero.
7422 if (VT == MVT::i1)
7423 return N1;
7424 if (N2CV && N2CV->isZero())
7425 return N1;
7426 break;
7427 case ISD::FP_ROUND:
7429 VT.bitsLE(N1.getValueType()) && N2C &&
7430 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
7431 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
7432 if (N1.getValueType() == VT) return N1; // noop conversion.
7433 break;
7434 case ISD::AssertSext:
7435 case ISD::AssertZext: {
7436 EVT EVT = cast<VTSDNode>(N2)->getVT();
7437 assert(VT == N1.getValueType() && "Not an inreg extend!");
7438 assert(VT.isInteger() && EVT.isInteger() &&
7439 "Cannot *_EXTEND_INREG FP types");
7440 assert(!EVT.isVector() &&
7441 "AssertSExt/AssertZExt type should be the vector element type "
7442 "rather than the vector type!");
7443 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
7444 if (VT.getScalarType() == EVT) return N1; // noop assertion.
7445 break;
7446 }
7448 EVT EVT = cast<VTSDNode>(N2)->getVT();
7449 assert(VT == N1.getValueType() && "Not an inreg extend!");
7450 assert(VT.isInteger() && EVT.isInteger() &&
7451 "Cannot *_EXTEND_INREG FP types");
7452 assert(EVT.isVector() == VT.isVector() &&
7453 "SIGN_EXTEND_INREG type should be vector iff the operand "
7454 "type is vector!");
7455 assert((!EVT.isVector() ||
7457 "Vector element counts must match in SIGN_EXTEND_INREG");
7458 assert(EVT.bitsLE(VT) && "Not extending!");
7459 if (EVT == VT) return N1; // Not actually extending
7460 break;
7461 }
7463 case ISD::FP_TO_UINT_SAT: {
7464 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
7465 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
7466 assert(N1.getValueType().isVector() == VT.isVector() &&
7467 "FP_TO_*INT_SAT type should be vector iff the operand type is "
7468 "vector!");
7469 assert((!VT.isVector() || VT.getVectorElementCount() ==
7471 "Vector element counts must match in FP_TO_*INT_SAT");
7472 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
7473 "Type to saturate to must be a scalar.");
7474 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
7475 "Not extending!");
7476 break;
7477 }
7480 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
7481 element type of the vector.");
7482
7483 // Extract from an undefined value or using an undefined index is undefined.
7484 if (N1.isUndef() || N2.isUndef())
7485 return getUNDEF(VT);
7486
7487 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
7488 // vectors. For scalable vectors we will provide appropriate support for
7489 // dealing with arbitrary indices.
7490 if (N2C && N1.getValueType().isFixedLengthVector() &&
7491 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
7492 return getUNDEF(VT);
7493
7494 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
7495 // expanding copies of large vectors from registers. This only works for
7496 // fixed length vectors, since we need to know the exact number of
7497 // elements.
7498 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
7500 unsigned Factor =
7503 N1.getOperand(N2C->getZExtValue() / Factor),
7504 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
7505 }
7506
7507 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
7508 // lowering is expanding large vector constants.
7509 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
7510 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
7513 "BUILD_VECTOR used for scalable vectors");
7514 unsigned Index =
7515 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
7516 SDValue Elt = N1.getOperand(Index);
7517
7518 if (VT != Elt.getValueType())
7519 // If the vector element type is not legal, the BUILD_VECTOR operands
7520 // are promoted and implicitly truncated, and the result implicitly
7521 // extended. Make that explicit here.
7522 Elt = getAnyExtOrTrunc(Elt, DL, VT);
7523
7524 return Elt;
7525 }
7526
7527 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
7528 // operations are lowered to scalars.
7529 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
7530 // If the indices are the same, return the inserted element else
7531 // if the indices are known different, extract the element from
7532 // the original vector.
7533 SDValue N1Op2 = N1.getOperand(2);
7534 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
7535
7536 if (N1Op2C && N2C) {
7537 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
7538 if (VT == N1.getOperand(1).getValueType())
7539 return N1.getOperand(1);
7540 if (VT.isFloatingPoint()) {
7542 return getFPExtendOrRound(N1.getOperand(1), DL, VT);
7543 }
7544 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
7545 }
7546 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
7547 }
7548 }
7549
7550 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
7551 // when vector types are scalarized and v1iX is legal.
7552 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
7553 // Here we are completely ignoring the extract element index (N2),
7554 // which is fine for fixed width vectors, since any index other than 0
7555 // is undefined anyway. However, this cannot be ignored for scalable
7556 // vectors - in theory we could support this, but we don't want to do this
7557 // without a profitability check.
7558 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7560 N1.getValueType().getVectorNumElements() == 1) {
7561 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
7562 N1.getOperand(1));
7563 }
7564 break;
7566 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
7567 assert(!N1.getValueType().isVector() && !VT.isVector() &&
7568 (N1.getValueType().isInteger() == VT.isInteger()) &&
7569 N1.getValueType() != VT &&
7570 "Wrong types for EXTRACT_ELEMENT!");
7571
7572 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
7573 // 64-bit integers into 32-bit parts. Instead of building the extract of
7574 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
7575 if (N1.getOpcode() == ISD::BUILD_PAIR)
7576 return N1.getOperand(N2C->getZExtValue());
7577
7578 // EXTRACT_ELEMENT of a constant int is also very common.
7579 if (N1C) {
7580 unsigned ElementSize = VT.getSizeInBits();
7581 unsigned Shift = ElementSize * N2C->getZExtValue();
7582 const APInt &Val = N1C->getAPIntValue();
7583 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
7584 }
7585 break;
7587 EVT N1VT = N1.getValueType();
7588 assert(VT.isVector() && N1VT.isVector() &&
7589 "Extract subvector VTs must be vectors!");
7591 "Extract subvector VTs must have the same element type!");
7592 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
7593 "Cannot extract a scalable vector from a fixed length vector!");
7594 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7596 "Extract subvector must be from larger vector to smaller vector!");
7597 assert(N2C && "Extract subvector index must be a constant");
7598 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7599 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
7600 N1VT.getVectorMinNumElements()) &&
7601 "Extract subvector overflow!");
7602 assert(N2C->getAPIntValue().getBitWidth() ==
7603 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7604 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
7605
7606 // Trivial extraction.
7607 if (VT == N1VT)
7608 return N1;
7609
7610 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
7611 if (N1.isUndef())
7612 return getUNDEF(VT);
7613
7614 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
7615 // the concat have the same type as the extract.
7616 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
7617 VT == N1.getOperand(0).getValueType()) {
7618 unsigned Factor = VT.getVectorMinNumElements();
7619 return N1.getOperand(N2C->getZExtValue() / Factor);
7620 }
7621
7622 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
7623 // during shuffle legalization.
7624 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
7625 VT == N1.getOperand(1).getValueType())
7626 return N1.getOperand(1);
7627 break;
7628 }
7629 }
7630
7631 // Perform trivial constant folding.
7632 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}, Flags))
7633 return SV;
7634
7635 // Canonicalize an UNDEF to the RHS, even over a constant.
7636 if (N1.isUndef()) {
7637 if (TLI->isCommutativeBinOp(Opcode)) {
7638 std::swap(N1, N2);
7639 } else {
7640 switch (Opcode) {
7641 case ISD::SUB:
7642 return getUNDEF(VT); // fold op(undef, arg2) -> undef
7644 case ISD::UDIV:
7645 case ISD::SDIV:
7646 case ISD::UREM:
7647 case ISD::SREM:
7648 case ISD::SSUBSAT:
7649 case ISD::USUBSAT:
7650 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
7651 }
7652 }
7653 }
7654
7655 // Fold a bunch of operators when the RHS is undef.
7656 if (N2.isUndef()) {
7657 switch (Opcode) {
7658 case ISD::XOR:
7659 if (N1.isUndef())
7660 // Handle undef ^ undef -> 0 special case. This is a common
7661 // idiom (misuse).
7662 return getConstant(0, DL, VT);
7663 [[fallthrough]];
7664 case ISD::ADD:
7665 case ISD::SUB:
7666 case ISD::UDIV:
7667 case ISD::SDIV:
7668 case ISD::UREM:
7669 case ISD::SREM:
7670 return getUNDEF(VT); // fold op(arg1, undef) -> undef
7671 case ISD::MUL:
7672 case ISD::AND:
7673 case ISD::SSUBSAT:
7674 case ISD::USUBSAT:
7675 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
7676 case ISD::OR:
7677 case ISD::SADDSAT:
7678 case ISD::UADDSAT:
7679 return getAllOnesConstant(DL, VT);
7680 }
7681 }
7682
7683 // Memoize this node if possible.
7684 SDNode *N;
7685 SDVTList VTs = getVTList(VT);
7686 SDValue Ops[] = {N1, N2};
7687 if (VT != MVT::Glue) {
7689 AddNodeIDNode(ID, Opcode, VTs, Ops);
7690 void *IP = nullptr;
7691 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7692 E->intersectFlagsWith(Flags);
7693 return SDValue(E, 0);
7694 }
7695
7696 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7697 N->setFlags(Flags);
7698 createOperands(N, Ops);
7699 CSEMap.InsertNode(N, IP);
7700 } else {
7701 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7702 createOperands(N, Ops);
7703 }
7704
7705 InsertNode(N);
7706 SDValue V = SDValue(N, 0);
7707 NewSDValueDbgMsg(V, "Creating new node: ", this);
7708 return V;
7709}
7710
7711SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7712 SDValue N1, SDValue N2, SDValue N3) {
7713 SDNodeFlags Flags;
7714 if (Inserter)
7715 Flags = Inserter->getFlags();
7716 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7717}
7718
7719SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7720 SDValue N1, SDValue N2, SDValue N3,
7721 const SDNodeFlags Flags) {
7723 N2.getOpcode() != ISD::DELETED_NODE &&
7724 N3.getOpcode() != ISD::DELETED_NODE &&
7725 "Operand is DELETED_NODE!");
7726 // Perform various simplifications.
7727 switch (Opcode) {
7728 case ISD::FMA:
7729 case ISD::FMAD: {
7730 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7731 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7732 N3.getValueType() == VT && "FMA types must match!");
7733 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
7734 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
7735 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
7736 if (N1CFP && N2CFP && N3CFP) {
7737 APFloat V1 = N1CFP->getValueAPF();
7738 const APFloat &V2 = N2CFP->getValueAPF();
7739 const APFloat &V3 = N3CFP->getValueAPF();
7740 if (Opcode == ISD::FMAD) {
7743 } else
7745 return getConstantFP(V1, DL, VT);
7746 }
7747 break;
7748 }
7749 case ISD::BUILD_VECTOR: {
7750 // Attempt to simplify BUILD_VECTOR.
7751 SDValue Ops[] = {N1, N2, N3};
7752 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7753 return V;
7754 break;
7755 }
7756 case ISD::CONCAT_VECTORS: {
7757 SDValue Ops[] = {N1, N2, N3};
7758 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7759 return V;
7760 break;
7761 }
7762 case ISD::SETCC: {
7763 assert(VT.isInteger() && "SETCC result type must be an integer!");
7764 assert(N1.getValueType() == N2.getValueType() &&
7765 "SETCC operands must have the same type!");
7766 assert(VT.isVector() == N1.getValueType().isVector() &&
7767 "SETCC type should be vector iff the operand type is vector!");
7768 assert((!VT.isVector() || VT.getVectorElementCount() ==
7770 "SETCC vector element counts must match!");
7771 // Use FoldSetCC to simplify SETCC's.
7772 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
7773 return V;
7774 // Vector constant folding.
7775 SDValue Ops[] = {N1, N2, N3};
7776 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7777 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
7778 return V;
7779 }
7780 break;
7781 }
7782 case ISD::SELECT:
7783 case ISD::VSELECT:
7784 if (SDValue V = simplifySelect(N1, N2, N3))
7785 return V;
7786 break;
7788 llvm_unreachable("should use getVectorShuffle constructor!");
7789 case ISD::VECTOR_SPLICE: {
7790 if (cast<ConstantSDNode>(N3)->isZero())
7791 return N1;
7792 break;
7793 }
7795 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
7796 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7797 // for scalable vectors where we will generate appropriate code to
7798 // deal with out-of-bounds cases correctly.
7799 if (N3C && N1.getValueType().isFixedLengthVector() &&
7801 return getUNDEF(VT);
7802
7803 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7804 if (N3.isUndef())
7805 return getUNDEF(VT);
7806
7807 // If the inserted element is an UNDEF, just use the input vector.
7808 if (N2.isUndef())
7809 return N1;
7810
7811 break;
7812 }
7813 case ISD::INSERT_SUBVECTOR: {
7814 // Inserting undef into undef is still undef.
7815 if (N1.isUndef() && N2.isUndef())
7816 return getUNDEF(VT);
7817
7818 EVT N2VT = N2.getValueType();
7819 assert(VT == N1.getValueType() &&
7820 "Dest and insert subvector source types must match!");
7821 assert(VT.isVector() && N2VT.isVector() &&
7822 "Insert subvector VTs must be vectors!");
7824 "Insert subvector VTs must have the same element type!");
7825 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7826 "Cannot insert a scalable vector into a fixed length vector!");
7827 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7829 "Insert subvector must be from smaller vector to larger vector!");
7830 assert(isa<ConstantSDNode>(N3) &&
7831 "Insert subvector index must be constant");
7832 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7833 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7835 "Insert subvector overflow!");
7837 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7838 "Constant index for INSERT_SUBVECTOR has an invalid size");
7839
7840 // Trivial insertion.
7841 if (VT == N2VT)
7842 return N2;
7843
7844 // If this is an insert of an extracted vector into an undef vector, we
7845 // can just use the input to the extract.
7846 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7847 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
7848 return N2.getOperand(0);
7849 break;
7850 }
7851 case ISD::BITCAST:
7852 // Fold bit_convert nodes from a type to themselves.
7853 if (N1.getValueType() == VT)
7854 return N1;
7855 break;
7856 case ISD::VP_TRUNCATE:
7857 case ISD::VP_SIGN_EXTEND:
7858 case ISD::VP_ZERO_EXTEND:
7859 // Don't create noop casts.
7860 if (N1.getValueType() == VT)
7861 return N1;
7862 break;
7863 case ISD::VECTOR_COMPRESS: {
7864 [[maybe_unused]] EVT VecVT = N1.getValueType();
7865 [[maybe_unused]] EVT MaskVT = N2.getValueType();
7866 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
7867 assert(VT == VecVT && "Vector and result type don't match.");
7868 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
7869 "All inputs must be vectors.");
7870 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
7872 "Vector and mask must have same number of elements.");
7873
7874 if (N1.isUndef() || N2.isUndef())
7875 return N3;
7876
7877 break;
7878 }
7879 }
7880
7881 // Memoize node if it doesn't produce a glue result.
7882 SDNode *N;
7883 SDVTList VTs = getVTList(VT);
7884 SDValue Ops[] = {N1, N2, N3};
7885 if (VT != MVT::Glue) {
7887 AddNodeIDNode(ID, Opcode, VTs, Ops);
7888 void *IP = nullptr;
7889 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7890 E->intersectFlagsWith(Flags);
7891 return SDValue(E, 0);
7892 }
7893
7894 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7895 N->setFlags(Flags);
7896 createOperands(N, Ops);
7897 CSEMap.InsertNode(N, IP);
7898 } else {
7899 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7900 createOperands(N, Ops);
7901 }
7902
7903 InsertNode(N);
7904 SDValue V = SDValue(N, 0);
7905 NewSDValueDbgMsg(V, "Creating new node: ", this);
7906 return V;
7907}
7908
7909SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7910 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7911 const SDNodeFlags Flags) {
7912 SDValue Ops[] = { N1, N2, N3, N4 };
7913 return getNode(Opcode, DL, VT, Ops, Flags);
7914}
7915
7916SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7917 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7918 SDNodeFlags Flags;
7919 if (Inserter)
7920 Flags = Inserter->getFlags();
7921 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
7922}
7923
7924SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7925 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7926 SDValue N5, const SDNodeFlags Flags) {
7927 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7928 return getNode(Opcode, DL, VT, Ops, Flags);
7929}
7930
7931SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7932 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7933 SDValue N5) {
7934 SDNodeFlags Flags;
7935 if (Inserter)
7936 Flags = Inserter->getFlags();
7937 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
7938}
7939
7940/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7941/// the incoming stack arguments to be loaded from the stack.
7943 SmallVector<SDValue, 8> ArgChains;
7944
7945 // Include the original chain at the beginning of the list. When this is
7946 // used by target LowerCall hooks, this helps legalize find the
7947 // CALLSEQ_BEGIN node.
7948 ArgChains.push_back(Chain);
7949
7950 // Add a chain value for each stack argument.
7951 for (SDNode *U : getEntryNode().getNode()->users())
7952 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))
7953 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
7954 if (FI->getIndex() < 0)
7955 ArgChains.push_back(SDValue(L, 1));
7956
7957 // Build a tokenfactor for all the chains.
7958 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7959}
7960
7961/// getMemsetValue - Vectorized representation of the memset value
7962/// operand.
7964 const SDLoc &dl) {
7965 assert(!Value.isUndef());
7966
7967 unsigned NumBits = VT.getScalarSizeInBits();
7968 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
7969 assert(C->getAPIntValue().getBitWidth() == 8);
7970 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
7971 if (VT.isInteger()) {
7972 bool IsOpaque = VT.getSizeInBits() > 64 ||
7973 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
7974 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
7975 }
7976 return DAG.getConstantFP(APFloat(VT.getFltSemantics(), Val), dl, VT);
7977 }
7978
7979 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7980 EVT IntVT = VT.getScalarType();
7981 if (!IntVT.isInteger())
7982 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
7983
7984 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
7985 if (NumBits > 8) {
7986 // Use a multiplication with 0x010101... to extend the input to the
7987 // required length.
7988 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
7989 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
7990 DAG.getConstant(Magic, dl, IntVT));
7991 }
7992
7993 if (VT != Value.getValueType() && !VT.isInteger())
7994 Value = DAG.getBitcast(VT.getScalarType(), Value);
7995 if (VT != Value.getValueType())
7996 Value = DAG.getSplatBuildVector(VT, dl, Value);
7997
7998 return Value;
7999}
8000
8001/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
8002/// used when a memcpy is turned into a memset when the source is a constant
8003/// string ptr.
8005 const TargetLowering &TLI,
8006 const ConstantDataArraySlice &Slice) {
8007 // Handle vector with all elements zero.
8008 if (Slice.Array == nullptr) {
8009 if (VT.isInteger())
8010 return DAG.getConstant(0, dl, VT);
8011 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
8012 return DAG.getConstantFP(0.0, dl, VT);
8013 if (VT.isVector()) {
8014 unsigned NumElts = VT.getVectorNumElements();
8015 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
8016 return DAG.getNode(ISD::BITCAST, dl, VT,
8017 DAG.getConstant(0, dl,
8019 EltVT, NumElts)));
8020 }
8021 llvm_unreachable("Expected type!");
8022 }
8023
8024 assert(!VT.isVector() && "Can't handle vector type here!");
8025 unsigned NumVTBits = VT.getSizeInBits();
8026 unsigned NumVTBytes = NumVTBits / 8;
8027 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
8028
8029 APInt Val(NumVTBits, 0);
8030 if (DAG.getDataLayout().isLittleEndian()) {
8031 for (unsigned i = 0; i != NumBytes; ++i)
8032 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
8033 } else {
8034 for (unsigned i = 0; i != NumBytes; ++i)
8035 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
8036 }
8037
8038 // If the "cost" of materializing the integer immediate is less than the cost
8039 // of a load, then it is cost effective to turn the load into the immediate.
8040 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
8041 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
8042 return DAG.getConstant(Val, dl, VT);
8043 return SDValue();
8044}
8045
8047 const SDLoc &DL,
8048 const SDNodeFlags Flags) {
8049 EVT VT = Base.getValueType();
8050 SDValue Index;
8051
8052 if (Offset.isScalable())
8053 Index = getVScale(DL, Base.getValueType(),
8054 APInt(Base.getValueSizeInBits().getFixedValue(),
8055 Offset.getKnownMinValue()));
8056 else
8057 Index = getConstant(Offset.getFixedValue(), DL, VT);
8058
8059 return getMemBasePlusOffset(Base, Index, DL, Flags);
8060}
8061
8063 const SDLoc &DL,
8064 const SDNodeFlags Flags) {
8065 assert(Offset.getValueType().isInteger());
8066 EVT BasePtrVT = Ptr.getValueType();
8067 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
8068}
8069
8070/// Returns true if memcpy source is constant data.
8072 uint64_t SrcDelta = 0;
8073 GlobalAddressSDNode *G = nullptr;
8074 if (Src.getOpcode() == ISD::GlobalAddress)
8075 G = cast<GlobalAddressSDNode>(Src);
8076 else if (Src.getOpcode() == ISD::ADD &&
8077 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
8078 Src.getOperand(1).getOpcode() == ISD::Constant) {
8079 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
8080 SrcDelta = Src.getConstantOperandVal(1);
8081 }
8082 if (!G)
8083 return false;
8084
8085 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
8086 SrcDelta + G->getOffset());
8087}
8088
8090 SelectionDAG &DAG) {
8091 // On Darwin, -Os means optimize for size without hurting performance, so
8092 // only really optimize for size when -Oz (MinSize) is used.
8094 return MF.getFunction().hasMinSize();
8095 return DAG.shouldOptForSize();
8096}
8097
8099 SmallVector<SDValue, 32> &OutChains, unsigned From,
8100 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
8101 SmallVector<SDValue, 16> &OutStoreChains) {
8102 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
8103 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
8104 SmallVector<SDValue, 16> GluedLoadChains;
8105 for (unsigned i = From; i < To; ++i) {
8106 OutChains.push_back(OutLoadChains[i]);
8107 GluedLoadChains.push_back(OutLoadChains[i]);
8108 }
8109
8110 // Chain for all loads.
8111 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
8112 GluedLoadChains);
8113
8114 for (unsigned i = From; i < To; ++i) {
8115 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
8116 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
8117 ST->getBasePtr(), ST->getMemoryVT(),
8118 ST->getMemOperand());
8119 OutChains.push_back(NewStore);
8120 }
8121}
8122
8124 SDValue Chain, SDValue Dst, SDValue Src,
8125 uint64_t Size, Align Alignment,
8126 bool isVol, bool AlwaysInline,
8127 MachinePointerInfo DstPtrInfo,
8128 MachinePointerInfo SrcPtrInfo,
8129 const AAMDNodes &AAInfo, AAResults *AA) {
8130 // Turn a memcpy of undef to nop.
8131 // FIXME: We need to honor volatile even is Src is undef.
8132 if (Src.isUndef())
8133 return Chain;
8134
8135 // Expand memcpy to a series of load and store ops if the size operand falls
8136 // below a certain threshold.
8137 // TODO: In the AlwaysInline case, if the size is big then generate a loop
8138 // rather than maybe a humongous number of loads and stores.
8139 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8140 const DataLayout &DL = DAG.getDataLayout();
8141 LLVMContext &C = *DAG.getContext();
8142 std::vector<EVT> MemOps;
8143 bool DstAlignCanChange = false;
8145 MachineFrameInfo &MFI = MF.getFrameInfo();
8146 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8147 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
8148 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
8149 DstAlignCanChange = true;
8150 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
8151 if (!SrcAlign || Alignment > *SrcAlign)
8152 SrcAlign = Alignment;
8153 assert(SrcAlign && "SrcAlign must be set");
8155 // If marked as volatile, perform a copy even when marked as constant.
8156 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
8157 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
8158 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
8159 const MemOp Op = isZeroConstant
8160 ? MemOp::Set(Size, DstAlignCanChange, Alignment,
8161 /*IsZeroMemset*/ true, isVol)
8162 : MemOp::Copy(Size, DstAlignCanChange, Alignment,
8163 *SrcAlign, isVol, CopyFromConstant);
8164 if (!TLI.findOptimalMemOpLowering(
8165 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
8166 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
8167 return SDValue();
8168
8169 if (DstAlignCanChange) {
8170 Type *Ty = MemOps[0].getTypeForEVT(C);
8171 Align NewAlign = DL.getABITypeAlign(Ty);
8172
8173 // Don't promote to an alignment that would require dynamic stack
8174 // realignment which may conflict with optimizations such as tail call
8175 // optimization.
8177 if (!TRI->hasStackRealignment(MF))
8178 if (MaybeAlign StackAlign = DL.getStackAlignment())
8179 NewAlign = std::min(NewAlign, *StackAlign);
8180
8181 if (NewAlign > Alignment) {
8182 // Give the stack frame object a larger alignment if needed.
8183 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
8184 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
8185 Alignment = NewAlign;
8186 }
8187 }
8188
8189 // Prepare AAInfo for loads/stores after lowering this memcpy.
8190 AAMDNodes NewAAInfo = AAInfo;
8191 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8192
8193 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
8194 bool isConstant =
8195 AA && SrcVal &&
8196 AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
8197
8198 MachineMemOperand::Flags MMOFlags =
8200 SmallVector<SDValue, 16> OutLoadChains;
8201 SmallVector<SDValue, 16> OutStoreChains;
8202 SmallVector<SDValue, 32> OutChains;
8203 unsigned NumMemOps = MemOps.size();
8204 uint64_t SrcOff = 0, DstOff = 0;
8205 for (unsigned i = 0; i != NumMemOps; ++i) {
8206 EVT VT = MemOps[i];
8207 unsigned VTSize = VT.getSizeInBits() / 8;
8208 SDValue Value, Store;
8209
8210 if (VTSize > Size) {
8211 // Issuing an unaligned load / store pair that overlaps with the previous
8212 // pair. Adjust the offset accordingly.
8213 assert(i == NumMemOps-1 && i != 0);
8214 SrcOff -= VTSize - Size;
8215 DstOff -= VTSize - Size;
8216 }
8217
8218 if (CopyFromConstant &&
8219 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
8220 // It's unlikely a store of a vector immediate can be done in a single
8221 // instruction. It would require a load from a constantpool first.
8222 // We only handle zero vectors here.
8223 // FIXME: Handle other cases where store of vector immediate is done in
8224 // a single instruction.
8225 ConstantDataArraySlice SubSlice;
8226 if (SrcOff < Slice.Length) {
8227 SubSlice = Slice;
8228 SubSlice.move(SrcOff);
8229 } else {
8230 // This is an out-of-bounds access and hence UB. Pretend we read zero.
8231 SubSlice.Array = nullptr;
8232 SubSlice.Offset = 0;
8233 SubSlice.Length = VTSize;
8234 }
8235 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
8236 if (Value.getNode()) {
8237 Store = DAG.getStore(
8238 Chain, dl, Value,
8239 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
8240 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
8241 OutChains.push_back(Store);
8242 }
8243 }
8244
8245 if (!Store.getNode()) {
8246 // The type might not be legal for the target. This should only happen
8247 // if the type is smaller than a legal type, as on PPC, so the right
8248 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
8249 // to Load/Store if NVT==VT.
8250 // FIXME does the case above also need this?
8251 EVT NVT = TLI.getTypeToTransformTo(C, VT);
8252 assert(NVT.bitsGE(VT));
8253
8254 bool isDereferenceable =
8255 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
8256 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
8257 if (isDereferenceable)
8259 if (isConstant)
8260 SrcMMOFlags |= MachineMemOperand::MOInvariant;
8261
8262 Value = DAG.getExtLoad(
8263 ISD::EXTLOAD, dl, NVT, Chain,
8264 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
8265 SrcPtrInfo.getWithOffset(SrcOff), VT,
8266 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
8267 OutLoadChains.push_back(Value.getValue(1));
8268
8269 Store = DAG.getTruncStore(
8270 Chain, dl, Value,
8271 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
8272 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
8273 OutStoreChains.push_back(Store);
8274 }
8275 SrcOff += VTSize;
8276 DstOff += VTSize;
8277 Size -= VTSize;
8278 }
8279
8280 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
8282 unsigned NumLdStInMemcpy = OutStoreChains.size();
8283
8284 if (NumLdStInMemcpy) {
8285 // It may be that memcpy might be converted to memset if it's memcpy
8286 // of constants. In such a case, we won't have loads and stores, but
8287 // just stores. In the absence of loads, there is nothing to gang up.
8288 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
8289 // If target does not care, just leave as it.
8290 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
8291 OutChains.push_back(OutLoadChains[i]);
8292 OutChains.push_back(OutStoreChains[i]);
8293 }
8294 } else {
8295 // Ld/St less than/equal limit set by target.
8296 if (NumLdStInMemcpy <= GluedLdStLimit) {
8297 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
8298 NumLdStInMemcpy, OutLoadChains,
8299 OutStoreChains);
8300 } else {
8301 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
8302 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
8303 unsigned GlueIter = 0;
8304
8305 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
8306 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
8307 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
8308
8309 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
8310 OutLoadChains, OutStoreChains);
8311 GlueIter += GluedLdStLimit;
8312 }
8313
8314 // Residual ld/st.
8315 if (RemainingLdStInMemcpy) {
8316 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
8317 RemainingLdStInMemcpy, OutLoadChains,
8318 OutStoreChains);
8319 }
8320 }
8321 }
8322 }
8323 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
8324}
8325
8327 SDValue Chain, SDValue Dst, SDValue Src,
8328 uint64_t Size, Align Alignment,
8329 bool isVol, bool AlwaysInline,
8330 MachinePointerInfo DstPtrInfo,
8331 MachinePointerInfo SrcPtrInfo,
8332 const AAMDNodes &AAInfo) {
8333 // Turn a memmove of undef to nop.
8334 // FIXME: We need to honor volatile even is Src is undef.
8335 if (Src.isUndef())
8336 return Chain;
8337
8338 // Expand memmove to a series of load and store ops if the size operand falls
8339 // below a certain threshold.
8340 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8341 const DataLayout &DL = DAG.getDataLayout();
8342 LLVMContext &C = *DAG.getContext();
8343 std::vector<EVT> MemOps;
8344 bool DstAlignCanChange = false;
8346 MachineFrameInfo &MFI = MF.getFrameInfo();
8347 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8348 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
8349 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
8350 DstAlignCanChange = true;
8351 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
8352 if (!SrcAlign || Alignment > *SrcAlign)
8353 SrcAlign = Alignment;
8354 assert(SrcAlign && "SrcAlign must be set");
8355 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
8356 if (!TLI.findOptimalMemOpLowering(
8357 MemOps, Limit,
8358 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
8359 /*IsVolatile*/ true),
8360 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
8361 MF.getFunction().getAttributes()))
8362 return SDValue();
8363
8364 if (DstAlignCanChange) {
8365 Type *Ty = MemOps[0].getTypeForEVT(C);
8366 Align NewAlign = DL.getABITypeAlign(Ty);
8367
8368 // Don't promote to an alignment that would require dynamic stack
8369 // realignment which may conflict with optimizations such as tail call
8370 // optimization.
8372 if (!TRI->hasStackRealignment(MF))
8373 if (MaybeAlign StackAlign = DL.getStackAlignment())
8374 NewAlign = std::min(NewAlign, *StackAlign);
8375
8376 if (NewAlign > Alignment) {
8377 // Give the stack frame object a larger alignment if needed.
8378 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
8379 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
8380 Alignment = NewAlign;
8381 }
8382 }
8383
8384 // Prepare AAInfo for loads/stores after lowering this memmove.
8385 AAMDNodes NewAAInfo = AAInfo;
8386 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8387
8388 MachineMemOperand::Flags MMOFlags =
8390 uint64_t SrcOff = 0, DstOff = 0;
8391 SmallVector<SDValue, 8> LoadValues;
8392 SmallVector<SDValue, 8> LoadChains;
8393 SmallVector<SDValue, 8> OutChains;
8394 unsigned NumMemOps = MemOps.size();
8395 for (unsigned i = 0; i < NumMemOps; i++) {
8396 EVT VT = MemOps[i];
8397 unsigned VTSize = VT.getSizeInBits() / 8;
8398 SDValue Value;
8399
8400 bool isDereferenceable =
8401 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
8402 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
8403 if (isDereferenceable)
8405
8406 Value = DAG.getLoad(
8407 VT, dl, Chain,
8408 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
8409 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
8410 LoadValues.push_back(Value);
8411 LoadChains.push_back(Value.getValue(1));
8412 SrcOff += VTSize;
8413 }
8414 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
8415 OutChains.clear();
8416 for (unsigned i = 0; i < NumMemOps; i++) {
8417 EVT VT = MemOps[i];
8418 unsigned VTSize = VT.getSizeInBits() / 8;
8419 SDValue Store;
8420
8421 Store = DAG.getStore(
8422 Chain, dl, LoadValues[i],
8423 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
8424 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
8425 OutChains.push_back(Store);
8426 DstOff += VTSize;
8427 }
8428
8429 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
8430}
8431
8432/// Lower the call to 'memset' intrinsic function into a series of store
8433/// operations.
8434///
8435/// \param DAG Selection DAG where lowered code is placed.
8436/// \param dl Link to corresponding IR location.
8437/// \param Chain Control flow dependency.
8438/// \param Dst Pointer to destination memory location.
8439/// \param Src Value of byte to write into the memory.
8440/// \param Size Number of bytes to write.
8441/// \param Alignment Alignment of the destination in bytes.
8442/// \param isVol True if destination is volatile.
8443/// \param AlwaysInline Makes sure no function call is generated.
8444/// \param DstPtrInfo IR information on the memory pointer.
8445/// \returns New head in the control flow, if lowering was successful, empty
8446/// SDValue otherwise.
8447///
8448/// The function tries to replace 'llvm.memset' intrinsic with several store
8449/// operations and value calculation code. This is usually profitable for small
8450/// memory size or when the semantic requires inlining.
8452 SDValue Chain, SDValue Dst, SDValue Src,
8453 uint64_t Size, Align Alignment, bool isVol,
8454 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
8455 const AAMDNodes &AAInfo) {
8456 // Turn a memset of undef to nop.
8457 // FIXME: We need to honor volatile even is Src is undef.
8458 if (Src.isUndef())
8459 return Chain;
8460
8461 // Expand memset to a series of load/store ops if the size operand
8462 // falls below a certain threshold.
8463 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8464 std::vector<EVT> MemOps;
8465 bool DstAlignCanChange = false;
8467 MachineFrameInfo &MFI = MF.getFrameInfo();
8468 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8469 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
8470 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
8471 DstAlignCanChange = true;
8472 bool IsZeroVal = isNullConstant(Src);
8473 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
8474
8475 if (!TLI.findOptimalMemOpLowering(
8476 MemOps, Limit,
8477 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
8478 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
8479 return SDValue();
8480
8481 if (DstAlignCanChange) {
8482 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
8483 const DataLayout &DL = DAG.getDataLayout();
8484 Align NewAlign = DL.getABITypeAlign(Ty);
8485
8486 // Don't promote to an alignment that would require dynamic stack
8487 // realignment which may conflict with optimizations such as tail call
8488 // optimization.
8490 if (!TRI->hasStackRealignment(MF))
8491 if (MaybeAlign StackAlign = DL.getStackAlignment())
8492 NewAlign = std::min(NewAlign, *StackAlign);
8493
8494 if (NewAlign > Alignment) {
8495 // Give the stack frame object a larger alignment if needed.
8496 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
8497 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
8498 Alignment = NewAlign;
8499 }
8500 }
8501
8502 SmallVector<SDValue, 8> OutChains;
8503 uint64_t DstOff = 0;
8504 unsigned NumMemOps = MemOps.size();
8505
8506 // Find the largest store and generate the bit pattern for it.
8507 EVT LargestVT = MemOps[0];
8508 for (unsigned i = 1; i < NumMemOps; i++)
8509 if (MemOps[i].bitsGT(LargestVT))
8510 LargestVT = MemOps[i];
8511 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
8512
8513 // Prepare AAInfo for loads/stores after lowering this memset.
8514 AAMDNodes NewAAInfo = AAInfo;
8515 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8516
8517 for (unsigned i = 0; i < NumMemOps; i++) {
8518 EVT VT = MemOps[i];
8519 unsigned VTSize = VT.getSizeInBits() / 8;
8520 if (VTSize > Size) {
8521 // Issuing an unaligned load / store pair that overlaps with the previous
8522 // pair. Adjust the offset accordingly.
8523 assert(i == NumMemOps-1 && i != 0);
8524 DstOff -= VTSize - Size;
8525 }
8526
8527 // If this store is smaller than the largest store see whether we can get
8528 // the smaller value for free with a truncate or extract vector element and
8529 // then store.
8530 SDValue Value = MemSetValue;
8531 if (VT.bitsLT(LargestVT)) {
8532 unsigned Index;
8533 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
8534 EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);
8535 if (!LargestVT.isVector() && !VT.isVector() &&
8536 TLI.isTruncateFree(LargestVT, VT))
8537 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
8538 else if (LargestVT.isVector() && !VT.isVector() &&
8540 LargestVT.getTypeForEVT(*DAG.getContext()),
8541 VT.getSizeInBits(), Index) &&
8542 TLI.isTypeLegal(SVT) &&
8543 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
8544 // Target which can combine store(extractelement VectorTy, Idx) can get
8545 // the smaller value for free.
8546 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);
8547 Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, TailValue,
8548 DAG.getVectorIdxConstant(Index, dl));
8549 } else
8550 Value = getMemsetValue(Src, VT, DAG, dl);
8551 }
8552 assert(Value.getValueType() == VT && "Value with wrong type.");
8553 SDValue Store = DAG.getStore(
8554 Chain, dl, Value,
8555 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
8556 DstPtrInfo.getWithOffset(DstOff), Alignment,
8558 NewAAInfo);
8559 OutChains.push_back(Store);
8560 DstOff += VT.getSizeInBits() / 8;
8561 Size -= VTSize;
8562 }
8563
8564 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
8565}
8566
8568 unsigned AS) {
8569 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
8570 // pointer operands can be losslessly bitcasted to pointers of address space 0
8571 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
8572 report_fatal_error("cannot lower memory intrinsic in address space " +
8573 Twine(AS));
8574 }
8575}
8576
8578 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
8579 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
8580 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
8581 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) {
8582 // Check to see if we should lower the memcpy to loads and stores first.
8583 // For cases within the target-specified limits, this is the best choice.
8584 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8585 if (ConstantSize) {
8586 // Memcpy with size zero? Just return the original chain.
8587 if (ConstantSize->isZero())
8588 return Chain;
8589
8591 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8592 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8593 if (Result.getNode())
8594 return Result;
8595 }
8596
8597 // Then check to see if we should lower the memcpy with target-specific
8598 // code. If the target chooses to do this, this is the next best.
8599 if (TSI) {
8600 SDValue Result = TSI->EmitTargetCodeForMemcpy(
8601 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
8602 DstPtrInfo, SrcPtrInfo);
8603 if (Result.getNode())
8604 return Result;
8605 }
8606
8607 // If we really need inline code and the target declined to provide it,
8608 // use a (potentially long) sequence of loads and stores.
8609 if (AlwaysInline) {
8610 assert(ConstantSize && "AlwaysInline requires a constant size!");
8612 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8613 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8614 }
8615
8618
8619 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
8620 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
8621 // respect volatile, so they may do things like read or write memory
8622 // beyond the given memory regions. But fixing this isn't easy, and most
8623 // people don't care.
8624
8625 // Emit a library call.
8628 Entry.Ty = PointerType::getUnqual(*getContext());
8629 Entry.Node = Dst; Args.push_back(Entry);
8630 Entry.Node = Src; Args.push_back(Entry);
8631
8632 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8633 Entry.Node = Size; Args.push_back(Entry);
8634 // FIXME: pass in SDLoc
8636 bool IsTailCall = false;
8637 if (OverrideTailCall.has_value()) {
8638 IsTailCall = *OverrideTailCall;
8639 } else {
8640 bool LowersToMemcpy =
8641 TLI->getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy");
8642 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
8643 IsTailCall = CI && CI->isTailCall() &&
8645 ReturnsFirstArg && LowersToMemcpy);
8646 }
8647
8648 CLI.setDebugLoc(dl)
8649 .setChain(Chain)
8650 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
8651 Dst.getValueType().getTypeForEVT(*getContext()),
8652 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
8653 TLI->getPointerTy(getDataLayout())),
8654 std::move(Args))
8656 .setTailCall(IsTailCall);
8657
8658 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8659 return CallResult.second;
8660}
8661
8663 SDValue Dst, SDValue Src, SDValue Size,
8664 Type *SizeTy, unsigned ElemSz,
8665 bool isTailCall,
8666 MachinePointerInfo DstPtrInfo,
8667 MachinePointerInfo SrcPtrInfo) {
8668 // Emit a library call.
8671 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8672 Entry.Node = Dst;
8673 Args.push_back(Entry);
8674
8675 Entry.Node = Src;
8676 Args.push_back(Entry);
8677
8678 Entry.Ty = SizeTy;
8679 Entry.Node = Size;
8680 Args.push_back(Entry);
8681
8682 RTLIB::Libcall LibraryCall =
8684 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8685 report_fatal_error("Unsupported element size");
8686
8688 CLI.setDebugLoc(dl)
8689 .setChain(Chain)
8690 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8692 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8693 TLI->getPointerTy(getDataLayout())),
8694 std::move(Args))
8696 .setTailCall(isTailCall);
8697
8698 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8699 return CallResult.second;
8700}
8701
8703 SDValue Src, SDValue Size, Align Alignment,
8704 bool isVol, const CallInst *CI,
8705 std::optional<bool> OverrideTailCall,
8706 MachinePointerInfo DstPtrInfo,
8707 MachinePointerInfo SrcPtrInfo,
8708 const AAMDNodes &AAInfo, AAResults *AA) {
8709 // Check to see if we should lower the memmove to loads and stores first.
8710 // For cases within the target-specified limits, this is the best choice.
8711 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8712 if (ConstantSize) {
8713 // Memmove with size zero? Just return the original chain.
8714 if (ConstantSize->isZero())
8715 return Chain;
8716
8718 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8719 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
8720 if (Result.getNode())
8721 return Result;
8722 }
8723
8724 // Then check to see if we should lower the memmove with target-specific
8725 // code. If the target chooses to do this, this is the next best.
8726 if (TSI) {
8727 SDValue Result =
8728 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
8729 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8730 if (Result.getNode())
8731 return Result;
8732 }
8733
8736
8737 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8738 // not be safe. See memcpy above for more details.
8739
8740 // Emit a library call.
8743 Entry.Ty = PointerType::getUnqual(*getContext());
8744 Entry.Node = Dst; Args.push_back(Entry);
8745 Entry.Node = Src; Args.push_back(Entry);
8746
8747 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8748 Entry.Node = Size; Args.push_back(Entry);
8749 // FIXME: pass in SDLoc
8751
8752 bool IsTailCall = false;
8753 if (OverrideTailCall.has_value()) {
8754 IsTailCall = *OverrideTailCall;
8755 } else {
8756 bool LowersToMemmove =
8757 TLI->getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove");
8758 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
8759 IsTailCall = CI && CI->isTailCall() &&
8761 ReturnsFirstArg && LowersToMemmove);
8762 }
8763
8764 CLI.setDebugLoc(dl)
8765 .setChain(Chain)
8766 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
8767 Dst.getValueType().getTypeForEVT(*getContext()),
8768 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
8769 TLI->getPointerTy(getDataLayout())),
8770 std::move(Args))
8772 .setTailCall(IsTailCall);
8773
8774 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8775 return CallResult.second;
8776}
8777
8779 SDValue Dst, SDValue Src, SDValue Size,
8780 Type *SizeTy, unsigned ElemSz,
8781 bool isTailCall,
8782 MachinePointerInfo DstPtrInfo,
8783 MachinePointerInfo SrcPtrInfo) {
8784 // Emit a library call.
8787 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8788 Entry.Node = Dst;
8789 Args.push_back(Entry);
8790
8791 Entry.Node = Src;
8792 Args.push_back(Entry);
8793
8794 Entry.Ty = SizeTy;
8795 Entry.Node = Size;
8796 Args.push_back(Entry);
8797
8798 RTLIB::Libcall LibraryCall =
8800 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8801 report_fatal_error("Unsupported element size");
8802
8804 CLI.setDebugLoc(dl)
8805 .setChain(Chain)
8806 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8808 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8809 TLI->getPointerTy(getDataLayout())),
8810 std::move(Args))
8812 .setTailCall(isTailCall);
8813
8814 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8815 return CallResult.second;
8816}
8817
8819 SDValue Src, SDValue Size, Align Alignment,
8820 bool isVol, bool AlwaysInline,
8821 const CallInst *CI,
8822 MachinePointerInfo DstPtrInfo,
8823 const AAMDNodes &AAInfo) {
8824 // Check to see if we should lower the memset to stores first.
8825 // For cases within the target-specified limits, this is the best choice.
8826 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8827 if (ConstantSize) {
8828 // Memset with size zero? Just return the original chain.
8829 if (ConstantSize->isZero())
8830 return Chain;
8831
8832 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8833 ConstantSize->getZExtValue(), Alignment,
8834 isVol, false, DstPtrInfo, AAInfo);
8835
8836 if (Result.getNode())
8837 return Result;
8838 }
8839
8840 // Then check to see if we should lower the memset with target-specific
8841 // code. If the target chooses to do this, this is the next best.
8842 if (TSI) {
8843 SDValue Result = TSI->EmitTargetCodeForMemset(
8844 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8845 if (Result.getNode())
8846 return Result;
8847 }
8848
8849 // If we really need inline code and the target declined to provide it,
8850 // use a (potentially long) sequence of loads and stores.
8851 if (AlwaysInline) {
8852 assert(ConstantSize && "AlwaysInline requires a constant size!");
8853 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8854 ConstantSize->getZExtValue(), Alignment,
8855 isVol, true, DstPtrInfo, AAInfo);
8856 assert(Result &&
8857 "getMemsetStores must return a valid sequence when AlwaysInline");
8858 return Result;
8859 }
8860
8862
8863 // Emit a library call.
8864 auto &Ctx = *getContext();
8865 const auto& DL = getDataLayout();
8866
8868 // FIXME: pass in SDLoc
8869 CLI.setDebugLoc(dl).setChain(Chain);
8870
8871 const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);
8872
8873 // Helper function to create an Entry from Node and Type.
8874 const auto CreateEntry = [](SDValue Node, Type *Ty) {
8876 Entry.Node = Node;
8877 Entry.Ty = Ty;
8878 return Entry;
8879 };
8880
8881 bool UseBZero = isNullConstant(Src) && BzeroName;
8882 // If zeroing out and bzero is present, use it.
8883 if (UseBZero) {
8885 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8886 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8887 CLI.setLibCallee(
8888 TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),
8889 getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));
8890 } else {
8892 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8893 Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8894 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8895 CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
8896 Dst.getValueType().getTypeForEVT(Ctx),
8897 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
8898 TLI->getPointerTy(DL)),
8899 std::move(Args));
8900 }
8901 bool LowersToMemset =
8902 TLI->getLibcallName(RTLIB::MEMSET) == StringRef("memset");
8903 // If we're going to use bzero, make sure not to tail call unless the
8904 // subsequent return doesn't need a value, as bzero doesn't return the first
8905 // arg unlike memset.
8906 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI) && !UseBZero;
8907 bool IsTailCall =
8908 CI && CI->isTailCall() &&
8909 isInTailCallPosition(*CI, getTarget(), ReturnsFirstArg && LowersToMemset);
8910 CLI.setDiscardResult().setTailCall(IsTailCall);
8911
8912 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8913 return CallResult.second;
8914}
8915
8918 Type *SizeTy, unsigned ElemSz,
8919 bool isTailCall,
8920 MachinePointerInfo DstPtrInfo) {
8921 // Emit a library call.
8924 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8925 Entry.Node = Dst;
8926 Args.push_back(Entry);
8927
8928 Entry.Ty = Type::getInt8Ty(*getContext());
8929 Entry.Node = Value;
8930 Args.push_back(Entry);
8931
8932 Entry.Ty = SizeTy;
8933 Entry.Node = Size;
8934 Args.push_back(Entry);
8935
8936 RTLIB::Libcall LibraryCall =
8938 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8939 report_fatal_error("Unsupported element size");
8940
8942 CLI.setDebugLoc(dl)
8943 .setChain(Chain)
8944 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8946 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8947 TLI->getPointerTy(getDataLayout())),
8948 std::move(Args))
8950 .setTailCall(isTailCall);
8951
8952 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8953 return CallResult.second;
8954}
8955
8956SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8957 SDVTList VTList, ArrayRef<SDValue> Ops,
8958 MachineMemOperand *MMO) {
8960 ID.AddInteger(MemVT.getRawBits());
8961 AddNodeIDNode(ID, Opcode, VTList, Ops);
8962 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8963 ID.AddInteger(MMO->getFlags());
8964 void* IP = nullptr;
8965 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8966 cast<AtomicSDNode>(E)->refineAlignment(MMO);
8967 return SDValue(E, 0);
8968 }
8969
8970 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8971 VTList, MemVT, MMO);
8972 createOperands(N, Ops);
8973
8974 CSEMap.InsertNode(N, IP);
8975 InsertNode(N);
8976 return SDValue(N, 0);
8977}
8978
8980 EVT MemVT, SDVTList VTs, SDValue Chain,
8981 SDValue Ptr, SDValue Cmp, SDValue Swp,
8982 MachineMemOperand *MMO) {
8983 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8985 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8986
8987 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8988 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8989}
8990
8991SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8992 SDValue Chain, SDValue Ptr, SDValue Val,
8993 MachineMemOperand *MMO) {
8994 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
8995 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
8996 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
8997 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
8998 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
8999 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
9000 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
9001 Opcode == ISD::ATOMIC_LOAD_FMIN ||
9002 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
9003 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
9004 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
9005 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
9006 Opcode == ISD::ATOMIC_STORE) &&
9007 "Invalid Atomic Op");
9008
9009 EVT VT = Val.getValueType();
9010
9011 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
9012 getVTList(VT, MVT::Other);
9013 SDValue Ops[] = {Chain, Ptr, Val};
9014 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
9015}
9016
9017SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9018 EVT VT, SDValue Chain, SDValue Ptr,
9019 MachineMemOperand *MMO) {
9020 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
9021
9022 SDVTList VTs = getVTList(VT, MVT::Other);
9023 SDValue Ops[] = {Chain, Ptr};
9024 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
9025}
9026
9027/// getMergeValues - Create a MERGE_VALUES node from the given operands.
9029 if (Ops.size() == 1)
9030 return Ops[0];
9031
9033 VTs.reserve(Ops.size());
9034 for (const SDValue &Op : Ops)
9035 VTs.push_back(Op.getValueType());
9036 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
9037}
9038
9040 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
9041 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
9043 const AAMDNodes &AAInfo) {
9044 if (Size.hasValue() && !Size.getValue())
9046
9048 MachineMemOperand *MMO =
9049 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
9050
9051 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
9052}
9053
9055 SDVTList VTList,
9056 ArrayRef<SDValue> Ops, EVT MemVT,
9057 MachineMemOperand *MMO) {
9058 assert(
9059 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
9060 Opcode == ISD::PREFETCH ||
9061 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
9062 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
9063 "Opcode is not a memory-accessing opcode!");
9064
9065 // Memoize the node unless it returns a glue result.
9067 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9069 AddNodeIDNode(ID, Opcode, VTList, Ops);
9070 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
9071 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
9072 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9073 ID.AddInteger(MMO->getFlags());
9074 ID.AddInteger(MemVT.getRawBits());
9075 void *IP = nullptr;
9076 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9077 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
9078 return SDValue(E, 0);
9079 }
9080
9081 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
9082 VTList, MemVT, MMO);
9083 createOperands(N, Ops);
9084
9085 CSEMap.InsertNode(N, IP);
9086 } else {
9087 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
9088 VTList, MemVT, MMO);
9089 createOperands(N, Ops);
9090 }
9091 InsertNode(N);
9092 SDValue V(N, 0);
9093 NewSDValueDbgMsg(V, "Creating new node: ", this);
9094 return V;
9095}
9096
9098 SDValue Chain, int FrameIndex,
9099 int64_t Size, int64_t Offset) {
9100 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
9101 const auto VTs = getVTList(MVT::Other);
9102 SDValue Ops[2] = {
9103 Chain,
9104 getFrameIndex(FrameIndex,
9105 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
9106 true)};
9107
9109 AddNodeIDNode(ID, Opcode, VTs, Ops);
9110 ID.AddInteger(FrameIndex);
9111 ID.AddInteger(Size);
9112 ID.AddInteger(Offset);
9113 void *IP = nullptr;
9114 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9115 return SDValue(E, 0);
9116
9117 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
9118 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
9119 createOperands(N, Ops);
9120 CSEMap.InsertNode(N, IP);
9121 InsertNode(N);
9122 SDValue V(N, 0);
9123 NewSDValueDbgMsg(V, "Creating new node: ", this);
9124 return V;
9125}
9126
9128 uint64_t Guid, uint64_t Index,
9129 uint32_t Attr) {
9130 const unsigned Opcode = ISD::PSEUDO_PROBE;
9131 const auto VTs = getVTList(MVT::Other);
9132 SDValue Ops[] = {Chain};
9134 AddNodeIDNode(ID, Opcode, VTs, Ops);
9135 ID.AddInteger(Guid);
9136 ID.AddInteger(Index);
9137 void *IP = nullptr;
9138 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
9139 return SDValue(E, 0);
9140
9141 auto *N = newSDNode<PseudoProbeSDNode>(
9142 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
9143 createOperands(N, Ops);
9144 CSEMap.InsertNode(N, IP);
9145 InsertNode(N);
9146 SDValue V(N, 0);
9147 NewSDValueDbgMsg(V, "Creating new node: ", this);
9148 return V;
9149}
9150
9151/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
9152/// MachinePointerInfo record from it. This is particularly useful because the
9153/// code generator has many cases where it doesn't bother passing in a
9154/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
9156 SelectionDAG &DAG, SDValue Ptr,
9157 int64_t Offset = 0) {
9158 // If this is FI+Offset, we can model it.
9159 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
9161 FI->getIndex(), Offset);
9162
9163 // If this is (FI+Offset1)+Offset2, we can model it.
9164 if (Ptr.getOpcode() != ISD::ADD ||
9165 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
9166 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
9167 return Info;
9168
9169 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
9171 DAG.getMachineFunction(), FI,
9172 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
9173}
9174
9175/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
9176/// MachinePointerInfo record from it. This is particularly useful because the
9177/// code generator has many cases where it doesn't bother passing in a
9178/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
9180 SelectionDAG &DAG, SDValue Ptr,
9181 SDValue OffsetOp) {
9182 // If the 'Offset' value isn't a constant, we can't handle this.
9183 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
9184 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
9185 if (OffsetOp.isUndef())
9186 return InferPointerInfo(Info, DAG, Ptr);
9187 return Info;
9188}
9189
9191 EVT VT, const SDLoc &dl, SDValue Chain,
9193 MachinePointerInfo PtrInfo, EVT MemVT,
9194 Align Alignment,
9195 MachineMemOperand::Flags MMOFlags,
9196 const AAMDNodes &AAInfo, const MDNode *Ranges) {
9197 assert(Chain.getValueType() == MVT::Other &&
9198 "Invalid chain type");
9199
9200 MMOFlags |= MachineMemOperand::MOLoad;
9201 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9202 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9203 // clients.
9204 if (PtrInfo.V.isNull())
9205 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
9206
9209 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
9210 Alignment, AAInfo, Ranges);
9211 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
9212}
9213
9215 EVT VT, const SDLoc &dl, SDValue Chain,
9216 SDValue Ptr, SDValue Offset, EVT MemVT,
9217 MachineMemOperand *MMO) {
9218 if (VT == MemVT) {
9219 ExtType = ISD::NON_EXTLOAD;
9220 } else if (ExtType == ISD::NON_EXTLOAD) {
9221 assert(VT == MemVT && "Non-extending load from different memory type!");
9222 } else {
9223 // Extending load.
9224 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
9225 "Should only be an extending load, not truncating!");
9226 assert(VT.isInteger() == MemVT.isInteger() &&
9227 "Cannot convert from FP to Int or Int -> FP!");
9228 assert(VT.isVector() == MemVT.isVector() &&
9229 "Cannot use an ext load to convert to or from a vector!");
9230 assert((!VT.isVector() ||
9232 "Cannot use an ext load to change the number of vector elements!");
9233 }
9234
9235 bool Indexed = AM != ISD::UNINDEXED;
9236 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9237
9238 SDVTList VTs = Indexed ?
9239 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
9240 SDValue Ops[] = { Chain, Ptr, Offset };
9242 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
9243 ID.AddInteger(MemVT.getRawBits());
9244 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
9245 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
9246 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9247 ID.AddInteger(MMO->getFlags());
9248 void *IP = nullptr;
9249 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9250 cast<LoadSDNode>(E)->refineAlignment(MMO);
9251 return SDValue(E, 0);
9252 }
9253 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9254 ExtType, MemVT, MMO);
9255 createOperands(N, Ops);
9256
9257 CSEMap.InsertNode(N, IP);
9258 InsertNode(N);
9259 SDValue V(N, 0);
9260 NewSDValueDbgMsg(V, "Creating new node: ", this);
9261 return V;
9262}
9263
9266 MaybeAlign Alignment,
9267 MachineMemOperand::Flags MMOFlags,
9268 const AAMDNodes &AAInfo, const MDNode *Ranges) {
9269 SDValue Undef = getUNDEF(Ptr.getValueType());
9270 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
9271 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
9272}
9273
9276 SDValue Undef = getUNDEF(Ptr.getValueType());
9277 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
9278 VT, MMO);
9279}
9280
9282 EVT VT, SDValue Chain, SDValue Ptr,
9283 MachinePointerInfo PtrInfo, EVT MemVT,
9284 MaybeAlign Alignment,
9285 MachineMemOperand::Flags MMOFlags,
9286 const AAMDNodes &AAInfo) {
9287 SDValue Undef = getUNDEF(Ptr.getValueType());
9288 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
9289 MemVT, Alignment, MMOFlags, AAInfo);
9290}
9291
9293 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
9294 MachineMemOperand *MMO) {
9295 SDValue Undef = getUNDEF(Ptr.getValueType());
9296 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
9297 MemVT, MMO);
9298}
9299
9303 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
9304 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
9305 // Don't propagate the invariant or dereferenceable flags.
9306 auto MMOFlags =
9307 LD->getMemOperand()->getFlags() &
9309 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
9310 LD->getChain(), Base, Offset, LD->getPointerInfo(),
9311 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
9312}
9313
9316 Align Alignment,
9317 MachineMemOperand::Flags MMOFlags,
9318 const AAMDNodes &AAInfo) {
9319 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9320
9321 MMOFlags |= MachineMemOperand::MOStore;
9322 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9323
9324 if (PtrInfo.V.isNull())
9325 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
9326
9329 MachineMemOperand *MMO =
9330 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
9331 return getStore(Chain, dl, Val, Ptr, MMO);
9332}
9333
9336 assert(Chain.getValueType() == MVT::Other &&
9337 "Invalid chain type");
9338 EVT VT = Val.getValueType();
9339 SDVTList VTs = getVTList(MVT::Other);
9340 SDValue Undef = getUNDEF(Ptr.getValueType());
9341 SDValue Ops[] = { Chain, Val, Ptr, Undef };
9343 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
9344 ID.AddInteger(VT.getRawBits());
9345 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
9346 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
9347 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9348 ID.AddInteger(MMO->getFlags());
9349 void *IP = nullptr;
9350 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9351 cast<StoreSDNode>(E)->refineAlignment(MMO);
9352 return SDValue(E, 0);
9353 }
9354 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9355 ISD::UNINDEXED, false, VT, MMO);
9356 createOperands(N, Ops);
9357
9358 CSEMap.InsertNode(N, IP);
9359 InsertNode(N);
9360 SDValue V(N, 0);
9361 NewSDValueDbgMsg(V, "Creating new node: ", this);
9362 return V;
9363}
9364
9367 EVT SVT, Align Alignment,
9368 MachineMemOperand::Flags MMOFlags,
9369 const AAMDNodes &AAInfo) {
9370 assert(Chain.getValueType() == MVT::Other &&
9371 "Invalid chain type");
9372
9373 MMOFlags |= MachineMemOperand::MOStore;
9374 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9375
9376 if (PtrInfo.V.isNull())
9377 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
9378
9381 PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
9382 AAInfo);
9383 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
9384}
9385
9387 SDValue Ptr, EVT SVT,
9388 MachineMemOperand *MMO) {
9389 EVT VT = Val.getValueType();
9390
9391 assert(Chain.getValueType() == MVT::Other &&
9392 "Invalid chain type");
9393 if (VT == SVT)
9394 return getStore(Chain, dl, Val, Ptr, MMO);
9395
9397 "Should only be a truncating store, not extending!");
9398 assert(VT.isInteger() == SVT.isInteger() &&
9399 "Can't do FP-INT conversion!");
9400 assert(VT.isVector() == SVT.isVector() &&
9401 "Cannot use trunc store to convert to or from a vector!");
9402 assert((!VT.isVector() ||
9404 "Cannot use trunc store to change the number of vector elements!");
9405
9406 SDVTList VTs = getVTList(MVT::Other);
9407 SDValue Undef = getUNDEF(Ptr.getValueType());
9408 SDValue Ops[] = { Chain, Val, Ptr, Undef };
9410 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
9411 ID.AddInteger(SVT.getRawBits());
9412 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
9413 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
9414 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9415 ID.AddInteger(MMO->getFlags());
9416 void *IP = nullptr;
9417 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9418 cast<StoreSDNode>(E)->refineAlignment(MMO);
9419 return SDValue(E, 0);
9420 }
9421 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9422 ISD::UNINDEXED, true, SVT, MMO);
9423 createOperands(N, Ops);
9424
9425 CSEMap.InsertNode(N, IP);
9426 InsertNode(N);
9427 SDValue V(N, 0);
9428 NewSDValueDbgMsg(V, "Creating new node: ", this);
9429 return V;
9430}
9431
9435 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
9436 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
9437 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9438 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
9440 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
9441 ID.AddInteger(ST->getMemoryVT().getRawBits());
9442 ID.AddInteger(ST->getRawSubclassData());
9443 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9444 ID.AddInteger(ST->getMemOperand()->getFlags());
9445 void *IP = nullptr;
9446 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9447 return SDValue(E, 0);
9448
9449 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9450 ST->isTruncatingStore(), ST->getMemoryVT(),
9451 ST->getMemOperand());
9452 createOperands(N, Ops);
9453
9454 CSEMap.InsertNode(N, IP);
9455 InsertNode(N);
9456 SDValue V(N, 0);
9457 NewSDValueDbgMsg(V, "Creating new node: ", this);
9458 return V;
9459}
9460
9462 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
9463 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
9464 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
9465 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9466 const MDNode *Ranges, bool IsExpanding) {
9467 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9468
9469 MMOFlags |= MachineMemOperand::MOLoad;
9470 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9471 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9472 // clients.
9473 if (PtrInfo.V.isNull())
9474 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
9475
9478 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
9479 Alignment, AAInfo, Ranges);
9480 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
9481 MMO, IsExpanding);
9482}
9483
9485 ISD::LoadExtType ExtType, EVT VT,
9486 const SDLoc &dl, SDValue Chain, SDValue Ptr,
9487 SDValue Offset, SDValue Mask, SDValue EVL,
9488 EVT MemVT, MachineMemOperand *MMO,
9489 bool IsExpanding) {
9490 bool Indexed = AM != ISD::UNINDEXED;
9491 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9492
9493 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9494 : getVTList(VT, MVT::Other);
9495 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
9497 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops);
9498 ID.AddInteger(MemVT.getRawBits());
9499 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
9500 dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9501 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9502 ID.AddInteger(MMO->getFlags());
9503 void *IP = nullptr;
9504 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9505 cast<VPLoadSDNode>(E)->refineAlignment(MMO);
9506 return SDValue(E, 0);
9507 }
9508 auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9509 ExtType, IsExpanding, MemVT, MMO);
9510 createOperands(N, Ops);
9511
9512 CSEMap.InsertNode(N, IP);
9513 InsertNode(N);
9514 SDValue V(N, 0);
9515 NewSDValueDbgMsg(V, "Creating new node: ", this);
9516 return V;
9517}
9518
9520 SDValue Ptr, SDValue Mask, SDValue EVL,
9521 MachinePointerInfo PtrInfo,
9522 MaybeAlign Alignment,
9523 MachineMemOperand::Flags MMOFlags,
9524 const AAMDNodes &AAInfo, const MDNode *Ranges,
9525 bool IsExpanding) {
9526 SDValue Undef = getUNDEF(Ptr.getValueType());
9527 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
9528 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
9529 IsExpanding);
9530}
9531
9533 SDValue Ptr, SDValue Mask, SDValue EVL,
9534 MachineMemOperand *MMO, bool IsExpanding) {
9535 SDValue Undef = getUNDEF(Ptr.getValueType());
9536 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
9537 Mask, EVL, VT, MMO, IsExpanding);
9538}
9539
9541 EVT VT, SDValue Chain, SDValue Ptr,
9542 SDValue Mask, SDValue EVL,
9543 MachinePointerInfo PtrInfo, EVT MemVT,
9544 MaybeAlign Alignment,
9545 MachineMemOperand::Flags MMOFlags,
9546 const AAMDNodes &AAInfo, bool IsExpanding) {
9547 SDValue Undef = getUNDEF(Ptr.getValueType());
9548 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
9549 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,
9550 IsExpanding);
9551}
9552
9554 EVT VT, SDValue Chain, SDValue Ptr,
9555 SDValue Mask, SDValue EVL, EVT MemVT,
9556 MachineMemOperand *MMO, bool IsExpanding) {
9557 SDValue Undef = getUNDEF(Ptr.getValueType());
9558 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
9559 EVL, MemVT, MMO, IsExpanding);
9560}
9561
9565 auto *LD = cast<VPLoadSDNode>(OrigLoad);
9566 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
9567 // Don't propagate the invariant or dereferenceable flags.
9568 auto MMOFlags =
9569 LD->getMemOperand()->getFlags() &
9571 return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
9572 LD->getChain(), Base, Offset, LD->getMask(),
9573 LD->getVectorLength(), LD->getPointerInfo(),
9574 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
9575 nullptr, LD->isExpandingLoad());
9576}
9577
9580 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
9581 ISD::MemIndexedMode AM, bool IsTruncating,
9582 bool IsCompressing) {
9583 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9584 bool Indexed = AM != ISD::UNINDEXED;
9585 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9586 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9587 : getVTList(MVT::Other);
9588 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
9590 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9591 ID.AddInteger(MemVT.getRawBits());
9592 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9593 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9594 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9595 ID.AddInteger(MMO->getFlags());
9596 void *IP = nullptr;
9597 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9598 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9599 return SDValue(E, 0);
9600 }
9601 auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9602 IsTruncating, IsCompressing, MemVT, MMO);
9603 createOperands(N, Ops);
9604
9605 CSEMap.InsertNode(N, IP);
9606 InsertNode(N);
9607 SDValue V(N, 0);
9608 NewSDValueDbgMsg(V, "Creating new node: ", this);
9609 return V;
9610}
9611
9613 SDValue Val, SDValue Ptr, SDValue Mask,
9614 SDValue EVL, MachinePointerInfo PtrInfo,
9615 EVT SVT, Align Alignment,
9616 MachineMemOperand::Flags MMOFlags,
9617 const AAMDNodes &AAInfo,
9618 bool IsCompressing) {
9619 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9620
9621 MMOFlags |= MachineMemOperand::MOStore;
9622 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9623
9624 if (PtrInfo.V.isNull())
9625 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
9626
9629 PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
9630 AAInfo);
9631 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
9632 IsCompressing);
9633}
9634
9636 SDValue Val, SDValue Ptr, SDValue Mask,
9637 SDValue EVL, EVT SVT,
9638 MachineMemOperand *MMO,
9639 bool IsCompressing) {
9640 EVT VT = Val.getValueType();
9641
9642 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9643 if (VT == SVT)
9644 return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask,
9645 EVL, VT, MMO, ISD::UNINDEXED,
9646 /*IsTruncating*/ false, IsCompressing);
9647
9649 "Should only be a truncating store, not extending!");
9650 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9651 assert(VT.isVector() == SVT.isVector() &&
9652 "Cannot use trunc store to convert to or from a vector!");
9653 assert((!VT.isVector() ||
9655 "Cannot use trunc store to change the number of vector elements!");
9656
9657 SDVTList VTs = getVTList(MVT::Other);
9658 SDValue Undef = getUNDEF(Ptr.getValueType());
9659 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
9661 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9662 ID.AddInteger(SVT.getRawBits());
9663 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9664 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9665 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9666 ID.AddInteger(MMO->getFlags());
9667 void *IP = nullptr;
9668 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9669 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9670 return SDValue(E, 0);
9671 }
9672 auto *N =
9673 newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9674 ISD::UNINDEXED, true, IsCompressing, SVT, MMO);
9675 createOperands(N, Ops);
9676
9677 CSEMap.InsertNode(N, IP);
9678 InsertNode(N);
9679 SDValue V(N, 0);
9680 NewSDValueDbgMsg(V, "Creating new node: ", this);
9681 return V;
9682}
9683
9687 auto *ST = cast<VPStoreSDNode>(OrigStore);
9688 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
9689 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9690 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
9691 Offset, ST->getMask(), ST->getVectorLength()};
9693 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9694 ID.AddInteger(ST->getMemoryVT().getRawBits());
9695 ID.AddInteger(ST->getRawSubclassData());
9696 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9697 ID.AddInteger(ST->getMemOperand()->getFlags());
9698 void *IP = nullptr;
9699 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9700 return SDValue(E, 0);
9701
9702 auto *N = newSDNode<VPStoreSDNode>(
9703 dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(),
9704 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
9705 createOperands(N, Ops);
9706
9707 CSEMap.InsertNode(N, IP);
9708 InsertNode(N);
9709 SDValue V(N, 0);
9710 NewSDValueDbgMsg(V, "Creating new node: ", this);
9711 return V;
9712}
9713
9715 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9716 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9717 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9718 bool Indexed = AM != ISD::UNINDEXED;
9719 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9720
9721 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9722 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9723 : getVTList(VT, MVT::Other);
9725 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
9726 ID.AddInteger(VT.getRawBits());
9727 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9728 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9729 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9730
9731 void *IP = nullptr;
9732 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9733 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9734 return SDValue(E, 0);
9735 }
9736
9737 auto *N =
9738 newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,
9739 ExtType, IsExpanding, MemVT, MMO);
9740 createOperands(N, Ops);
9741 CSEMap.InsertNode(N, IP);
9742 InsertNode(N);
9743 SDValue V(N, 0);
9744 NewSDValueDbgMsg(V, "Creating new node: ", this);
9745 return V;
9746}
9747
9749 SDValue Ptr, SDValue Stride,
9750 SDValue Mask, SDValue EVL,
9751 MachineMemOperand *MMO,
9752 bool IsExpanding) {
9753 SDValue Undef = getUNDEF(Ptr.getValueType());
9755 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9756}
9757
9759 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9760 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9761 MachineMemOperand *MMO, bool IsExpanding) {
9762 SDValue Undef = getUNDEF(Ptr.getValueType());
9763 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9764 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9765}
9766
9768 SDValue Val, SDValue Ptr,
9769 SDValue Offset, SDValue Stride,
9770 SDValue Mask, SDValue EVL, EVT MemVT,
9771 MachineMemOperand *MMO,
9773 bool IsTruncating, bool IsCompressing) {
9774 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9775 bool Indexed = AM != ISD::UNINDEXED;
9776 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9777 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9778 : getVTList(MVT::Other);
9779 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9781 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9782 ID.AddInteger(MemVT.getRawBits());
9783 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9784 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9785 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9786 void *IP = nullptr;
9787 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9788 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9789 return SDValue(E, 0);
9790 }
9791 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9792 VTs, AM, IsTruncating,
9793 IsCompressing, MemVT, MMO);
9794 createOperands(N, Ops);
9795
9796 CSEMap.InsertNode(N, IP);
9797 InsertNode(N);
9798 SDValue V(N, 0);
9799 NewSDValueDbgMsg(V, "Creating new node: ", this);
9800 return V;
9801}
9802
9804 SDValue Val, SDValue Ptr,
9805 SDValue Stride, SDValue Mask,
9806 SDValue EVL, EVT SVT,
9807 MachineMemOperand *MMO,
9808 bool IsCompressing) {
9809 EVT VT = Val.getValueType();
9810
9811 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9812 if (VT == SVT)
9813 return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()),
9814 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED,
9815 /*IsTruncating*/ false, IsCompressing);
9816
9818 "Should only be a truncating store, not extending!");
9819 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9820 assert(VT.isVector() == SVT.isVector() &&
9821 "Cannot use trunc store to convert to or from a vector!");
9822 assert((!VT.isVector() ||
9824 "Cannot use trunc store to change the number of vector elements!");
9825
9826 SDVTList VTs = getVTList(MVT::Other);
9827 SDValue Undef = getUNDEF(Ptr.getValueType());
9828 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9830 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9831 ID.AddInteger(SVT.getRawBits());
9832 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9833 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9834 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9835 void *IP = nullptr;
9836 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9837 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9838 return SDValue(E, 0);
9839 }
9840 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9841 VTs, ISD::UNINDEXED, true,
9842 IsCompressing, SVT, MMO);
9843 createOperands(N, Ops);
9844
9845 CSEMap.InsertNode(N, IP);
9846 InsertNode(N);
9847 SDValue V(N, 0);
9848 NewSDValueDbgMsg(V, "Creating new node: ", this);
9849 return V;
9850}
9851
9854 ISD::MemIndexType IndexType) {
9855 assert(Ops.size() == 6 && "Incompatible number of operands");
9856
9858 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops);
9859 ID.AddInteger(VT.getRawBits());
9860 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9861 dl.getIROrder(), VTs, VT, MMO, IndexType));
9862 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9863 ID.AddInteger(MMO->getFlags());
9864 void *IP = nullptr;
9865 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9866 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9867 return SDValue(E, 0);
9868 }
9869
9870 auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9871 VT, MMO, IndexType);
9872 createOperands(N, Ops);
9873
9874 assert(N->getMask().getValueType().getVectorElementCount() ==
9875 N->getValueType(0).getVectorElementCount() &&
9876 "Vector width mismatch between mask and data");
9877 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9878 N->getValueType(0).getVectorElementCount().isScalable() &&
9879 "Scalable flags of index and data do not match");
9881 N->getIndex().getValueType().getVectorElementCount(),
9882 N->getValueType(0).getVectorElementCount()) &&
9883 "Vector width mismatch between index and data");
9884 assert(isa<ConstantSDNode>(N->getScale()) &&
9885 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9886 "Scale should be a constant power of 2");
9887
9888 CSEMap.InsertNode(N, IP);
9889 InsertNode(N);
9890 SDValue V(N, 0);
9891 NewSDValueDbgMsg(V, "Creating new node: ", this);
9892 return V;
9893}
9894
9897 MachineMemOperand *MMO,
9898 ISD::MemIndexType IndexType) {
9899 assert(Ops.size() == 7 && "Incompatible number of operands");
9900
9902 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops);
9903 ID.AddInteger(VT.getRawBits());
9904 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9905 dl.getIROrder(), VTs, VT, MMO, IndexType));
9906 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9907 ID.AddInteger(MMO->getFlags());
9908 void *IP = nullptr;
9909 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9910 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9911 return SDValue(E, 0);
9912 }
9913 auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9914 VT, MMO, IndexType);
9915 createOperands(N, Ops);
9916
9917 assert(N->getMask().getValueType().getVectorElementCount() ==
9918 N->getValue().getValueType().getVectorElementCount() &&
9919 "Vector width mismatch between mask and data");
9920 assert(
9921 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9922 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9923 "Scalable flags of index and data do not match");
9925 N->getIndex().getValueType().getVectorElementCount(),
9926 N->getValue().getValueType().getVectorElementCount()) &&
9927 "Vector width mismatch between index and data");
9928 assert(isa<ConstantSDNode>(N->getScale()) &&
9929 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9930 "Scale should be a constant power of 2");
9931
9932 CSEMap.InsertNode(N, IP);
9933 InsertNode(N);
9934 SDValue V(N, 0);
9935 NewSDValueDbgMsg(V, "Creating new node: ", this);
9936 return V;
9937}
9938
9941 SDValue PassThru, EVT MemVT,
9942 MachineMemOperand *MMO,
9944 ISD::LoadExtType ExtTy, bool isExpanding) {
9945 bool Indexed = AM != ISD::UNINDEXED;
9946 assert((Indexed || Offset.isUndef()) &&
9947 "Unindexed masked load with an offset!");
9948 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9949 : getVTList(VT, MVT::Other);
9950 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9952 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
9953 ID.AddInteger(MemVT.getRawBits());
9954 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9955 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9956 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9957 ID.AddInteger(MMO->getFlags());
9958 void *IP = nullptr;
9959 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9960 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9961 return SDValue(E, 0);
9962 }
9963 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9964 AM, ExtTy, isExpanding, MemVT, MMO);
9965 createOperands(N, Ops);
9966
9967 CSEMap.InsertNode(N, IP);
9968 InsertNode(N);
9969 SDValue V(N, 0);
9970 NewSDValueDbgMsg(V, "Creating new node: ", this);
9971 return V;
9972}
9973
9977 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
9978 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9979 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
9980 Offset, LD->getMask(), LD->getPassThru(),
9981 LD->getMemoryVT(), LD->getMemOperand(), AM,
9982 LD->getExtensionType(), LD->isExpandingLoad());
9983}
9984
9987 SDValue Mask, EVT MemVT,
9988 MachineMemOperand *MMO,
9989 ISD::MemIndexedMode AM, bool IsTruncating,
9990 bool IsCompressing) {
9991 assert(Chain.getValueType() == MVT::Other &&
9992 "Invalid chain type");
9993 bool Indexed = AM != ISD::UNINDEXED;
9994 assert((Indexed || Offset.isUndef()) &&
9995 "Unindexed masked store with an offset!");
9996 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9997 : getVTList(MVT::Other);
9998 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
10000 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
10001 ID.AddInteger(MemVT.getRawBits());
10002 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
10003 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
10004 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10005 ID.AddInteger(MMO->getFlags());
10006 void *IP = nullptr;
10007 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
10008 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
10009 return SDValue(E, 0);
10010 }
10011 auto *N =
10012 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
10013 IsTruncating, IsCompressing, MemVT, MMO);
10014 createOperands(N, Ops);
10015
10016 CSEMap.InsertNode(N, IP);
10017 InsertNode(N);
10018 SDValue V(N, 0);
10019 NewSDValueDbgMsg(V, "Creating new node: ", this);
10020 return V;
10021}
10022
10026 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
10027 assert(ST->getOffset().isUndef() &&
10028 "Masked store is already a indexed store!");
10029 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
10030 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
10031 AM, ST->isTruncatingStore(), ST->isCompressingStore());
10032}
10033
10036 MachineMemOperand *MMO,
10037 ISD::MemIndexType IndexType,
10038 ISD::LoadExtType ExtTy) {
10039 assert(Ops.size() == 6 && "Incompatible number of operands");
10040
10042 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
10043 ID.AddInteger(MemVT.getRawBits());
10044 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
10045 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
10046 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10047 ID.AddInteger(MMO->getFlags());
10048 void *IP = nullptr;
10049 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
10050 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
10051 return SDValue(E, 0);
10052 }
10053
10054 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
10055 VTs, MemVT, MMO, IndexType, ExtTy);
10056 createOperands(N, Ops);
10057
10058 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
10059 "Incompatible type of the PassThru value in MaskedGatherSDNode");
10060 assert(N->getMask().getValueType().getVectorElementCount() ==
10061 N->getValueType(0).getVectorElementCount() &&
10062 "Vector width mismatch between mask and data");
10063 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10064 N->getValueType(0).getVectorElementCount().isScalable() &&
10065 "Scalable flags of index and data do not match");
10067 N->getIndex().getValueType().getVectorElementCount(),
10068 N->getValueType(0).getVectorElementCount()) &&
10069 "Vector width mismatch between index and data");
10070 assert(isa<ConstantSDNode>(N->getScale()) &&
10071 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10072 "Scale should be a constant power of 2");
10073
10074 CSEMap.InsertNode(N, IP);
10075 InsertNode(N);
10076 SDValue V(N, 0);
10077 NewSDValueDbgMsg(V, "Creating new node: ", this);
10078 return V;
10079}
10080
10083 MachineMemOperand *MMO,
10084 ISD::MemIndexType IndexType,
10085 bool IsTrunc) {
10086 assert(Ops.size() == 6 && "Incompatible number of operands");
10087
10089 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
10090 ID.AddInteger(MemVT.getRawBits());
10091 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
10092 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
10093 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10094 ID.AddInteger(MMO->getFlags());
10095 void *IP = nullptr;
10096 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
10097 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
10098 return SDValue(E, 0);
10099 }
10100
10101 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
10102 VTs, MemVT, MMO, IndexType, IsTrunc);
10103 createOperands(N, Ops);
10104
10105 assert(N->getMask().getValueType().getVectorElementCount() ==
10106 N->getValue().getValueType().getVectorElementCount() &&
10107 "Vector width mismatch between mask and data");
10108 assert(
10109 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10110 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10111 "Scalable flags of index and data do not match");
10113 N->getIndex().getValueType().getVectorElementCount(),
10114 N->getValue().getValueType().getVectorElementCount()) &&
10115 "Vector width mismatch between index and data");
10116 assert(isa<ConstantSDNode>(N->getScale()) &&
10117 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10118 "Scale should be a constant power of 2");
10119
10120 CSEMap.InsertNode(N, IP);
10121 InsertNode(N);
10122 SDValue V(N, 0);
10123 NewSDValueDbgMsg(V, "Creating new node: ", this);
10124 return V;
10125}
10126
10128 const SDLoc &dl, ArrayRef<SDValue> Ops,
10129 MachineMemOperand *MMO,
10130 ISD::MemIndexType IndexType) {
10131 assert(Ops.size() == 7 && "Incompatible number of operands");
10132
10135 ID.AddInteger(MemVT.getRawBits());
10136 ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
10137 dl.getIROrder(), VTs, MemVT, MMO, IndexType));
10138 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10139 ID.AddInteger(MMO->getFlags());
10140 void *IP = nullptr;
10141 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
10142 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
10143 return SDValue(E, 0);
10144 }
10145
10146 auto *N = newSDNode<MaskedHistogramSDNode>(dl.getIROrder(), dl.getDebugLoc(),
10147 VTs, MemVT, MMO, IndexType);
10148 createOperands(N, Ops);
10149
10150 assert(N->getMask().getValueType().getVectorElementCount() ==
10151 N->getIndex().getValueType().getVectorElementCount() &&
10152 "Vector width mismatch between mask and data");
10153 assert(isa<ConstantSDNode>(N->getScale()) &&
10154 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10155 "Scale should be a constant power of 2");
10156 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
10157
10158 CSEMap.InsertNode(N, IP);
10159 InsertNode(N);
10160 SDValue V(N, 0);
10161 NewSDValueDbgMsg(V, "Creating new node: ", this);
10162 return V;
10163}
10164
10166 EVT MemVT, MachineMemOperand *MMO) {
10167 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10168 SDVTList VTs = getVTList(MVT::Other);
10169 SDValue Ops[] = {Chain, Ptr};
10172 ID.AddInteger(MemVT.getRawBits());
10173 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10174 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
10175 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10176 ID.AddInteger(MMO->getFlags());
10177 void *IP = nullptr;
10178 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
10179 return SDValue(E, 0);
10180
10181 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),
10182 dl.getDebugLoc(), VTs, MemVT, MMO);
10183 createOperands(N, Ops);
10184
10185 CSEMap.InsertNode(N, IP);
10186 InsertNode(N);
10187 SDValue V(N, 0);
10188 NewSDValueDbgMsg(V, "Creating new node: ", this);
10189 return V;
10190}
10191
10193 EVT MemVT, MachineMemOperand *MMO) {
10194 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10195 SDVTList VTs = getVTList(MVT::Other);
10196 SDValue Ops[] = {Chain, Ptr};
10199 ID.AddInteger(MemVT.getRawBits());
10200 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10201 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
10202 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10203 ID.AddInteger(MMO->getFlags());
10204 void *IP = nullptr;
10205 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
10206 return SDValue(E, 0);
10207
10208 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),
10209 dl.getDebugLoc(), VTs, MemVT, MMO);
10210 createOperands(N, Ops);
10211
10212 CSEMap.InsertNode(N, IP);
10213 InsertNode(N);
10214 SDValue V(N, 0);
10215 NewSDValueDbgMsg(V, "Creating new node: ", this);
10216 return V;
10217}
10218
10220 // select undef, T, F --> T (if T is a constant), otherwise F
10221 // select, ?, undef, F --> F
10222 // select, ?, T, undef --> T
10223 if (Cond.isUndef())
10224 return isConstantValueOfAnyType(T) ? T : F;
10225 if (T.isUndef())
10226 return F;
10227 if (F.isUndef())
10228 return T;
10229
10230 // select true, T, F --> T
10231 // select false, T, F --> F
10232 if (auto C = isBoolConstant(Cond, /*AllowTruncation=*/true))
10233 return *C ? T : F;
10234
10235 // select ?, T, T --> T
10236 if (T == F)
10237 return T;
10238
10239 return SDValue();
10240}
10241
10243 // shift undef, Y --> 0 (can always assume that the undef value is 0)
10244 if (X.isUndef())
10245 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
10246 // shift X, undef --> undef (because it may shift by the bitwidth)
10247 if (Y.isUndef())
10248 return getUNDEF(X.getValueType());
10249
10250 // shift 0, Y --> 0
10251 // shift X, 0 --> X
10253 return X;
10254
10255 // shift X, C >= bitwidth(X) --> undef
10256 // All vector elements must be too big (or undef) to avoid partial undefs.
10257 auto isShiftTooBig = [X](ConstantSDNode *Val) {
10258 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
10259 };
10260 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
10261 return getUNDEF(X.getValueType());
10262
10263 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
10264 if (X.getValueType().getScalarType() == MVT::i1)
10265 return X;
10266
10267 return SDValue();
10268}
10269
10271 SDNodeFlags Flags) {
10272 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
10273 // (an undef operand can be chosen to be Nan/Inf), then the result of this
10274 // operation is poison. That result can be relaxed to undef.
10275 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
10276 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
10277 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
10278 (YC && YC->getValueAPF().isNaN());
10279 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
10280 (YC && YC->getValueAPF().isInfinity());
10281
10282 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
10283 return getUNDEF(X.getValueType());
10284
10285 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
10286 return getUNDEF(X.getValueType());
10287
10288 if (!YC)
10289 return SDValue();
10290
10291 // X + -0.0 --> X
10292 if (Opcode == ISD::FADD)
10293 if (YC->getValueAPF().isNegZero())
10294 return X;
10295
10296 // X - +0.0 --> X
10297 if (Opcode == ISD::FSUB)
10298 if (YC->getValueAPF().isPosZero())
10299 return X;
10300
10301 // X * 1.0 --> X
10302 // X / 1.0 --> X
10303 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
10304 if (YC->getValueAPF().isExactlyValue(1.0))
10305 return X;
10306
10307 // X * 0.0 --> 0.0
10308 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
10309 if (YC->getValueAPF().isZero())
10310 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
10311
10312 return SDValue();
10313}
10314
10316 SDValue Ptr, SDValue SV, unsigned Align) {
10317 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
10318 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
10319}
10320
10321SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
10322 ArrayRef<SDUse> Ops) {
10323 switch (Ops.size()) {
10324 case 0: return getNode(Opcode, DL, VT);
10325 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
10326 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
10327 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
10328 default: break;
10329 }
10330
10331 // Copy from an SDUse array into an SDValue array for use with
10332 // the regular getNode logic.
10333 SmallVector<SDValue, 8> NewOps(Ops);
10334 return getNode(Opcode, DL, VT, NewOps);
10335}
10336
10337SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
10338 ArrayRef<SDValue> Ops) {
10339 SDNodeFlags Flags;
10340 if (Inserter)
10341 Flags = Inserter->getFlags();
10342 return getNode(Opcode, DL, VT, Ops, Flags);
10343}
10344
10345SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
10346 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
10347 unsigned NumOps = Ops.size();
10348 switch (NumOps) {
10349 case 0: return getNode(Opcode, DL, VT);
10350 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
10351 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
10352 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
10353 default: break;
10354 }
10355
10356#ifndef NDEBUG
10357 for (const auto &Op : Ops)
10358 assert(Op.getOpcode() != ISD::DELETED_NODE &&
10359 "Operand is DELETED_NODE!");
10360#endif
10361
10362 switch (Opcode) {
10363 default: break;
10364 case ISD::BUILD_VECTOR:
10365 // Attempt to simplify BUILD_VECTOR.
10366 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
10367 return V;
10368 break;
10370 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
10371 return V;
10372 break;
10373 case ISD::SELECT_CC:
10374 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
10375 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
10376 "LHS and RHS of condition must have same type!");
10377 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
10378 "True and False arms of SelectCC must have same type!");
10379 assert(Ops[2].getValueType() == VT &&
10380 "select_cc node must be of same type as true and false value!");
10381 assert((!Ops[0].getValueType().isVector() ||
10382 Ops[0].getValueType().getVectorElementCount() ==
10383 VT.getVectorElementCount()) &&
10384 "Expected select_cc with vector result to have the same sized "
10385 "comparison type!");
10386 break;
10387 case ISD::BR_CC:
10388 assert(NumOps == 5 && "BR_CC takes 5 operands!");
10389 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
10390 "LHS/RHS of comparison should match types!");
10391 break;
10392 case ISD::VP_ADD:
10393 case ISD::VP_SUB:
10394 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
10395 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
10396 Opcode = ISD::VP_XOR;
10397 break;
10398 case ISD::VP_MUL:
10399 // If it is VP_MUL mask operation then turn it to VP_AND
10400 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
10401 Opcode = ISD::VP_AND;
10402 break;
10403 case ISD::VP_REDUCE_MUL:
10404 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
10405 if (VT == MVT::i1)
10406 Opcode = ISD::VP_REDUCE_AND;
10407 break;
10408 case ISD::VP_REDUCE_ADD:
10409 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
10410 if (VT == MVT::i1)
10411 Opcode = ISD::VP_REDUCE_XOR;
10412 break;
10413 case ISD::VP_REDUCE_SMAX:
10414 case ISD::VP_REDUCE_UMIN:
10415 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
10416 // VP_REDUCE_AND.
10417 if (VT == MVT::i1)
10418 Opcode = ISD::VP_REDUCE_AND;
10419 break;
10420 case ISD::VP_REDUCE_SMIN:
10421 case ISD::VP_REDUCE_UMAX:
10422 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
10423 // VP_REDUCE_OR.
10424 if (VT == MVT::i1)
10425 Opcode = ISD::VP_REDUCE_OR;
10426 break;
10427 }
10428
10429 // Memoize nodes.
10430 SDNode *N;
10431 SDVTList VTs = getVTList(VT);
10432
10433 if (VT != MVT::Glue) {
10435 AddNodeIDNode(ID, Opcode, VTs, Ops);
10436 void *IP = nullptr;
10437
10438 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
10439 E->intersectFlagsWith(Flags);
10440 return SDValue(E, 0);
10441 }
10442
10443 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
10444 createOperands(N, Ops);
10445
10446 CSEMap.InsertNode(N, IP);
10447 } else {
10448 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
10449 createOperands(N, Ops);
10450 }
10451
10452 N->setFlags(Flags);
10453 InsertNode(N);
10454 SDValue V(N, 0);
10455 NewSDValueDbgMsg(V, "Creating new node: ", this);
10456 return V;
10457}
10458
10459SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10460 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
10461 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
10462}
10463
10464SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10465 ArrayRef<SDValue> Ops) {
10466 SDNodeFlags Flags;
10467 if (Inserter)
10468 Flags = Inserter->getFlags();
10469 return getNode(Opcode, DL, VTList, Ops, Flags);
10470}
10471
10472SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10473 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
10474 if (VTList.NumVTs == 1)
10475 return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
10476
10477#ifndef NDEBUG
10478 for (const auto &Op : Ops)
10479 assert(Op.getOpcode() != ISD::DELETED_NODE &&
10480 "Operand is DELETED_NODE!");
10481#endif
10482
10483 switch (Opcode) {
10484 case ISD::SADDO:
10485 case ISD::UADDO:
10486 case ISD::SSUBO:
10487 case ISD::USUBO: {
10488 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
10489 "Invalid add/sub overflow op!");
10490 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
10491 Ops[0].getValueType() == Ops[1].getValueType() &&
10492 Ops[0].getValueType() == VTList.VTs[0] &&
10493 "Binary operator types must match!");
10494 SDValue N1 = Ops[0], N2 = Ops[1];
10495 canonicalizeCommutativeBinop(Opcode, N1, N2);
10496
10497 // (X +- 0) -> X with zero-overflow.
10498 ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,
10499 /*AllowTruncation*/ true);
10500 if (N2CV && N2CV->isZero()) {
10501 SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
10502 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
10503 }
10504
10505 if (VTList.VTs[0].isVector() &&
10506 VTList.VTs[0].getVectorElementType() == MVT::i1 &&
10507 VTList.VTs[1].getVectorElementType() == MVT::i1) {
10508 SDValue F1 = getFreeze(N1);
10509 SDValue F2 = getFreeze(N2);
10510 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
10511 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
10512 return getNode(ISD::MERGE_VALUES, DL, VTList,
10513 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
10514 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
10515 Flags);
10516 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
10517 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
10518 SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
10519 return getNode(ISD::MERGE_VALUES, DL, VTList,
10520 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
10521 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
10522 Flags);
10523 }
10524 }
10525 break;
10526 }
10527 case ISD::SADDO_CARRY:
10528 case ISD::UADDO_CARRY:
10529 case ISD::SSUBO_CARRY:
10530 case ISD::USUBO_CARRY:
10531 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
10532 "Invalid add/sub overflow op!");
10533 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
10534 Ops[0].getValueType() == Ops[1].getValueType() &&
10535 Ops[0].getValueType() == VTList.VTs[0] &&
10536 Ops[2].getValueType() == VTList.VTs[1] &&
10537 "Binary operator types must match!");
10538 break;
10539 case ISD::SMUL_LOHI:
10540 case ISD::UMUL_LOHI: {
10541 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
10542 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
10543 VTList.VTs[0] == Ops[0].getValueType() &&
10544 VTList.VTs[0] == Ops[1].getValueType() &&
10545 "Binary operator types must match!");
10546 // Constant fold.
10547 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Ops[0]);
10548 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ops[1]);
10549 if (LHS && RHS) {
10550 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
10551 unsigned OutWidth = Width * 2;
10552 APInt Val = LHS->getAPIntValue();
10553 APInt Mul = RHS->getAPIntValue();
10554 if (Opcode == ISD::SMUL_LOHI) {
10555 Val = Val.sext(OutWidth);
10556 Mul = Mul.sext(OutWidth);
10557 } else {
10558 Val = Val.zext(OutWidth);
10559 Mul = Mul.zext(OutWidth);
10560 }
10561 Val *= Mul;
10562
10563 SDValue Hi =
10564 getConstant(Val.extractBits(Width, Width), DL, VTList.VTs[0]);
10565 SDValue Lo = getConstant(Val.trunc(Width), DL, VTList.VTs[0]);
10566 return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags);
10567 }
10568 break;
10569 }
10570 case ISD::FFREXP: {
10571 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
10572 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
10573 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
10574
10575 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Ops[0])) {
10576 int FrexpExp;
10577 APFloat FrexpMant =
10578 frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);
10579 SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);
10580 SDValue Result1 =
10581 getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]);
10582 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);
10583 }
10584
10585 break;
10586 }
10588 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
10589 "Invalid STRICT_FP_EXTEND!");
10590 assert(VTList.VTs[0].isFloatingPoint() &&
10591 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
10592 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
10593 "STRICT_FP_EXTEND result type should be vector iff the operand "
10594 "type is vector!");
10595 assert((!VTList.VTs[0].isVector() ||
10596 VTList.VTs[0].getVectorElementCount() ==
10597 Ops[1].getValueType().getVectorElementCount()) &&
10598 "Vector element count mismatch!");
10599 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
10600 "Invalid fpext node, dst <= src!");
10601 break;
10603 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
10604 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
10605 "STRICT_FP_ROUND result type should be vector iff the operand "
10606 "type is vector!");
10607 assert((!VTList.VTs[0].isVector() ||
10608 VTList.VTs[0].getVectorElementCount() ==
10609 Ops[1].getValueType().getVectorElementCount()) &&
10610 "Vector element count mismatch!");
10611 assert(VTList.VTs[0].isFloatingPoint() &&
10612 Ops[1].getValueType().isFloatingPoint() &&
10613 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
10614 Ops[2].getOpcode() == ISD::TargetConstant &&
10615 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
10616 "Invalid STRICT_FP_ROUND!");
10617 break;
10618#if 0
10619 // FIXME: figure out how to safely handle things like
10620 // int foo(int x) { return 1 << (x & 255); }
10621 // int bar() { return foo(256); }
10622 case ISD::SRA_PARTS:
10623 case ISD::SRL_PARTS:
10624 case ISD::SHL_PARTS:
10625 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
10626 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
10627 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10628 else if (N3.getOpcode() == ISD::AND)
10629 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
10630 // If the and is only masking out bits that cannot effect the shift,
10631 // eliminate the and.
10632 unsigned NumBits = VT.getScalarSizeInBits()*2;
10633 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
10634 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10635 }
10636 break;
10637#endif
10638 }
10639
10640 // Memoize the node unless it returns a glue result.
10641 SDNode *N;
10642 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10644 AddNodeIDNode(ID, Opcode, VTList, Ops);
10645 void *IP = nullptr;
10646 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
10647 E->intersectFlagsWith(Flags);
10648 return SDValue(E, 0);
10649 }
10650
10651 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10652 createOperands(N, Ops);
10653 CSEMap.InsertNode(N, IP);
10654 } else {
10655 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10656 createOperands(N, Ops);
10657 }
10658
10659 N->setFlags(Flags);
10660 InsertNode(N);
10661 SDValue V(N, 0);
10662 NewSDValueDbgMsg(V, "Creating new node: ", this);
10663 return V;
10664}
10665
10666SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10667 SDVTList VTList) {
10668 return getNode(Opcode, DL, VTList, ArrayRef<SDValue>());
10669}
10670
10671SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10672 SDValue N1) {
10673 SDValue Ops[] = { N1 };
10674 return getNode(Opcode, DL, VTList, Ops);
10675}
10676
10677SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10678 SDValue N1, SDValue N2) {
10679 SDValue Ops[] = { N1, N2 };
10680 return getNode(Opcode, DL, VTList, Ops);
10681}
10682
10683SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10684 SDValue N1, SDValue N2, SDValue N3) {
10685 SDValue Ops[] = { N1, N2, N3 };
10686 return getNode(Opcode, DL, VTList, Ops);
10687}
10688
10689SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10690 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10691 SDValue Ops[] = { N1, N2, N3, N4 };
10692 return getNode(Opcode, DL, VTList, Ops);
10693}
10694
10695SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10696 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10697 SDValue N5) {
10698 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10699 return getNode(Opcode, DL, VTList, Ops);
10700}
10701
10703 if (!VT.isExtended())
10704 return makeVTList(SDNode::getValueTypeList(VT.getSimpleVT()), 1);
10705
10706 return makeVTList(&(*EVTs.insert(VT).first), 1);
10707}
10708
10711 ID.AddInteger(2U);
10712 ID.AddInteger(VT1.getRawBits());
10713 ID.AddInteger(VT2.getRawBits());
10714
10715 void *IP = nullptr;
10716 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10717 if (!Result) {
10718 EVT *Array = Allocator.Allocate<EVT>(2);
10719 Array[0] = VT1;
10720 Array[1] = VT2;
10721 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10722 VTListMap.InsertNode(Result, IP);
10723 }
10724 return Result->getSDVTList();
10725}
10726
10729 ID.AddInteger(3U);
10730 ID.AddInteger(VT1.getRawBits());
10731 ID.AddInteger(VT2.getRawBits());
10732 ID.AddInteger(VT3.getRawBits());
10733
10734 void *IP = nullptr;
10735 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10736 if (!Result) {
10737 EVT *Array = Allocator.Allocate<EVT>(3);
10738 Array[0] = VT1;
10739 Array[1] = VT2;
10740 Array[2] = VT3;
10741 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10742 VTListMap.InsertNode(Result, IP);
10743 }
10744 return Result->getSDVTList();
10745}
10746
10749 ID.AddInteger(4U);
10750 ID.AddInteger(VT1.getRawBits());
10751 ID.AddInteger(VT2.getRawBits());
10752 ID.AddInteger(VT3.getRawBits());
10753 ID.AddInteger(VT4.getRawBits());
10754
10755 void *IP = nullptr;
10756 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10757 if (!Result) {
10758 EVT *Array = Allocator.Allocate<EVT>(4);
10759 Array[0] = VT1;
10760 Array[1] = VT2;
10761 Array[2] = VT3;
10762 Array[3] = VT4;
10763 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10764 VTListMap.InsertNode(Result, IP);
10765 }
10766 return Result->getSDVTList();
10767}
10768
10770 unsigned NumVTs = VTs.size();
10772 ID.AddInteger(NumVTs);
10773 for (unsigned index = 0; index < NumVTs; index++) {
10774 ID.AddInteger(VTs[index].getRawBits());
10775 }
10776
10777 void *IP = nullptr;
10778 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10779 if (!Result) {
10780 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
10781 llvm::copy(VTs, Array);
10782 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10783 VTListMap.InsertNode(Result, IP);
10784 }
10785 return Result->getSDVTList();
10786}
10787
10788
10789/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10790/// specified operands. If the resultant node already exists in the DAG,
10791/// this does not modify the specified node, instead it returns the node that
10792/// already exists. If the resultant node does not exist in the DAG, the
10793/// input node is returned. As a degenerate case, if you specify the same
10794/// input operands as the node already has, the input node is returned.
10796 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10797
10798 // Check to see if there is no change.
10799 if (Op == N->getOperand(0)) return N;
10800
10801 // See if the modified node already exists.
10802 void *InsertPos = nullptr;
10803 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10804 return Existing;
10805
10806 // Nope it doesn't. Remove the node from its current place in the maps.
10807 if (InsertPos)
10808 if (!RemoveNodeFromCSEMaps(N))
10809 InsertPos = nullptr;
10810
10811 // Now we update the operands.
10812 N->OperandList[0].set(Op);
10813
10815 // If this gets put into a CSE map, add it.
10816 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10817 return N;
10818}
10819
10821 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10822
10823 // Check to see if there is no change.
10824 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
10825 return N; // No operands changed, just return the input node.
10826
10827 // See if the modified node already exists.
10828 void *InsertPos = nullptr;
10829 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10830 return Existing;
10831
10832 // Nope it doesn't. Remove the node from its current place in the maps.
10833 if (InsertPos)
10834 if (!RemoveNodeFromCSEMaps(N))
10835 InsertPos = nullptr;
10836
10837 // Now we update the operands.
10838 if (N->OperandList[0] != Op1)
10839 N->OperandList[0].set(Op1);
10840 if (N->OperandList[1] != Op2)
10841 N->OperandList[1].set(Op2);
10842
10844 // If this gets put into a CSE map, add it.
10845 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10846 return N;
10847}
10848
10851 SDValue Ops[] = { Op1, Op2, Op3 };
10852 return UpdateNodeOperands(N, Ops);
10853}
10854
10857 SDValue Op3, SDValue Op4) {
10858 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10859 return UpdateNodeOperands(N, Ops);
10860}
10861
10864 SDValue Op3, SDValue Op4, SDValue Op5) {
10865 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10866 return UpdateNodeOperands(N, Ops);
10867}
10868
10871 unsigned NumOps = Ops.size();
10872 assert(N->getNumOperands() == NumOps &&
10873 "Update with wrong number of operands");
10874
10875 // If no operands changed just return the input node.
10876 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
10877 return N;
10878
10879 // See if the modified node already exists.
10880 void *InsertPos = nullptr;
10881 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10882 return Existing;
10883
10884 // Nope it doesn't. Remove the node from its current place in the maps.
10885 if (InsertPos)
10886 if (!RemoveNodeFromCSEMaps(N))
10887 InsertPos = nullptr;
10888
10889 // Now we update the operands.
10890 for (unsigned i = 0; i != NumOps; ++i)
10891 if (N->OperandList[i] != Ops[i])
10892 N->OperandList[i].set(Ops[i]);
10893
10895 // If this gets put into a CSE map, add it.
10896 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10897 return N;
10898}
10899
10900/// DropOperands - Release the operands and set this node to have
10901/// zero operands.
10903 // Unlike the code in MorphNodeTo that does this, we don't need to
10904 // watch for dead nodes here.
10905 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10906 SDUse &Use = *I++;
10907 Use.set(SDValue());
10908 }
10909}
10910
10912 ArrayRef<MachineMemOperand *> NewMemRefs) {
10913 if (NewMemRefs.empty()) {
10914 N->clearMemRefs();
10915 return;
10916 }
10917
10918 // Check if we can avoid allocating by storing a single reference directly.
10919 if (NewMemRefs.size() == 1) {
10920 N->MemRefs = NewMemRefs[0];
10921 N->NumMemRefs = 1;
10922 return;
10923 }
10924
10925 MachineMemOperand **MemRefsBuffer =
10926 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
10927 llvm::copy(NewMemRefs, MemRefsBuffer);
10928 N->MemRefs = MemRefsBuffer;
10929 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10930}
10931
10932/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10933/// machine opcode.
10934///
10936 EVT VT) {
10937 SDVTList VTs = getVTList(VT);
10938 return SelectNodeTo(N, MachineOpc, VTs, {});
10939}
10940
10942 EVT VT, SDValue Op1) {
10943 SDVTList VTs = getVTList(VT);
10944 SDValue Ops[] = { Op1 };
10945 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10946}
10947
10949 EVT VT, SDValue Op1,
10950 SDValue Op2) {
10951 SDVTList VTs = getVTList(VT);
10952 SDValue Ops[] = { Op1, Op2 };
10953 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10954}
10955
10957 EVT VT, SDValue Op1,
10958 SDValue Op2, SDValue Op3) {
10959 SDVTList VTs = getVTList(VT);
10960 SDValue Ops[] = { Op1, Op2, Op3 };
10961 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10962}
10963
10965 EVT VT, ArrayRef<SDValue> Ops) {
10966 SDVTList VTs = getVTList(VT);
10967 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10968}
10969
10971 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10972 SDVTList VTs = getVTList(VT1, VT2);
10973 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10974}
10975
10977 EVT VT1, EVT VT2) {
10978 SDVTList VTs = getVTList(VT1, VT2);
10979 return SelectNodeTo(N, MachineOpc, VTs, {});
10980}
10981
10983 EVT VT1, EVT VT2, EVT VT3,
10984 ArrayRef<SDValue> Ops) {
10985 SDVTList VTs = getVTList(VT1, VT2, VT3);
10986 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10987}
10988
10990 EVT VT1, EVT VT2,
10991 SDValue Op1, SDValue Op2) {
10992 SDVTList VTs = getVTList(VT1, VT2);
10993 SDValue Ops[] = { Op1, Op2 };
10994 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10995}
10996
10998 SDVTList VTs,ArrayRef<SDValue> Ops) {
10999 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
11000 // Reset the NodeID to -1.
11001 New->setNodeId(-1);
11002 if (New != N) {
11003 ReplaceAllUsesWith(N, New);
11005 }
11006 return New;
11007}
11008
11009/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
11010/// the line number information on the merged node since it is not possible to
11011/// preserve the information that operation is associated with multiple lines.
11012/// This will make the debugger working better at -O0, were there is a higher
11013/// probability having other instructions associated with that line.
11014///
11015/// For IROrder, we keep the smaller of the two
11016SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
11017 DebugLoc NLoc = N->getDebugLoc();
11018 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
11019 N->setDebugLoc(DebugLoc());
11020 }
11021 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
11022 N->setIROrder(Order);
11023 return N;
11024}
11025
11026/// MorphNodeTo - This *mutates* the specified node to have the specified
11027/// return type, opcode, and operands.
11028///
11029/// Note that MorphNodeTo returns the resultant node. If there is already a
11030/// node of the specified opcode and operands, it returns that node instead of
11031/// the current one. Note that the SDLoc need not be the same.
11032///
11033/// Using MorphNodeTo is faster than creating a new node and swapping it in
11034/// with ReplaceAllUsesWith both because it often avoids allocating a new
11035/// node, and because it doesn't require CSE recalculation for any of
11036/// the node's users.
11037///
11038/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
11039/// As a consequence it isn't appropriate to use from within the DAG combiner or
11040/// the legalizer which maintain worklists that would need to be updated when
11041/// deleting things.
11043 SDVTList VTs, ArrayRef<SDValue> Ops) {
11044 // If an identical node already exists, use it.
11045 void *IP = nullptr;
11046 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
11048 AddNodeIDNode(ID, Opc, VTs, Ops);
11049 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
11050 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
11051 }
11052
11053 if (!RemoveNodeFromCSEMaps(N))
11054 IP = nullptr;
11055
11056 // Start the morphing.
11057 N->NodeType = Opc;
11058 N->ValueList = VTs.VTs;
11059 N->NumValues = VTs.NumVTs;
11060
11061 // Clear the operands list, updating used nodes to remove this from their
11062 // use list. Keep track of any operands that become dead as a result.
11063 SmallPtrSet<SDNode*, 16> DeadNodeSet;
11064 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
11065 SDUse &Use = *I++;
11066 SDNode *Used = Use.getNode();
11067 Use.set(SDValue());
11068 if (Used->use_empty())
11069 DeadNodeSet.insert(Used);
11070 }
11071
11072 // For MachineNode, initialize the memory references information.
11073 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
11074 MN->clearMemRefs();
11075
11076 // Swap for an appropriately sized array from the recycler.
11077 removeOperands(N);
11078 createOperands(N, Ops);
11079
11080 // Delete any nodes that are still dead after adding the uses for the
11081 // new operands.
11082 if (!DeadNodeSet.empty()) {
11083 SmallVector<SDNode *, 16> DeadNodes;
11084 for (SDNode *N : DeadNodeSet)
11085 if (N->use_empty())
11086 DeadNodes.push_back(N);
11087 RemoveDeadNodes(DeadNodes);
11088 }
11089
11090 if (IP)
11091 CSEMap.InsertNode(N, IP); // Memoize the new node.
11092 return N;
11093}
11094
11096 unsigned OrigOpc = Node->getOpcode();
11097 unsigned NewOpc;
11098 switch (OrigOpc) {
11099 default:
11100 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
11101#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11102 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
11103#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11104 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
11105#include "llvm/IR/ConstrainedOps.def"
11106 }
11107
11108 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
11109
11110 // We're taking this node out of the chain, so we need to re-link things.
11111 SDValue InputChain = Node->getOperand(0);
11112 SDValue OutputChain = SDValue(Node, 1);
11113 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
11114
11116 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
11117 Ops.push_back(Node->getOperand(i));
11118
11119 SDVTList VTs = getVTList(Node->getValueType(0));
11120 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
11121
11122 // MorphNodeTo can operate in two ways: if an existing node with the
11123 // specified operands exists, it can just return it. Otherwise, it
11124 // updates the node in place to have the requested operands.
11125 if (Res == Node) {
11126 // If we updated the node in place, reset the node ID. To the isel,
11127 // this should be just like a newly allocated machine node.
11128 Res->setNodeId(-1);
11129 } else {
11130 ReplaceAllUsesWith(Node, Res);
11131 RemoveDeadNode(Node);
11132 }
11133
11134 return Res;
11135}
11136
11137/// getMachineNode - These are used for target selectors to create a new node
11138/// with specified return type(s), MachineInstr opcode, and operands.
11139///
11140/// Note that getMachineNode returns the resultant node. If there is already a
11141/// node of the specified opcode and operands, it returns that node instead of
11142/// the current one.
11144 EVT VT) {
11145 SDVTList VTs = getVTList(VT);
11146 return getMachineNode(Opcode, dl, VTs, {});
11147}
11148
11150 EVT VT, SDValue Op1) {
11151 SDVTList VTs = getVTList(VT);
11152 SDValue Ops[] = { Op1 };
11153 return getMachineNode(Opcode, dl, VTs, Ops);
11154}
11155
11157 EVT VT, SDValue Op1, SDValue Op2) {
11158 SDVTList VTs = getVTList(VT);
11159 SDValue Ops[] = { Op1, Op2 };
11160 return getMachineNode(Opcode, dl, VTs, Ops);
11161}
11162
11164 EVT VT, SDValue Op1, SDValue Op2,
11165 SDValue Op3) {
11166 SDVTList VTs = getVTList(VT);
11167 SDValue Ops[] = { Op1, Op2, Op3 };
11168 return getMachineNode(Opcode, dl, VTs, Ops);
11169}
11170
11172 EVT VT, ArrayRef<SDValue> Ops) {
11173 SDVTList VTs = getVTList(VT);
11174 return getMachineNode(Opcode, dl, VTs, Ops);
11175}
11176
11178 EVT VT1, EVT VT2, SDValue Op1,
11179 SDValue Op2) {
11180 SDVTList VTs = getVTList(VT1, VT2);
11181 SDValue Ops[] = { Op1, Op2 };
11182 return getMachineNode(Opcode, dl, VTs, Ops);
11183}
11184
11186 EVT VT1, EVT VT2, SDValue Op1,
11187 SDValue Op2, SDValue Op3) {
11188 SDVTList VTs = getVTList(VT1, VT2);
11189 SDValue Ops[] = { Op1, Op2, Op3 };
11190 return getMachineNode(Opcode, dl, VTs, Ops);
11191}
11192
11194 EVT VT1, EVT VT2,
11195 ArrayRef<SDValue> Ops) {
11196 SDVTList VTs = getVTList(VT1, VT2);
11197 return getMachineNode(Opcode, dl, VTs, Ops);
11198}
11199
11201 EVT VT1, EVT VT2, EVT VT3,
11202 SDValue Op1, SDValue Op2) {
11203 SDVTList VTs = getVTList(VT1, VT2, VT3);
11204 SDValue Ops[] = { Op1, Op2 };
11205 return getMachineNode(Opcode, dl, VTs, Ops);
11206}
11207
11209 EVT VT1, EVT VT2, EVT VT3,
11210 SDValue Op1, SDValue Op2,
11211 SDValue Op3) {
11212 SDVTList VTs = getVTList(VT1, VT2, VT3);
11213 SDValue Ops[] = { Op1, Op2, Op3 };
11214 return getMachineNode(Opcode, dl, VTs, Ops);
11215}
11216
11218 EVT VT1, EVT VT2, EVT VT3,
11219 ArrayRef<SDValue> Ops) {
11220 SDVTList VTs = getVTList(VT1, VT2, VT3);
11221 return getMachineNode(Opcode, dl, VTs, Ops);
11222}
11223
11225 ArrayRef<EVT> ResultTys,
11226 ArrayRef<SDValue> Ops) {
11227 SDVTList VTs = getVTList(ResultTys);
11228 return getMachineNode(Opcode, dl, VTs, Ops);
11229}
11230
11232 SDVTList VTs,
11233 ArrayRef<SDValue> Ops) {
11234 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
11236 void *IP = nullptr;
11237
11238 if (DoCSE) {
11240 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
11241 IP = nullptr;
11242 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
11243 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
11244 }
11245 }
11246
11247 // Allocate a new MachineSDNode.
11248 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
11249 createOperands(N, Ops);
11250
11251 if (DoCSE)
11252 CSEMap.InsertNode(N, IP);
11253
11254 InsertNode(N);
11255 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
11256 return N;
11257}
11258
11259/// getTargetExtractSubreg - A convenience function for creating
11260/// TargetOpcode::EXTRACT_SUBREG nodes.
11262 SDValue Operand) {
11263 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
11264 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
11265 VT, Operand, SRIdxVal);
11266 return SDValue(Subreg, 0);
11267}
11268
11269/// getTargetInsertSubreg - A convenience function for creating
11270/// TargetOpcode::INSERT_SUBREG nodes.
11272 SDValue Operand, SDValue Subreg) {
11273 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
11274 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
11275 VT, Operand, Subreg, SRIdxVal);
11276 return SDValue(Result, 0);
11277}
11278
11279/// getNodeIfExists - Get the specified node if it's already available, or
11280/// else return NULL.
11282 ArrayRef<SDValue> Ops) {
11283 SDNodeFlags Flags;
11284 if (Inserter)
11285 Flags = Inserter->getFlags();
11286 return getNodeIfExists(Opcode, VTList, Ops, Flags);
11287}
11288
11291 const SDNodeFlags Flags) {
11292 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
11294 AddNodeIDNode(ID, Opcode, VTList, Ops);
11295 void *IP = nullptr;
11296 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
11297 E->intersectFlagsWith(Flags);
11298 return E;
11299 }
11300 }
11301 return nullptr;
11302}
11303
11304/// doesNodeExist - Check if a node exists without modifying its flags.
11305bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
11306 ArrayRef<SDValue> Ops) {
11307 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
11309 AddNodeIDNode(ID, Opcode, VTList, Ops);
11310 void *IP = nullptr;
11311 if (FindNodeOrInsertPos(ID, SDLoc(), IP))
11312 return true;
11313 }
11314 return false;
11315}
11316
11317/// getDbgValue - Creates a SDDbgValue node.
11318///
11319/// SDNode
11321 SDNode *N, unsigned R, bool IsIndirect,
11322 const DebugLoc &DL, unsigned O) {
11323 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11324 "Expected inlined-at fields to agree");
11325 return new (DbgInfo->getAlloc())
11326 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
11327 {}, IsIndirect, DL, O,
11328 /*IsVariadic=*/false);
11329}
11330
11331/// Constant
11333 DIExpression *Expr,
11334 const Value *C,
11335 const DebugLoc &DL, unsigned O) {
11336 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11337 "Expected inlined-at fields to agree");
11338 return new (DbgInfo->getAlloc())
11339 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
11340 /*IsIndirect=*/false, DL, O,
11341 /*IsVariadic=*/false);
11342}
11343
11344/// FrameIndex
11346 DIExpression *Expr, unsigned FI,
11347 bool IsIndirect,
11348 const DebugLoc &DL,
11349 unsigned O) {
11350 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11351 "Expected inlined-at fields to agree");
11352 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
11353}
11354
11355/// FrameIndex with dependencies
11357 DIExpression *Expr, unsigned FI,
11358 ArrayRef<SDNode *> Dependencies,
11359 bool IsIndirect,
11360 const DebugLoc &DL,
11361 unsigned O) {
11362 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11363 "Expected inlined-at fields to agree");
11364 return new (DbgInfo->getAlloc())
11365 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
11366 Dependencies, IsIndirect, DL, O,
11367 /*IsVariadic=*/false);
11368}
11369
11370/// VReg
11372 unsigned VReg, bool IsIndirect,
11373 const DebugLoc &DL, unsigned O) {
11374 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11375 "Expected inlined-at fields to agree");
11376 return new (DbgInfo->getAlloc())
11377 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
11378 {}, IsIndirect, DL, O,
11379 /*IsVariadic=*/false);
11380}
11381
11384 ArrayRef<SDNode *> Dependencies,
11385 bool IsIndirect, const DebugLoc &DL,
11386 unsigned O, bool IsVariadic) {
11387 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11388 "Expected inlined-at fields to agree");
11389 return new (DbgInfo->getAlloc())
11390 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
11391 DL, O, IsVariadic);
11392}
11393
11395 unsigned OffsetInBits, unsigned SizeInBits,
11396 bool InvalidateDbg) {
11397 SDNode *FromNode = From.getNode();
11398 SDNode *ToNode = To.getNode();
11399 assert(FromNode && ToNode && "Can't modify dbg values");
11400
11401 // PR35338
11402 // TODO: assert(From != To && "Redundant dbg value transfer");
11403 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
11404 if (From == To || FromNode == ToNode)
11405 return;
11406
11407 if (!FromNode->getHasDebugValue())
11408 return;
11409
11410 SDDbgOperand FromLocOp =
11411 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
11413
11415 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
11416 if (Dbg->isInvalidated())
11417 continue;
11418
11419 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
11420
11421 // Create a new location ops vector that is equal to the old vector, but
11422 // with each instance of FromLocOp replaced with ToLocOp.
11423 bool Changed = false;
11424 auto NewLocOps = Dbg->copyLocationOps();
11425 std::replace_if(
11426 NewLocOps.begin(), NewLocOps.end(),
11427 [&Changed, FromLocOp](const SDDbgOperand &Op) {
11428 bool Match = Op == FromLocOp;
11429 Changed |= Match;
11430 return Match;
11431 },
11432 ToLocOp);
11433 // Ignore this SDDbgValue if we didn't find a matching location.
11434 if (!Changed)
11435 continue;
11436
11437 DIVariable *Var = Dbg->getVariable();
11438 auto *Expr = Dbg->getExpression();
11439 // If a fragment is requested, update the expression.
11440 if (SizeInBits) {
11441 // When splitting a larger (e.g., sign-extended) value whose
11442 // lower bits are described with an SDDbgValue, do not attempt
11443 // to transfer the SDDbgValue to the upper bits.
11444 if (auto FI = Expr->getFragmentInfo())
11445 if (OffsetInBits + SizeInBits > FI->SizeInBits)
11446 continue;
11447 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
11448 SizeInBits);
11449 if (!Fragment)
11450 continue;
11451 Expr = *Fragment;
11452 }
11453
11454 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
11455 // Clone the SDDbgValue and move it to To.
11456 SDDbgValue *Clone = getDbgValueList(
11457 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
11458 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
11459 Dbg->isVariadic());
11460 ClonedDVs.push_back(Clone);
11461
11462 if (InvalidateDbg) {
11463 // Invalidate value and indicate the SDDbgValue should not be emitted.
11464 Dbg->setIsInvalidated();
11465 Dbg->setIsEmitted();
11466 }
11467 }
11468
11469 for (SDDbgValue *Dbg : ClonedDVs) {
11470 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
11471 "Transferred DbgValues should depend on the new SDNode");
11472 AddDbgValue(Dbg, false);
11473 }
11474}
11475
11477 if (!N.getHasDebugValue())
11478 return;
11479
11480 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
11481 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Node))
11482 return SDDbgOperand::fromFrameIdx(FISDN->getIndex());
11483 return SDDbgOperand::fromNode(Node, ResNo);
11484 };
11485
11487 for (auto *DV : GetDbgValues(&N)) {
11488 if (DV->isInvalidated())
11489 continue;
11490 switch (N.getOpcode()) {
11491 default:
11492 break;
11493 case ISD::ADD: {
11494 SDValue N0 = N.getOperand(0);
11495 SDValue N1 = N.getOperand(1);
11496 if (!isa<ConstantSDNode>(N0)) {
11497 bool RHSConstant = isa<ConstantSDNode>(N1);
11499 if (RHSConstant)
11500 Offset = N.getConstantOperandVal(1);
11501 // We are not allowed to turn indirect debug values variadic, so
11502 // don't salvage those.
11503 if (!RHSConstant && DV->isIndirect())
11504 continue;
11505
11506 // Rewrite an ADD constant node into a DIExpression. Since we are
11507 // performing arithmetic to compute the variable's *value* in the
11508 // DIExpression, we need to mark the expression with a
11509 // DW_OP_stack_value.
11510 auto *DIExpr = DV->getExpression();
11511 auto NewLocOps = DV->copyLocationOps();
11512 bool Changed = false;
11513 size_t OrigLocOpsSize = NewLocOps.size();
11514 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
11515 // We're not given a ResNo to compare against because the whole
11516 // node is going away. We know that any ISD::ADD only has one
11517 // result, so we can assume any node match is using the result.
11518 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
11519 NewLocOps[i].getSDNode() != &N)
11520 continue;
11521 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
11522 if (RHSConstant) {
11525 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
11526 } else {
11527 // Convert to a variadic expression (if not already).
11528 // convertToVariadicExpression() returns a const pointer, so we use
11529 // a temporary const variable here.
11530 const auto *TmpDIExpr =
11534 ExprOps.push_back(NewLocOps.size());
11535 ExprOps.push_back(dwarf::DW_OP_plus);
11538 NewLocOps.push_back(RHS);
11539 DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);
11540 }
11541 Changed = true;
11542 }
11543 (void)Changed;
11544 assert(Changed && "Salvage target doesn't use N");
11545
11546 bool IsVariadic =
11547 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
11548
11549 auto AdditionalDependencies = DV->getAdditionalDependencies();
11550 SDDbgValue *Clone = getDbgValueList(
11551 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
11552 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
11553 ClonedDVs.push_back(Clone);
11554 DV->setIsInvalidated();
11555 DV->setIsEmitted();
11556 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
11557 N0.getNode()->dumprFull(this);
11558 dbgs() << " into " << *DIExpr << '\n');
11559 }
11560 break;
11561 }
11562 case ISD::TRUNCATE: {
11563 SDValue N0 = N.getOperand(0);
11564 TypeSize FromSize = N0.getValueSizeInBits();
11565 TypeSize ToSize = N.getValueSizeInBits(0);
11566
11567 DIExpression *DbgExpression = DV->getExpression();
11568 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);
11569 auto NewLocOps = DV->copyLocationOps();
11570 bool Changed = false;
11571 for (size_t i = 0; i < NewLocOps.size(); ++i) {
11572 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
11573 NewLocOps[i].getSDNode() != &N)
11574 continue;
11575
11576 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
11577 DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);
11578 Changed = true;
11579 }
11580 assert(Changed && "Salvage target doesn't use N");
11581 (void)Changed;
11582
11583 SDDbgValue *Clone =
11584 getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,
11585 DV->getAdditionalDependencies(), DV->isIndirect(),
11586 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
11587
11588 ClonedDVs.push_back(Clone);
11589 DV->setIsInvalidated();
11590 DV->setIsEmitted();
11591 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
11592 dbgs() << " into " << *DbgExpression << '\n');
11593 break;
11594 }
11595 }
11596 }
11597
11598 for (SDDbgValue *Dbg : ClonedDVs) {
11599 assert((!Dbg->getSDNodes().empty() ||
11600 llvm::any_of(Dbg->getLocationOps(),
11601 [&](const SDDbgOperand &Op) {
11602 return Op.getKind() == SDDbgOperand::FRAMEIX;
11603 })) &&
11604 "Salvaged DbgValue should depend on a new SDNode");
11605 AddDbgValue(Dbg, false);
11606 }
11607}
11608
11609/// Creates a SDDbgLabel node.
11611 const DebugLoc &DL, unsigned O) {
11612 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
11613 "Expected inlined-at fields to agree");
11614 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
11615}
11616
11617namespace {
11618
11619/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
11620/// pointed to by a use iterator is deleted, increment the use iterator
11621/// so that it doesn't dangle.
11622///
11623class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
11626
11627 void NodeDeleted(SDNode *N, SDNode *E) override {
11628 // Increment the iterator as needed.
11629 while (UI != UE && N == UI->getUser())
11630 ++UI;
11631 }
11632
11633public:
11634 RAUWUpdateListener(SelectionDAG &d,
11637 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
11638};
11639
11640} // end anonymous namespace
11641
11642/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11643/// This can cause recursive merging of nodes in the DAG.
11644///
11645/// This version assumes From has a single result value.
11646///
11648 SDNode *From = FromN.getNode();
11649 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
11650 "Cannot replace with this method!");
11651 assert(From != To.getNode() && "Cannot replace uses of with self");
11652
11653 // Preserve Debug Values
11654 transferDbgValues(FromN, To);
11655 // Preserve extra info.
11656 copyExtraInfo(From, To.getNode());
11657
11658 // Iterate over all the existing uses of From. New uses will be added
11659 // to the beginning of the use list, which we avoid visiting.
11660 // This specifically avoids visiting uses of From that arise while the
11661 // replacement is happening, because any such uses would be the result
11662 // of CSE: If an existing node looks like From after one of its operands
11663 // is replaced by To, we don't want to replace of all its users with To
11664 // too. See PR3018 for more info.
11665 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11666 RAUWUpdateListener Listener(*this, UI, UE);
11667 while (UI != UE) {
11668 SDNode *User = UI->getUser();
11669
11670 // This node is about to morph, remove its old self from the CSE maps.
11671 RemoveNodeFromCSEMaps(User);
11672
11673 // A user can appear in a use list multiple times, and when this
11674 // happens the uses are usually next to each other in the list.
11675 // To help reduce the number of CSE recomputations, process all
11676 // the uses of this user that we can find this way.
11677 do {
11678 SDUse &Use = *UI;
11679 ++UI;
11680 Use.set(To);
11681 if (To->isDivergent() != From->isDivergent())
11683 } while (UI != UE && UI->getUser() == User);
11684 // Now that we have modified User, add it back to the CSE maps. If it
11685 // already exists there, recursively merge the results together.
11686 AddModifiedNodeToCSEMaps(User);
11687 }
11688
11689 // If we just RAUW'd the root, take note.
11690 if (FromN == getRoot())
11691 setRoot(To);
11692}
11693
11694/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11695/// This can cause recursive merging of nodes in the DAG.
11696///
11697/// This version assumes that for each value of From, there is a
11698/// corresponding value in To in the same position with the same type.
11699///
11701#ifndef NDEBUG
11702 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11703 assert((!From->hasAnyUseOfValue(i) ||
11704 From->getValueType(i) == To->getValueType(i)) &&
11705 "Cannot use this version of ReplaceAllUsesWith!");
11706#endif
11707
11708 // Handle the trivial case.
11709 if (From == To)
11710 return;
11711
11712 // Preserve Debug Info. Only do this if there's a use.
11713 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11714 if (From->hasAnyUseOfValue(i)) {
11715 assert((i < To->getNumValues()) && "Invalid To location");
11717 }
11718 // Preserve extra info.
11719 copyExtraInfo(From, To);
11720
11721 // Iterate over just the existing users of From. See the comments in
11722 // the ReplaceAllUsesWith above.
11723 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11724 RAUWUpdateListener Listener(*this, UI, UE);
11725 while (UI != UE) {
11726 SDNode *User = UI->getUser();
11727
11728 // This node is about to morph, remove its old self from the CSE maps.
11729 RemoveNodeFromCSEMaps(User);
11730
11731 // A user can appear in a use list multiple times, and when this
11732 // happens the uses are usually next to each other in the list.
11733 // To help reduce the number of CSE recomputations, process all
11734 // the uses of this user that we can find this way.
11735 do {
11736 SDUse &Use = *UI;
11737 ++UI;
11738 Use.setNode(To);
11739 if (To->isDivergent() != From->isDivergent())
11741 } while (UI != UE && UI->getUser() == User);
11742
11743 // Now that we have modified User, add it back to the CSE maps. If it
11744 // already exists there, recursively merge the results together.
11745 AddModifiedNodeToCSEMaps(User);
11746 }
11747
11748 // If we just RAUW'd the root, take note.
11749 if (From == getRoot().getNode())
11750 setRoot(SDValue(To, getRoot().getResNo()));
11751}
11752
11753/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11754/// This can cause recursive merging of nodes in the DAG.
11755///
11756/// This version can replace From with any result values. To must match the
11757/// number and types of values returned by From.
11759 if (From->getNumValues() == 1) // Handle the simple case efficiently.
11760 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
11761
11762 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11763 // Preserve Debug Info.
11764 transferDbgValues(SDValue(From, i), To[i]);
11765 // Preserve extra info.
11766 copyExtraInfo(From, To[i].getNode());
11767 }
11768
11769 // Iterate over just the existing users of From. See the comments in
11770 // the ReplaceAllUsesWith above.
11771 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11772 RAUWUpdateListener Listener(*this, UI, UE);
11773 while (UI != UE) {
11774 SDNode *User = UI->getUser();
11775
11776 // This node is about to morph, remove its old self from the CSE maps.
11777 RemoveNodeFromCSEMaps(User);
11778
11779 // A user can appear in a use list multiple times, and when this happens the
11780 // uses are usually next to each other in the list. To help reduce the
11781 // number of CSE and divergence recomputations, process all the uses of this
11782 // user that we can find this way.
11783 bool To_IsDivergent = false;
11784 do {
11785 SDUse &Use = *UI;
11786 const SDValue &ToOp = To[Use.getResNo()];
11787 ++UI;
11788 Use.set(ToOp);
11789 To_IsDivergent |= ToOp->isDivergent();
11790 } while (UI != UE && UI->getUser() == User);
11791
11792 if (To_IsDivergent != From->isDivergent())
11794
11795 // Now that we have modified User, add it back to the CSE maps. If it
11796 // already exists there, recursively merge the results together.
11797 AddModifiedNodeToCSEMaps(User);
11798 }
11799
11800 // If we just RAUW'd the root, take note.
11801 if (From == getRoot().getNode())
11802 setRoot(SDValue(To[getRoot().getResNo()]));
11803}
11804
11805/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11806/// uses of other values produced by From.getNode() alone. The Deleted
11807/// vector is handled the same way as for ReplaceAllUsesWith.
11809 // Handle the really simple, really trivial case efficiently.
11810 if (From == To) return;
11811
11812 // Handle the simple, trivial, case efficiently.
11813 if (From.getNode()->getNumValues() == 1) {
11815 return;
11816 }
11817
11818 // Preserve Debug Info.
11820 copyExtraInfo(From.getNode(), To.getNode());
11821
11822 // Iterate over just the existing users of From. See the comments in
11823 // the ReplaceAllUsesWith above.
11824 SDNode::use_iterator UI = From.getNode()->use_begin(),
11825 UE = From.getNode()->use_end();
11826 RAUWUpdateListener Listener(*this, UI, UE);
11827 while (UI != UE) {
11828 SDNode *User = UI->getUser();
11829 bool UserRemovedFromCSEMaps = false;
11830
11831 // A user can appear in a use list multiple times, and when this
11832 // happens the uses are usually next to each other in the list.
11833 // To help reduce the number of CSE recomputations, process all
11834 // the uses of this user that we can find this way.
11835 do {
11836 SDUse &Use = *UI;
11837
11838 // Skip uses of different values from the same node.
11839 if (Use.getResNo() != From.getResNo()) {
11840 ++UI;
11841 continue;
11842 }
11843
11844 // If this node hasn't been modified yet, it's still in the CSE maps,
11845 // so remove its old self from the CSE maps.
11846 if (!UserRemovedFromCSEMaps) {
11847 RemoveNodeFromCSEMaps(User);
11848 UserRemovedFromCSEMaps = true;
11849 }
11850
11851 ++UI;
11852 Use.set(To);
11853 if (To->isDivergent() != From->isDivergent())
11855 } while (UI != UE && UI->getUser() == User);
11856 // We are iterating over all uses of the From node, so if a use
11857 // doesn't use the specific value, no changes are made.
11858 if (!UserRemovedFromCSEMaps)
11859 continue;
11860
11861 // Now that we have modified User, add it back to the CSE maps. If it
11862 // already exists there, recursively merge the results together.
11863 AddModifiedNodeToCSEMaps(User);
11864 }
11865
11866 // If we just RAUW'd the root, take note.
11867 if (From == getRoot())
11868 setRoot(To);
11869}
11870
11871namespace {
11872
11873/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11874/// to record information about a use.
11875struct UseMemo {
11876 SDNode *User;
11877 unsigned Index;
11878 SDUse *Use;
11879};
11880
11881/// operator< - Sort Memos by User.
11882bool operator<(const UseMemo &L, const UseMemo &R) {
11883 return (intptr_t)L.User < (intptr_t)R.User;
11884}
11885
11886/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11887/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11888/// the node already has been taken care of recursively.
11889class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11891
11892 void NodeDeleted(SDNode *N, SDNode *E) override {
11893 for (UseMemo &Memo : Uses)
11894 if (Memo.User == N)
11895 Memo.User = nullptr;
11896 }
11897
11898public:
11899 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
11900 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11901};
11902
11903} // end anonymous namespace
11904
11905/// Return true if a glue output should propagate divergence information.
11907 switch (Node->getOpcode()) {
11908 case ISD::CopyFromReg:
11909 case ISD::CopyToReg:
11910 return false;
11911 default:
11912 return true;
11913 }
11914
11915 llvm_unreachable("covered opcode switch");
11916}
11917
11919 if (TLI->isSDNodeAlwaysUniform(N)) {
11920 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11921 "Conflicting divergence information!");
11922 return false;
11923 }
11924 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11925 return true;
11926 for (const auto &Op : N->ops()) {
11927 EVT VT = Op.getValueType();
11928
11929 // Skip Chain. It does not carry divergence.
11930 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
11931 (VT != MVT::Glue || gluePropagatesDivergence(Op.getNode())))
11932 return true;
11933 }
11934 return false;
11935}
11936
11938 SmallVector<SDNode *, 16> Worklist(1, N);
11939 do {
11940 N = Worklist.pop_back_val();
11941 bool IsDivergent = calculateDivergence(N);
11942 if (N->SDNodeBits.IsDivergent != IsDivergent) {
11943 N->SDNodeBits.IsDivergent = IsDivergent;
11944 llvm::append_range(Worklist, N->users());
11945 }
11946 } while (!Worklist.empty());
11947}
11948
11949void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11951 Order.reserve(AllNodes.size());
11952 for (auto &N : allnodes()) {
11953 unsigned NOps = N.getNumOperands();
11954 Degree[&N] = NOps;
11955 if (0 == NOps)
11956 Order.push_back(&N);
11957 }
11958 for (size_t I = 0; I != Order.size(); ++I) {
11959 SDNode *N = Order[I];
11960 for (auto *U : N->users()) {
11961 unsigned &UnsortedOps = Degree[U];
11962 if (0 == --UnsortedOps)
11963 Order.push_back(U);
11964 }
11965 }
11966}
11967
11968#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
11969void SelectionDAG::VerifyDAGDivergence() {
11970 std::vector<SDNode *> TopoOrder;
11971 CreateTopologicalOrder(TopoOrder);
11972 for (auto *N : TopoOrder) {
11973 assert(calculateDivergence(N) == N->isDivergent() &&
11974 "Divergence bit inconsistency detected");
11975 }
11976}
11977#endif
11978
11979/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11980/// uses of other values produced by From.getNode() alone. The same value
11981/// may appear in both the From and To list. The Deleted vector is
11982/// handled the same way as for ReplaceAllUsesWith.
11984 const SDValue *To,
11985 unsigned Num){
11986 // Handle the simple, trivial case efficiently.
11987 if (Num == 1)
11988 return ReplaceAllUsesOfValueWith(*From, *To);
11989
11990 transferDbgValues(*From, *To);
11991 copyExtraInfo(From->getNode(), To->getNode());
11992
11993 // Read up all the uses and make records of them. This helps
11994 // processing new uses that are introduced during the
11995 // replacement process.
11997 for (unsigned i = 0; i != Num; ++i) {
11998 unsigned FromResNo = From[i].getResNo();
11999 SDNode *FromNode = From[i].getNode();
12000 for (SDUse &Use : FromNode->uses()) {
12001 if (Use.getResNo() == FromResNo) {
12002 UseMemo Memo = {Use.getUser(), i, &Use};
12003 Uses.push_back(Memo);
12004 }
12005 }
12006 }
12007
12008 // Sort the uses, so that all the uses from a given User are together.
12010 RAUOVWUpdateListener Listener(*this, Uses);
12011
12012 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
12013 UseIndex != UseIndexEnd; ) {
12014 // We know that this user uses some value of From. If it is the right
12015 // value, update it.
12016 SDNode *User = Uses[UseIndex].User;
12017 // If the node has been deleted by recursive CSE updates when updating
12018 // another node, then just skip this entry.
12019 if (User == nullptr) {
12020 ++UseIndex;
12021 continue;
12022 }
12023
12024 // This node is about to morph, remove its old self from the CSE maps.
12025 RemoveNodeFromCSEMaps(User);
12026
12027 // The Uses array is sorted, so all the uses for a given User
12028 // are next to each other in the list.
12029 // To help reduce the number of CSE recomputations, process all
12030 // the uses of this user that we can find this way.
12031 do {
12032 unsigned i = Uses[UseIndex].Index;
12033 SDUse &Use = *Uses[UseIndex].Use;
12034 ++UseIndex;
12035
12036 Use.set(To[i]);
12037 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
12038
12039 // Now that we have modified User, add it back to the CSE maps. If it
12040 // already exists there, recursively merge the results together.
12041 AddModifiedNodeToCSEMaps(User);
12042 }
12043}
12044
12045/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
12046/// based on their topological order. It returns the maximum id and a vector
12047/// of the SDNodes* in assigned order by reference.
12049 unsigned DAGSize = 0;
12050
12051 // SortedPos tracks the progress of the algorithm. Nodes before it are
12052 // sorted, nodes after it are unsorted. When the algorithm completes
12053 // it is at the end of the list.
12054 allnodes_iterator SortedPos = allnodes_begin();
12055
12056 // Visit all the nodes. Move nodes with no operands to the front of
12057 // the list immediately. Annotate nodes that do have operands with their
12058 // operand count. Before we do this, the Node Id fields of the nodes
12059 // may contain arbitrary values. After, the Node Id fields for nodes
12060 // before SortedPos will contain the topological sort index, and the
12061 // Node Id fields for nodes At SortedPos and after will contain the
12062 // count of outstanding operands.
12064 checkForCycles(&N, this);
12065 unsigned Degree = N.getNumOperands();
12066 if (Degree == 0) {
12067 // A node with no uses, add it to the result array immediately.
12068 N.setNodeId(DAGSize++);
12069 allnodes_iterator Q(&N);
12070 if (Q != SortedPos)
12071 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
12072 assert(SortedPos != AllNodes.end() && "Overran node list");
12073 ++SortedPos;
12074 } else {
12075 // Temporarily use the Node Id as scratch space for the degree count.
12076 N.setNodeId(Degree);
12077 }
12078 }
12079
12080 // Visit all the nodes. As we iterate, move nodes into sorted order,
12081 // such that by the time the end is reached all nodes will be sorted.
12082 for (SDNode &Node : allnodes()) {
12083 SDNode *N = &Node;
12084 checkForCycles(N, this);
12085 // N is in sorted position, so all its uses have one less operand
12086 // that needs to be sorted.
12087 for (SDNode *P : N->users()) {
12088 unsigned Degree = P->getNodeId();
12089 assert(Degree != 0 && "Invalid node degree");
12090 --Degree;
12091 if (Degree == 0) {
12092 // All of P's operands are sorted, so P may sorted now.
12093 P->setNodeId(DAGSize++);
12094 if (P->getIterator() != SortedPos)
12095 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
12096 assert(SortedPos != AllNodes.end() && "Overran node list");
12097 ++SortedPos;
12098 } else {
12099 // Update P's outstanding operand count.
12100 P->setNodeId(Degree);
12101 }
12102 }
12103 if (Node.getIterator() == SortedPos) {
12104#ifndef NDEBUG
12106 SDNode *S = &*++I;
12107 dbgs() << "Overran sorted position:\n";
12108 S->dumprFull(this); dbgs() << "\n";
12109 dbgs() << "Checking if this is due to cycles\n";
12110 checkForCycles(this, true);
12111#endif
12112 llvm_unreachable(nullptr);
12113 }
12114 }
12115
12116 assert(SortedPos == AllNodes.end() &&
12117 "Topological sort incomplete!");
12118 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
12119 "First node in topological sort is not the entry token!");
12120 assert(AllNodes.front().getNodeId() == 0 &&
12121 "First node in topological sort has non-zero id!");
12122 assert(AllNodes.front().getNumOperands() == 0 &&
12123 "First node in topological sort has operands!");
12124 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
12125 "Last node in topologic sort has unexpected id!");
12126 assert(AllNodes.back().use_empty() &&
12127 "Last node in topologic sort has users!");
12128 assert(DAGSize == allnodes_size() && "Node count mismatch!");
12129 return DAGSize;
12130}
12131
12132/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
12133/// value is produced by SD.
12134void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
12135 for (SDNode *SD : DB->getSDNodes()) {
12136 if (!SD)
12137 continue;
12138 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
12139 SD->setHasDebugValue(true);
12140 }
12141 DbgInfo->add(DB, isParameter);
12142}
12143
12144void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
12145
12147 SDValue NewMemOpChain) {
12148 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
12149 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
12150 // The new memory operation must have the same position as the old load in
12151 // terms of memory dependency. Create a TokenFactor for the old load and new
12152 // memory operation and update uses of the old load's output chain to use that
12153 // TokenFactor.
12154 if (OldChain == NewMemOpChain || OldChain.use_empty())
12155 return NewMemOpChain;
12156
12157 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
12158 OldChain, NewMemOpChain);
12159 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
12160 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
12161 return TokenFactor;
12162}
12163
12165 SDValue NewMemOp) {
12166 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
12167 SDValue OldChain = SDValue(OldLoad, 1);
12168 SDValue NewMemOpChain = NewMemOp.getValue(1);
12169 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
12170}
12171
12173 Function **OutFunction) {
12174 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
12175
12176 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
12177 auto *Module = MF->getFunction().getParent();
12178 auto *Function = Module->getFunction(Symbol);
12179
12180 if (OutFunction != nullptr)
12181 *OutFunction = Function;
12182
12183 if (Function != nullptr) {
12184 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
12185 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
12186 }
12187
12188 std::string ErrorStr;
12189 raw_string_ostream ErrorFormatter(ErrorStr);
12190 ErrorFormatter << "Undefined external symbol ";
12191 ErrorFormatter << '"' << Symbol << '"';
12192 report_fatal_error(Twine(ErrorStr));
12193}
12194
12195//===----------------------------------------------------------------------===//
12196// SDNode Class
12197//===----------------------------------------------------------------------===//
12198
12200 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
12201 return Const != nullptr && Const->isZero();
12202}
12203
12205 return V.isUndef() || isNullConstant(V);
12206}
12207
12209 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
12210 return Const != nullptr && Const->isZero() && !Const->isNegative();
12211}
12212
12214 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
12215 return Const != nullptr && Const->isAllOnes();
12216}
12217
12219 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
12220 return Const != nullptr && Const->isOne();
12221}
12222
12224 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
12225 return Const != nullptr && Const->isMinSignedValue();
12226}
12227
12228bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
12229 unsigned OperandNo) {
12230 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
12231 // TODO: Target-specific opcodes could be added.
12232 if (auto *ConstV = isConstOrConstSplat(V, /*AllowUndefs*/ false,
12233 /*AllowTruncation*/ true)) {
12234 APInt Const = ConstV->getAPIntValue().trunc(V.getScalarValueSizeInBits());
12235 switch (Opcode) {
12236 case ISD::ADD:
12237 case ISD::OR:
12238 case ISD::XOR:
12239 case ISD::UMAX:
12240 return Const.isZero();
12241 case ISD::MUL:
12242 return Const.isOne();
12243 case ISD::AND:
12244 case ISD::UMIN:
12245 return Const.isAllOnes();
12246 case ISD::SMAX:
12247 return Const.isMinSignedValue();
12248 case ISD::SMIN:
12249 return Const.isMaxSignedValue();
12250 case ISD::SUB:
12251 case ISD::SHL:
12252 case ISD::SRA:
12253 case ISD::SRL:
12254 return OperandNo == 1 && Const.isZero();
12255 case ISD::UDIV:
12256 case ISD::SDIV:
12257 return OperandNo == 1 && Const.isOne();
12258 }
12259 } else if (auto *ConstFP = isConstOrConstSplatFP(V)) {
12260 switch (Opcode) {
12261 case ISD::FADD:
12262 return ConstFP->isZero() &&
12263 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
12264 case ISD::FSUB:
12265 return OperandNo == 1 && ConstFP->isZero() &&
12266 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
12267 case ISD::FMUL:
12268 return ConstFP->isExactlyValue(1.0);
12269 case ISD::FDIV:
12270 return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
12271 case ISD::FMINNUM:
12272 case ISD::FMAXNUM: {
12273 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12274 EVT VT = V.getValueType();
12275 const fltSemantics &Semantics = VT.getFltSemantics();
12276 APFloat NeutralAF = !Flags.hasNoNaNs()
12277 ? APFloat::getQNaN(Semantics)
12278 : !Flags.hasNoInfs()
12279 ? APFloat::getInf(Semantics)
12280 : APFloat::getLargest(Semantics);
12281 if (Opcode == ISD::FMAXNUM)
12282 NeutralAF.changeSign();
12283
12284 return ConstFP->isExactlyValue(NeutralAF);
12285 }
12286 }
12287 }
12288 return false;
12289}
12290
12292 while (V.getOpcode() == ISD::BITCAST)
12293 V = V.getOperand(0);
12294 return V;
12295}
12296
12298 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
12299 V = V.getOperand(0);
12300 return V;
12301}
12302
12304 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
12305 V = V.getOperand(0);
12306 return V;
12307}
12308
12310 while (V.getOpcode() == ISD::TRUNCATE)
12311 V = V.getOperand(0);
12312 return V;
12313}
12314
12315bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
12316 if (V.getOpcode() != ISD::XOR)
12317 return false;
12318 V = peekThroughBitcasts(V.getOperand(1));
12319 unsigned NumBits = V.getScalarValueSizeInBits();
12320 ConstantSDNode *C =
12321 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
12322 return C && (C->getAPIntValue().countr_one() >= NumBits);
12323}
12324
12326 bool AllowTruncation) {
12327 EVT VT = N.getValueType();
12328 APInt DemandedElts = VT.isFixedLengthVector()
12330 : APInt(1, 1);
12331 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
12332}
12333
12335 bool AllowUndefs,
12336 bool AllowTruncation) {
12337 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
12338 return CN;
12339
12340 // SplatVectors can truncate their operands. Ignore that case here unless
12341 // AllowTruncation is set.
12342 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
12343 EVT VecEltVT = N->getValueType(0).getVectorElementType();
12344 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
12345 EVT CVT = CN->getValueType(0);
12346 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
12347 if (AllowTruncation || CVT == VecEltVT)
12348 return CN;
12349 }
12350 }
12351
12352 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
12353 BitVector UndefElements;
12354 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
12355
12356 // BuildVectors can truncate their operands. Ignore that case here unless
12357 // AllowTruncation is set.
12358 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
12359 if (CN && (UndefElements.none() || AllowUndefs)) {
12360 EVT CVT = CN->getValueType(0);
12361 EVT NSVT = N.getValueType().getScalarType();
12362 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
12363 if (AllowTruncation || (CVT == NSVT))
12364 return CN;
12365 }
12366 }
12367
12368 return nullptr;
12369}
12370
12372 EVT VT = N.getValueType();
12373 APInt DemandedElts = VT.isFixedLengthVector()
12375 : APInt(1, 1);
12376 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
12377}
12378
12380 const APInt &DemandedElts,
12381 bool AllowUndefs) {
12382 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
12383 return CN;
12384
12385 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
12386 BitVector UndefElements;
12387 ConstantFPSDNode *CN =
12388 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
12389 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
12390 if (CN && (UndefElements.none() || AllowUndefs))
12391 return CN;
12392 }
12393
12394 if (N.getOpcode() == ISD::SPLAT_VECTOR)
12395 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
12396 return CN;
12397
12398 return nullptr;
12399}
12400
12401bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
12402 // TODO: may want to use peekThroughBitcast() here.
12403 ConstantSDNode *C =
12404 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
12405 return C && C->isZero();
12406}
12407
12408bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
12409 ConstantSDNode *C =
12410 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
12411 return C && C->isOne();
12412}
12413
12414bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
12416 unsigned BitWidth = N.getScalarValueSizeInBits();
12417 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
12418 return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
12419}
12420
12422 DropOperands();
12423}
12424
12425MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
12426 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
12427 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
12428 MemSDNodeBits.IsVolatile = MMO->isVolatile();
12429 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
12430 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
12431 MemSDNodeBits.IsInvariant = MMO->isInvariant();
12432
12433 // We check here that the size of the memory operand fits within the size of
12434 // the MMO. This is because the MMO might indicate only a possible address
12435 // range instead of specifying the affected memory addresses precisely.
12436 assert(
12437 (!MMO->getType().isValid() ||
12439 "Size mismatch!");
12440}
12441
12442/// Profile - Gather unique data for the node.
12443///
12445 AddNodeIDNode(ID, this);
12446}
12447
12448namespace {
12449
12450 struct EVTArray {
12451 std::vector<EVT> VTs;
12452
12453 EVTArray() {
12454 VTs.reserve(MVT::VALUETYPE_SIZE);
12455 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
12456 VTs.push_back(MVT((MVT::SimpleValueType)i));
12457 }
12458 };
12459
12460} // end anonymous namespace
12461
12462/// getValueTypeList - Return a pointer to the specified value type.
12463///
12464const EVT *SDNode::getValueTypeList(MVT VT) {
12465 static EVTArray SimpleVTArray;
12466
12467 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
12468 return &SimpleVTArray.VTs[VT.SimpleTy];
12469}
12470
12471/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
12472/// indicated value. This method ignores uses of other values defined by this
12473/// operation.
12474bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
12475 assert(Value < getNumValues() && "Bad value!");
12476
12477 // TODO: Only iterate over uses of a given value of the node
12478 for (SDUse &U : uses()) {
12479 if (U.getResNo() == Value) {
12480 if (NUses == 0)
12481 return false;
12482 --NUses;
12483 }
12484 }
12485
12486 // Found exactly the right number of uses?
12487 return NUses == 0;
12488}
12489
12490/// hasAnyUseOfValue - Return true if there are any use of the indicated
12491/// value. This method ignores uses of other values defined by this operation.
12492bool SDNode::hasAnyUseOfValue(unsigned Value) const {
12493 assert(Value < getNumValues() && "Bad value!");
12494
12495 for (SDUse &U : uses())
12496 if (U.getResNo() == Value)
12497 return true;
12498
12499 return false;
12500}
12501
12502/// isOnlyUserOf - Return true if this node is the only use of N.
12503bool SDNode::isOnlyUserOf(const SDNode *N) const {
12504 bool Seen = false;
12505 for (const SDNode *User : N->users()) {
12506 if (User == this)
12507 Seen = true;
12508 else
12509 return false;
12510 }
12511
12512 return Seen;
12513}
12514
12515/// Return true if the only users of N are contained in Nodes.
12517 bool Seen = false;
12518 for (const SDNode *User : N->users()) {
12519 if (llvm::is_contained(Nodes, User))
12520 Seen = true;
12521 else
12522 return false;
12523 }
12524
12525 return Seen;
12526}
12527
12528/// isOperand - Return true if this node is an operand of N.
12529bool SDValue::isOperandOf(const SDNode *N) const {
12530 return is_contained(N->op_values(), *this);
12531}
12532
12533bool SDNode::isOperandOf(const SDNode *N) const {
12534 return any_of(N->op_values(),
12535 [this](SDValue Op) { return this == Op.getNode(); });
12536}
12537
12538/// reachesChainWithoutSideEffects - Return true if this operand (which must
12539/// be a chain) reaches the specified operand without crossing any
12540/// side-effecting instructions on any chain path. In practice, this looks
12541/// through token factors and non-volatile loads. In order to remain efficient,
12542/// this only looks a couple of nodes in, it does not do an exhaustive search.
12543///
12544/// Note that we only need to examine chains when we're searching for
12545/// side-effects; SelectionDAG requires that all side-effects are represented
12546/// by chains, even if another operand would force a specific ordering. This
12547/// constraint is necessary to allow transformations like splitting loads.
12549 unsigned Depth) const {
12550 if (*this == Dest) return true;
12551
12552 // Don't search too deeply, we just want to be able to see through
12553 // TokenFactor's etc.
12554 if (Depth == 0) return false;
12555
12556 // If this is a token factor, all inputs to the TF happen in parallel.
12557 if (getOpcode() == ISD::TokenFactor) {
12558 // First, try a shallow search.
12559 if (is_contained((*this)->ops(), Dest)) {
12560 // We found the chain we want as an operand of this TokenFactor.
12561 // Essentially, we reach the chain without side-effects if we could
12562 // serialize the TokenFactor into a simple chain of operations with
12563 // Dest as the last operation. This is automatically true if the
12564 // chain has one use: there are no other ordering constraints.
12565 // If the chain has more than one use, we give up: some other
12566 // use of Dest might force a side-effect between Dest and the current
12567 // node.
12568 if (Dest.hasOneUse())
12569 return true;
12570 }
12571 // Next, try a deep search: check whether every operand of the TokenFactor
12572 // reaches Dest.
12573 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
12574 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
12575 });
12576 }
12577
12578 // Loads don't have side effects, look through them.
12579 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
12580 if (Ld->isUnordered())
12581 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
12582 }
12583 return false;
12584}
12585
12586bool SDNode::hasPredecessor(const SDNode *N) const {
12589 Worklist.push_back(this);
12590 return hasPredecessorHelper(N, Visited, Worklist);
12591}
12592
12594 this->Flags &= Flags;
12595}
12596
12597SDValue
12599 ArrayRef<ISD::NodeType> CandidateBinOps,
12600 bool AllowPartials) {
12601 // The pattern must end in an extract from index 0.
12602 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
12603 !isNullConstant(Extract->getOperand(1)))
12604 return SDValue();
12605
12606 // Match against one of the candidate binary ops.
12607 SDValue Op = Extract->getOperand(0);
12608 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
12609 return Op.getOpcode() == unsigned(BinOp);
12610 }))
12611 return SDValue();
12612
12613 // Floating-point reductions may require relaxed constraints on the final step
12614 // of the reduction because they may reorder intermediate operations.
12615 unsigned CandidateBinOp = Op.getOpcode();
12616 if (Op.getValueType().isFloatingPoint()) {
12617 SDNodeFlags Flags = Op->getFlags();
12618 switch (CandidateBinOp) {
12619 case ISD::FADD:
12620 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
12621 return SDValue();
12622 break;
12623 default:
12624 llvm_unreachable("Unhandled FP opcode for binop reduction");
12625 }
12626 }
12627
12628 // Matching failed - attempt to see if we did enough stages that a partial
12629 // reduction from a subvector is possible.
12630 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
12631 if (!AllowPartials || !Op)
12632 return SDValue();
12633 EVT OpVT = Op.getValueType();
12634 EVT OpSVT = OpVT.getScalarType();
12635 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
12636 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
12637 return SDValue();
12638 BinOp = (ISD::NodeType)CandidateBinOp;
12639 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
12641 };
12642
12643 // At each stage, we're looking for something that looks like:
12644 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
12645 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
12646 // i32 undef, i32 undef, i32 undef, i32 undef>
12647 // %a = binop <8 x i32> %op, %s
12648 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
12649 // we expect something like:
12650 // <4,5,6,7,u,u,u,u>
12651 // <2,3,u,u,u,u,u,u>
12652 // <1,u,u,u,u,u,u,u>
12653 // While a partial reduction match would be:
12654 // <2,3,u,u,u,u,u,u>
12655 // <1,u,u,u,u,u,u,u>
12656 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
12657 SDValue PrevOp;
12658 for (unsigned i = 0; i < Stages; ++i) {
12659 unsigned MaskEnd = (1 << i);
12660
12661 if (Op.getOpcode() != CandidateBinOp)
12662 return PartialReduction(PrevOp, MaskEnd);
12663
12664 SDValue Op0 = Op.getOperand(0);
12665 SDValue Op1 = Op.getOperand(1);
12666
12667 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
12668 if (Shuffle) {
12669 Op = Op1;
12670 } else {
12671 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
12672 Op = Op0;
12673 }
12674
12675 // The first operand of the shuffle should be the same as the other operand
12676 // of the binop.
12677 if (!Shuffle || Shuffle->getOperand(0) != Op)
12678 return PartialReduction(PrevOp, MaskEnd);
12679
12680 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
12681 for (int Index = 0; Index < (int)MaskEnd; ++Index)
12682 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
12683 return PartialReduction(PrevOp, MaskEnd);
12684
12685 PrevOp = Op;
12686 }
12687
12688 // Handle subvector reductions, which tend to appear after the shuffle
12689 // reduction stages.
12690 while (Op.getOpcode() == CandidateBinOp) {
12691 unsigned NumElts = Op.getValueType().getVectorNumElements();
12692 SDValue Op0 = Op.getOperand(0);
12693 SDValue Op1 = Op.getOperand(1);
12694 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12696 Op0.getOperand(0) != Op1.getOperand(0))
12697 break;
12698 SDValue Src = Op0.getOperand(0);
12699 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12700 if (NumSrcElts != (2 * NumElts))
12701 break;
12702 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
12703 Op1.getConstantOperandAPInt(1) == NumElts) &&
12704 !(Op1.getConstantOperandAPInt(1) == 0 &&
12705 Op0.getConstantOperandAPInt(1) == NumElts))
12706 break;
12707 Op = Src;
12708 }
12709
12710 BinOp = (ISD::NodeType)CandidateBinOp;
12711 return Op;
12712}
12713
12715 EVT VT = N->getValueType(0);
12716 EVT EltVT = VT.getVectorElementType();
12717 unsigned NE = VT.getVectorNumElements();
12718
12719 SDLoc dl(N);
12720
12721 // If ResNE is 0, fully unroll the vector op.
12722 if (ResNE == 0)
12723 ResNE = NE;
12724 else if (NE > ResNE)
12725 NE = ResNE;
12726
12727 if (N->getNumValues() == 2) {
12728 SmallVector<SDValue, 8> Scalars0, Scalars1;
12729 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12730 EVT VT1 = N->getValueType(1);
12731 EVT EltVT1 = VT1.getVectorElementType();
12732
12733 unsigned i;
12734 for (i = 0; i != NE; ++i) {
12735 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12736 SDValue Operand = N->getOperand(j);
12737 EVT OperandVT = Operand.getValueType();
12738
12739 // A vector operand; extract a single element.
12740 EVT OperandEltVT = OperandVT.getVectorElementType();
12741 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12742 Operand, getVectorIdxConstant(i, dl));
12743 }
12744
12745 SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);
12746 Scalars0.push_back(EltOp);
12747 Scalars1.push_back(EltOp.getValue(1));
12748 }
12749
12750 for (; i < ResNE; ++i) {
12751 Scalars0.push_back(getUNDEF(EltVT));
12752 Scalars1.push_back(getUNDEF(EltVT1));
12753 }
12754
12755 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
12756 EVT VecVT1 = EVT::getVectorVT(*getContext(), EltVT1, ResNE);
12757 SDValue Vec0 = getBuildVector(VecVT, dl, Scalars0);
12758 SDValue Vec1 = getBuildVector(VecVT1, dl, Scalars1);
12759 return getMergeValues({Vec0, Vec1}, dl);
12760 }
12761
12762 assert(N->getNumValues() == 1 &&
12763 "Can't unroll a vector with multiple results!");
12764
12766 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12767
12768 unsigned i;
12769 for (i= 0; i != NE; ++i) {
12770 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12771 SDValue Operand = N->getOperand(j);
12772 EVT OperandVT = Operand.getValueType();
12773 if (OperandVT.isVector()) {
12774 // A vector operand; extract a single element.
12775 EVT OperandEltVT = OperandVT.getVectorElementType();
12776 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12777 Operand, getVectorIdxConstant(i, dl));
12778 } else {
12779 // A scalar operand; just use it as is.
12780 Operands[j] = Operand;
12781 }
12782 }
12783
12784 switch (N->getOpcode()) {
12785 default: {
12786 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
12787 N->getFlags()));
12788 break;
12789 }
12790 case ISD::VSELECT:
12791 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
12792 break;
12793 case ISD::SHL:
12794 case ISD::SRA:
12795 case ISD::SRL:
12796 case ISD::ROTL:
12797 case ISD::ROTR:
12798 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
12800 Operands[1])));
12801 break;
12803 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
12804 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
12805 Operands[0],
12806 getValueType(ExtVT)));
12807 break;
12808 }
12809 case ISD::ADDRSPACECAST: {
12810 const auto *ASC = cast<AddrSpaceCastSDNode>(N);
12811 Scalars.push_back(getAddrSpaceCast(dl, EltVT, Operands[0],
12812 ASC->getSrcAddressSpace(),
12813 ASC->getDestAddressSpace()));
12814 break;
12815 }
12816 }
12817 }
12818
12819 for (; i < ResNE; ++i)
12820 Scalars.push_back(getUNDEF(EltVT));
12821
12822 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
12823 return getBuildVector(VecVT, dl, Scalars);
12824}
12825
12826std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12827 SDNode *N, unsigned ResNE) {
12828 unsigned Opcode = N->getOpcode();
12829 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12830 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12831 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12832 "Expected an overflow opcode");
12833
12834 EVT ResVT = N->getValueType(0);
12835 EVT OvVT = N->getValueType(1);
12836 EVT ResEltVT = ResVT.getVectorElementType();
12837 EVT OvEltVT = OvVT.getVectorElementType();
12838 SDLoc dl(N);
12839
12840 // If ResNE is 0, fully unroll the vector op.
12841 unsigned NE = ResVT.getVectorNumElements();
12842 if (ResNE == 0)
12843 ResNE = NE;
12844 else if (NE > ResNE)
12845 NE = ResNE;
12846
12847 SmallVector<SDValue, 8> LHSScalars;
12848 SmallVector<SDValue, 8> RHSScalars;
12849 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
12850 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
12851
12852 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
12853 SDVTList VTs = getVTList(ResEltVT, SVT);
12854 SmallVector<SDValue, 8> ResScalars;
12855 SmallVector<SDValue, 8> OvScalars;
12856 for (unsigned i = 0; i < NE; ++i) {
12857 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12858 SDValue Ov =
12859 getSelect(dl, OvEltVT, Res.getValue(1),
12860 getBoolConstant(true, dl, OvEltVT, ResVT),
12861 getConstant(0, dl, OvEltVT));
12862
12863 ResScalars.push_back(Res);
12864 OvScalars.push_back(Ov);
12865 }
12866
12867 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
12868 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
12869
12870 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
12871 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
12872 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
12873 getBuildVector(NewOvVT, dl, OvScalars));
12874}
12875
12878 unsigned Bytes,
12879 int Dist) const {
12880 if (LD->isVolatile() || Base->isVolatile())
12881 return false;
12882 // TODO: probably too restrictive for atomics, revisit
12883 if (!LD->isSimple())
12884 return false;
12885 if (LD->isIndexed() || Base->isIndexed())
12886 return false;
12887 if (LD->getChain() != Base->getChain())
12888 return false;
12889 EVT VT = LD->getMemoryVT();
12890 if (VT.getSizeInBits() / 8 != Bytes)
12891 return false;
12892
12893 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
12894 auto LocDecomp = BaseIndexOffset::match(LD, *this);
12895
12896 int64_t Offset = 0;
12897 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
12898 return (Dist * (int64_t)Bytes == Offset);
12899 return false;
12900}
12901
12902/// InferPtrAlignment - Infer alignment of a load / store address. Return
12903/// std::nullopt if it cannot be inferred.
12905 // If this is a GlobalAddress + cst, return the alignment.
12906 const GlobalValue *GV = nullptr;
12907 int64_t GVOffset = 0;
12908 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
12909 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12910 KnownBits Known(PtrWidth);
12912 unsigned AlignBits = Known.countMinTrailingZeros();
12913 if (AlignBits)
12914 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
12915 }
12916
12917 // If this is a direct reference to a stack slot, use information about the
12918 // stack slot's alignment.
12919 int FrameIdx = INT_MIN;
12920 int64_t FrameOffset = 0;
12921 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
12922 FrameIdx = FI->getIndex();
12923 } else if (isBaseWithConstantOffset(Ptr) &&
12924 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
12925 // Handle FI+Cst
12926 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
12927 FrameOffset = Ptr.getConstantOperandVal(1);
12928 }
12929
12930 if (FrameIdx != INT_MIN) {
12932 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
12933 }
12934
12935 return std::nullopt;
12936}
12937
12938/// Split the scalar node with EXTRACT_ELEMENT using the provided
12939/// VTs and return the low/high part.
12940std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12941 const SDLoc &DL,
12942 const EVT &LoVT,
12943 const EVT &HiVT) {
12944 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12945 "Split node must be a scalar type");
12946 SDValue Lo =
12948 SDValue Hi =
12950 return std::make_pair(Lo, Hi);
12951}
12952
12953/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12954/// which is split (or expanded) into two not necessarily identical pieces.
12955std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12956 // Currently all types are split in half.
12957 EVT LoVT, HiVT;
12958 if (!VT.isVector())
12959 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
12960 else
12961 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
12962
12963 return std::make_pair(LoVT, HiVT);
12964}
12965
12966/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12967/// type, dependent on an enveloping VT that has been split into two identical
12968/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12969std::pair<EVT, EVT>
12971 bool *HiIsEmpty) const {
12972 EVT EltTp = VT.getVectorElementType();
12973 // Examples:
12974 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
12975 // custom VL=9 with enveloping VL=8/8 yields 8/1
12976 // custom VL=10 with enveloping VL=8/8 yields 8/2
12977 // etc.
12978 ElementCount VTNumElts = VT.getVectorElementCount();
12979 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12980 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12981 "Mixing fixed width and scalable vectors when enveloping a type");
12982 EVT LoVT, HiVT;
12983 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12984 LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12985 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
12986 *HiIsEmpty = false;
12987 } else {
12988 // Flag that hi type has zero storage size, but return split envelop type
12989 // (this would be easier if vector types with zero elements were allowed).
12990 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
12991 HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12992 *HiIsEmpty = true;
12993 }
12994 return std::make_pair(LoVT, HiVT);
12995}
12996
12997/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12998/// low/high part.
12999std::pair<SDValue, SDValue>
13000SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
13001 const EVT &HiVT) {
13002 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
13003 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
13004 "Splitting vector with an invalid mixture of fixed and scalable "
13005 "vector types");
13007 N.getValueType().getVectorMinNumElements() &&
13008 "More vector elements requested than available!");
13009 SDValue Lo, Hi;
13010 Lo =
13012 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
13013 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
13014 // IDX with the runtime scaling factor of the result vector type. For
13015 // fixed-width result vectors, that runtime scaling factor is 1.
13018 return std::make_pair(Lo, Hi);
13019}
13020
13021std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
13022 const SDLoc &DL) {
13023 // Split the vector length parameter.
13024 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
13025 EVT VT = N.getValueType();
13027 "Expecting the mask to be an evenly-sized vector");
13028 unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
13029 SDValue HalfNumElts =
13030 VecVT.isFixedLengthVector()
13031 ? getConstant(HalfMinNumElts, DL, VT)
13032 : getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
13033 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);
13034 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);
13035 return std::make_pair(Lo, Hi);
13036}
13037
13038/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
13040 EVT VT = N.getValueType();
13043 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
13045}
13046
13049 unsigned Start, unsigned Count,
13050 EVT EltVT) {
13051 EVT VT = Op.getValueType();
13052 if (Count == 0)
13053 Count = VT.getVectorNumElements();
13054 if (EltVT == EVT())
13055 EltVT = VT.getVectorElementType();
13056 SDLoc SL(Op);
13057 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
13058 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
13059 getVectorIdxConstant(i, SL)));
13060 }
13061}
13062
13063// getAddressSpace - Return the address space this GlobalAddress belongs to.
13065 return getGlobal()->getType()->getAddressSpace();
13066}
13067
13070 return Val.MachineCPVal->getType();
13071 return Val.ConstVal->getType();
13072}
13073
13074bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
13075 unsigned &SplatBitSize,
13076 bool &HasAnyUndefs,
13077 unsigned MinSplatBits,
13078 bool IsBigEndian) const {
13079 EVT VT = getValueType(0);
13080 assert(VT.isVector() && "Expected a vector type");
13081 unsigned VecWidth = VT.getSizeInBits();
13082 if (MinSplatBits > VecWidth)
13083 return false;
13084
13085 // FIXME: The widths are based on this node's type, but build vectors can
13086 // truncate their operands.
13087 SplatValue = APInt(VecWidth, 0);
13088 SplatUndef = APInt(VecWidth, 0);
13089
13090 // Get the bits. Bits with undefined values (when the corresponding element
13091 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
13092 // in SplatValue. If any of the values are not constant, give up and return
13093 // false.
13094 unsigned int NumOps = getNumOperands();
13095 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
13096 unsigned EltWidth = VT.getScalarSizeInBits();
13097
13098 for (unsigned j = 0; j < NumOps; ++j) {
13099 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
13100 SDValue OpVal = getOperand(i);
13101 unsigned BitPos = j * EltWidth;
13102
13103 if (OpVal.isUndef())
13104 SplatUndef.setBits(BitPos, BitPos + EltWidth);
13105 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
13106 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
13107 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
13108 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
13109 else
13110 return false;
13111 }
13112
13113 // The build_vector is all constants or undefs. Find the smallest element
13114 // size that splats the vector.
13115 HasAnyUndefs = (SplatUndef != 0);
13116
13117 // FIXME: This does not work for vectors with elements less than 8 bits.
13118 while (VecWidth > 8) {
13119 // If we can't split in half, stop here.
13120 if (VecWidth & 1)
13121 break;
13122
13123 unsigned HalfSize = VecWidth / 2;
13124 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
13125 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
13126 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
13127 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
13128
13129 // If the two halves do not match (ignoring undef bits), stop here.
13130 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
13131 MinSplatBits > HalfSize)
13132 break;
13133
13134 SplatValue = HighValue | LowValue;
13135 SplatUndef = HighUndef & LowUndef;
13136
13137 VecWidth = HalfSize;
13138 }
13139
13140 // FIXME: The loop above only tries to split in halves. But if the input
13141 // vector for example is <3 x i16> it wouldn't be able to detect a
13142 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
13143 // optimizations. I guess that back in the days when this helper was created
13144 // vectors normally was power-of-2 sized.
13145
13146 SplatBitSize = VecWidth;
13147 return true;
13148}
13149
13151 BitVector *UndefElements) const {
13152 unsigned NumOps = getNumOperands();
13153 if (UndefElements) {
13154 UndefElements->clear();
13155 UndefElements->resize(NumOps);
13156 }
13157 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
13158 if (!DemandedElts)
13159 return SDValue();
13160 SDValue Splatted;
13161 for (unsigned i = 0; i != NumOps; ++i) {
13162 if (!DemandedElts[i])
13163 continue;
13164 SDValue Op = getOperand(i);
13165 if (Op.isUndef()) {
13166 if (UndefElements)
13167 (*UndefElements)[i] = true;
13168 } else if (!Splatted) {
13169 Splatted = Op;
13170 } else if (Splatted != Op) {
13171 return SDValue();
13172 }
13173 }
13174
13175 if (!Splatted) {
13176 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
13177 assert(getOperand(FirstDemandedIdx).isUndef() &&
13178 "Can only have a splat without a constant for all undefs.");
13179 return getOperand(FirstDemandedIdx);
13180 }
13181
13182 return Splatted;
13183}
13184
13186 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
13187 return getSplatValue(DemandedElts, UndefElements);
13188}
13189
13191 SmallVectorImpl<SDValue> &Sequence,
13192 BitVector *UndefElements) const {
13193 unsigned NumOps = getNumOperands();
13194 Sequence.clear();
13195 if (UndefElements) {
13196 UndefElements->clear();
13197 UndefElements->resize(NumOps);
13198 }
13199 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
13200 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
13201 return false;
13202
13203 // Set the undefs even if we don't find a sequence (like getSplatValue).
13204 if (UndefElements)
13205 for (unsigned I = 0; I != NumOps; ++I)
13206 if (DemandedElts[I] && getOperand(I).isUndef())
13207 (*UndefElements)[I] = true;
13208
13209 // Iteratively widen the sequence length looking for repetitions.
13210 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
13211 Sequence.append(SeqLen, SDValue());
13212 for (unsigned I = 0; I != NumOps; ++I) {
13213 if (!DemandedElts[I])
13214 continue;
13215 SDValue &SeqOp = Sequence[I % SeqLen];
13217 if (Op.isUndef()) {
13218 if (!SeqOp)
13219 SeqOp = Op;
13220 continue;
13221 }
13222 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
13223 Sequence.clear();
13224 break;
13225 }
13226 SeqOp = Op;
13227 }
13228 if (!Sequence.empty())
13229 return true;
13230 }
13231
13232 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
13233 return false;
13234}
13235
13237 BitVector *UndefElements) const {
13238 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
13239 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
13240}
13241
13244 BitVector *UndefElements) const {
13245 return dyn_cast_or_null<ConstantSDNode>(
13246 getSplatValue(DemandedElts, UndefElements));
13247}
13248
13251 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
13252}
13253
13256 BitVector *UndefElements) const {
13257 return dyn_cast_or_null<ConstantFPSDNode>(
13258 getSplatValue(DemandedElts, UndefElements));
13259}
13260
13263 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
13264}
13265
13266int32_t
13268 uint32_t BitWidth) const {
13269 if (ConstantFPSDNode *CN =
13270 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
13271 bool IsExact;
13272 APSInt IntVal(BitWidth);
13273 const APFloat &APF = CN->getValueAPF();
13274 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
13275 APFloat::opOK ||
13276 !IsExact)
13277 return -1;
13278
13279 return IntVal.exactLogBase2();
13280 }
13281 return -1;
13282}
13283
13285 bool IsLittleEndian, unsigned DstEltSizeInBits,
13286 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
13287 // Early-out if this contains anything but Undef/Constant/ConstantFP.
13288 if (!isConstant())
13289 return false;
13290
13291 unsigned NumSrcOps = getNumOperands();
13292 unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();
13293 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13294 "Invalid bitcast scale");
13295
13296 // Extract raw src bits.
13297 SmallVector<APInt> SrcBitElements(NumSrcOps,
13298 APInt::getZero(SrcEltSizeInBits));
13299 BitVector SrcUndeElements(NumSrcOps, false);
13300
13301 for (unsigned I = 0; I != NumSrcOps; ++I) {
13303 if (Op.isUndef()) {
13304 SrcUndeElements.set(I);
13305 continue;
13306 }
13307 auto *CInt = dyn_cast<ConstantSDNode>(Op);
13308 auto *CFP = dyn_cast<ConstantFPSDNode>(Op);
13309 assert((CInt || CFP) && "Unknown constant");
13310 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
13311 : CFP->getValueAPF().bitcastToAPInt();
13312 }
13313
13314 // Recast to dst width.
13315 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
13316 SrcBitElements, UndefElements, SrcUndeElements);
13317 return true;
13318}
13319
13320void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
13321 unsigned DstEltSizeInBits,
13322 SmallVectorImpl<APInt> &DstBitElements,
13323 ArrayRef<APInt> SrcBitElements,
13324 BitVector &DstUndefElements,
13325 const BitVector &SrcUndefElements) {
13326 unsigned NumSrcOps = SrcBitElements.size();
13327 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
13328 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13329 "Invalid bitcast scale");
13330 assert(NumSrcOps == SrcUndefElements.size() &&
13331 "Vector size mismatch");
13332
13333 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
13334 DstUndefElements.clear();
13335 DstUndefElements.resize(NumDstOps, false);
13336 DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));
13337
13338 // Concatenate src elements constant bits together into dst element.
13339 if (SrcEltSizeInBits <= DstEltSizeInBits) {
13340 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
13341 for (unsigned I = 0; I != NumDstOps; ++I) {
13342 DstUndefElements.set(I);
13343 APInt &DstBits = DstBitElements[I];
13344 for (unsigned J = 0; J != Scale; ++J) {
13345 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13346 if (SrcUndefElements[Idx])
13347 continue;
13348 DstUndefElements.reset(I);
13349 const APInt &SrcBits = SrcBitElements[Idx];
13350 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
13351 "Illegal constant bitwidths");
13352 DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);
13353 }
13354 }
13355 return;
13356 }
13357
13358 // Split src element constant bits into dst elements.
13359 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
13360 for (unsigned I = 0; I != NumSrcOps; ++I) {
13361 if (SrcUndefElements[I]) {
13362 DstUndefElements.set(I * Scale, (I + 1) * Scale);
13363 continue;
13364 }
13365 const APInt &SrcBits = SrcBitElements[I];
13366 for (unsigned J = 0; J != Scale; ++J) {
13367 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13368 APInt &DstBits = DstBitElements[Idx];
13369 DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
13370 }
13371 }
13372}
13373
13375 for (const SDValue &Op : op_values()) {
13376 unsigned Opc = Op.getOpcode();
13377 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
13378 return false;
13379 }
13380 return true;
13381}
13382
13383std::optional<std::pair<APInt, APInt>>
13385 unsigned NumOps = getNumOperands();
13386 if (NumOps < 2)
13387 return std::nullopt;
13388
13389 if (!isa<ConstantSDNode>(getOperand(0)) ||
13390 !isa<ConstantSDNode>(getOperand(1)))
13391 return std::nullopt;
13392
13393 unsigned EltSize = getValueType(0).getScalarSizeInBits();
13394 APInt Start = getConstantOperandAPInt(0).trunc(EltSize);
13395 APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start;
13396
13397 if (Stride.isZero())
13398 return std::nullopt;
13399
13400 for (unsigned i = 2; i < NumOps; ++i) {
13401 if (!isa<ConstantSDNode>(getOperand(i)))
13402 return std::nullopt;
13403
13404 APInt Val = getConstantOperandAPInt(i).trunc(EltSize);
13405 if (Val != (Start + (Stride * i)))
13406 return std::nullopt;
13407 }
13408
13409 return std::make_pair(Start, Stride);
13410}
13411
13412bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
13413 // Find the first non-undef value in the shuffle mask.
13414 unsigned i, e;
13415 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
13416 /* search */;
13417
13418 // If all elements are undefined, this shuffle can be considered a splat
13419 // (although it should eventually get simplified away completely).
13420 if (i == e)
13421 return true;
13422
13423 // Make sure all remaining elements are either undef or the same as the first
13424 // non-undef value.
13425 for (int Idx = Mask[i]; i != e; ++i)
13426 if (Mask[i] >= 0 && Mask[i] != Idx)
13427 return false;
13428 return true;
13429}
13430
13431// Returns true if it is a constant integer BuildVector or constant integer,
13432// possibly hidden by a bitcast.
13434 SDValue N, bool AllowOpaques) const {
13436
13437 if (auto *C = dyn_cast<ConstantSDNode>(N))
13438 return AllowOpaques || !C->isOpaque();
13439
13441 return true;
13442
13443 // Treat a GlobalAddress supporting constant offset folding as a
13444 // constant integer.
13445 if (auto *GA = dyn_cast<GlobalAddressSDNode>(N))
13446 if (GA->getOpcode() == ISD::GlobalAddress &&
13447 TLI->isOffsetFoldingLegal(GA))
13448 return true;
13449
13450 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
13451 isa<ConstantSDNode>(N.getOperand(0)))
13452 return true;
13453 return false;
13454}
13455
13456// Returns true if it is a constant float BuildVector or constant float.
13458 if (isa<ConstantFPSDNode>(N))
13459 return true;
13460
13462 return true;
13463
13464 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
13465 isa<ConstantFPSDNode>(N.getOperand(0)))
13466 return true;
13467
13468 return false;
13469}
13470
13472 bool AllowTruncation) const {
13473 ConstantSDNode *Const = isConstOrConstSplat(N, false, AllowTruncation);
13474 if (!Const)
13475 return std::nullopt;
13476
13477 const APInt &CVal = Const->getAPIntValue();
13478 switch (TLI->getBooleanContents(N.getValueType())) {
13480 if (CVal.isOne())
13481 return true;
13482 if (CVal.isZero())
13483 return false;
13484 return std::nullopt;
13486 if (CVal.isAllOnes())
13487 return true;
13488 if (CVal.isZero())
13489 return false;
13490 return std::nullopt;
13492 return CVal[0];
13493 }
13494 llvm_unreachable("Unknown BooleanContent enum");
13495}
13496
13497void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
13498 assert(!Node->OperandList && "Node already has operands");
13500 "too many operands to fit into SDNode");
13501 SDUse *Ops = OperandRecycler.allocate(
13502 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
13503
13504 bool IsDivergent = false;
13505 for (unsigned I = 0; I != Vals.size(); ++I) {
13506 Ops[I].setUser(Node);
13507 Ops[I].setInitial(Vals[I]);
13508 EVT VT = Ops[I].getValueType();
13509
13510 // Skip Chain. It does not carry divergence.
13511 if (VT != MVT::Other &&
13512 (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
13513 Ops[I].getNode()->isDivergent()) {
13514 IsDivergent = true;
13515 }
13516 }
13517 Node->NumOperands = Vals.size();
13518 Node->OperandList = Ops;
13519 if (!TLI->isSDNodeAlwaysUniform(Node)) {
13520 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
13521 Node->SDNodeBits.IsDivergent = IsDivergent;
13522 }
13524}
13525
13528 size_t Limit = SDNode::getMaxNumOperands();
13529 while (Vals.size() > Limit) {
13530 unsigned SliceIdx = Vals.size() - Limit;
13531 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
13532 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
13533 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
13534 Vals.emplace_back(NewTF);
13535 }
13536 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
13537}
13538
13540 EVT VT, SDNodeFlags Flags) {
13541 switch (Opcode) {
13542 default:
13543 return SDValue();
13544 case ISD::ADD:
13545 case ISD::OR:
13546 case ISD::XOR:
13547 case ISD::UMAX:
13548 return getConstant(0, DL, VT);
13549 case ISD::MUL:
13550 return getConstant(1, DL, VT);
13551 case ISD::AND:
13552 case ISD::UMIN:
13553 return getAllOnesConstant(DL, VT);
13554 case ISD::SMAX:
13556 case ISD::SMIN:
13558 case ISD::FADD:
13559 // If flags allow, prefer positive zero since it's generally cheaper
13560 // to materialize on most targets.
13561 return getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
13562 case ISD::FMUL:
13563 return getConstantFP(1.0, DL, VT);
13564 case ISD::FMINNUM:
13565 case ISD::FMAXNUM: {
13566 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13567 const fltSemantics &Semantics = VT.getFltSemantics();
13568 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
13569 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
13570 APFloat::getLargest(Semantics);
13571 if (Opcode == ISD::FMAXNUM)
13572 NeutralAF.changeSign();
13573
13574 return getConstantFP(NeutralAF, DL, VT);
13575 }
13576 case ISD::FMINIMUM:
13577 case ISD::FMAXIMUM: {
13578 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
13579 const fltSemantics &Semantics = VT.getFltSemantics();
13580 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
13581 : APFloat::getLargest(Semantics);
13582 if (Opcode == ISD::FMAXIMUM)
13583 NeutralAF.changeSign();
13584
13585 return getConstantFP(NeutralAF, DL, VT);
13586 }
13587
13588 }
13589}
13590
13591/// Helper used to make a call to a library function that has one argument of
13592/// pointer type.
13593///
13594/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
13595/// used to get or set floating-point state. They have one argument of pointer
13596/// type, which points to the memory region containing bits of the
13597/// floating-point state. The value returned by such function is ignored in the
13598/// created call.
13599///
13600/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
13601/// \param Ptr Pointer used to save/load state.
13602/// \param InChain Ingoing token chain.
13603/// \returns Outgoing chain token.
13605 SDValue InChain,
13606 const SDLoc &DLoc) {
13607 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
13610 Entry.Node = Ptr;
13611 Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext());
13612 Args.push_back(Entry);
13613 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
13614 SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),
13615 TLI->getPointerTy(getDataLayout()));
13617 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
13618 TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee,
13619 std::move(Args));
13620 return TLI->LowerCallTo(CLI).second;
13621}
13622
13624 assert(From && To && "Invalid SDNode; empty source SDValue?");
13625 auto I = SDEI.find(From);
13626 if (I == SDEI.end())
13627 return;
13628
13629 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
13630 // the iterator, hence the need to make a copy to prevent a use-after-free.
13631 NodeExtraInfo NEI = I->second;
13632 if (LLVM_LIKELY(!NEI.PCSections) && LLVM_LIKELY(!NEI.MMRA)) {
13633 // No deep copy required for the types of extra info set.
13634 //
13635 // FIXME: Investigate if other types of extra info also need deep copy. This
13636 // depends on the types of nodes they can be attached to: if some extra info
13637 // is only ever attached to nodes where a replacement To node is always the
13638 // node where later use and propagation of the extra info has the intended
13639 // semantics, no deep copy is required.
13640 SDEI[To] = std::move(NEI);
13641 return;
13642 }
13643
13644 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
13645 // through the replacement of From with To. Otherwise, replacements of a node
13646 // (From) with more complex nodes (To and its operands) may result in lost
13647 // extra info where the root node (To) is insignificant in further propagating
13648 // and using extra info when further lowering to MIR.
13649 //
13650 // In the first step pre-populate the visited set with the nodes reachable
13651 // from the old From node. This avoids copying NodeExtraInfo to parts of the
13652 // DAG that is not new and should be left untouched.
13653 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
13654 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
13655 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
13656 if (MaxDepth == 0) {
13657 // Remember this node in case we need to increase MaxDepth and continue
13658 // populating FromReach from this node.
13659 Leafs.emplace_back(N);
13660 return;
13661 }
13662 if (!FromReach.insert(N).second)
13663 return;
13664 for (const SDValue &Op : N->op_values())
13665 Self(Self, Op.getNode(), MaxDepth - 1);
13666 };
13667
13668 // Copy extra info to To and all its transitive operands (that are new).
13670 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
13671 if (FromReach.contains(N))
13672 return true;
13673 if (!Visited.insert(N).second)
13674 return true;
13675 if (getEntryNode().getNode() == N)
13676 return false;
13677 for (const SDValue &Op : N->op_values()) {
13678 if (!Self(Self, Op.getNode()))
13679 return false;
13680 }
13681 // Copy only if entry node was not reached.
13682 SDEI[N] = NEI;
13683 return true;
13684 };
13685
13686 // We first try with a lower MaxDepth, assuming that the path to common
13687 // operands between From and To is relatively short. This significantly
13688 // improves performance in the common case. The initial MaxDepth is big
13689 // enough to avoid retry in the common case; the last MaxDepth is large
13690 // enough to avoid having to use the fallback below (and protects from
13691 // potential stack exhaustion from recursion).
13692 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
13693 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
13694 // StartFrom is the previous (or initial) set of leafs reachable at the
13695 // previous maximum depth.
13697 std::swap(StartFrom, Leafs);
13698 for (const SDNode *N : StartFrom)
13699 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
13700 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
13701 return;
13702 // This should happen very rarely (reached the entry node).
13703 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
13704 assert(!Leafs.empty());
13705 }
13706
13707 // This should not happen - but if it did, that means the subgraph reachable
13708 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
13709 // could not visit all reachable common operands. Consequently, we were able
13710 // to reach the entry node.
13711 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
13712 assert(false && "From subgraph too complex - increase max. MaxDepth?");
13713 // Best-effort fallback if assertions disabled.
13714 SDEI[To] = std::move(NEI);
13715}
13716
13717#ifndef NDEBUG
13718static void checkForCyclesHelper(const SDNode *N,
13721 const llvm::SelectionDAG *DAG) {
13722 // If this node has already been checked, don't check it again.
13723 if (Checked.count(N))
13724 return;
13725
13726 // If a node has already been visited on this depth-first walk, reject it as
13727 // a cycle.
13728 if (!Visited.insert(N).second) {
13729 errs() << "Detected cycle in SelectionDAG\n";
13730 dbgs() << "Offending node:\n";
13731 N->dumprFull(DAG); dbgs() << "\n";
13732 abort();
13733 }
13734
13735 for (const SDValue &Op : N->op_values())
13736 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
13737
13738 Checked.insert(N);
13739 Visited.erase(N);
13740}
13741#endif
13742
13744 const llvm::SelectionDAG *DAG,
13745 bool force) {
13746#ifndef NDEBUG
13747 bool check = force;
13748#ifdef EXPENSIVE_CHECKS
13749 check = true;
13750#endif // EXPENSIVE_CHECKS
13751 if (check) {
13752 assert(N && "Checking nonexistent SDNode");
13755 checkForCyclesHelper(N, visited, checked, DAG);
13756 }
13757#endif // !NDEBUG
13758}
13759
13760void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13761 checkForCycles(DAG->getRoot().getNode(), DAG, force);
13762}
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
Definition: Compiler.h:553
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:319
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Given that RA is a live value
#define LLVM_DEBUG(...)
Definition: Debug.h:106
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
uint32_t Index
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines a hash set that can be used to remove duplication of nodes in a graph.
#define _
iv users
Definition: IVUsers.cpp:48
static const unsigned MaxDepth
static LVOptions Options
Definition: LVOptions.cpp:25
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:533
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
mir Rename Register Operands
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file contains some templates that are useful if you are working with the STL at all.
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, SDVTList VTList, ArrayRef< SDValue > OpList)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static cl::opt< bool > EnableMemCpyDAGOpt("enable-memcpy-dag-opt", cl::Hidden, cl::init(true), cl::desc("Gang up loads and stores generated by inlining of memcpy"))
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList)
AddNodeIDValueTypes - Value type lists are intern'd so we can represent them solely with their pointe...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA)
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)
AddNodeIDOpcode - Add the node opcode to the NodeID data.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static cl::opt< int > MaxLdStGlue("ldstmemcpy-glue-max", cl::desc("Number limit for gluing ld/st of memcpy."), cl::Hidden, cl::init(0))
static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef< SDValue > Ops)
AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode, SDNode *FPNode)
Given a store node StoreNode, return true if it is safe to fold that node into FPNode,...
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N)
If this is an SDNode with special info, add this info to the NodeID data.
static bool gluePropagatesDivergence(const SDNode *Node)
Return true if a glue output should propagate divergence information.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void VerifySDNode(SDNode *N, const TargetLowering *TLI)
VerifySDNode - Check the given SDNode. Aborts if it is invalid.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS)
static cl::opt< unsigned > MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(8192), cl::desc("DAG combiner limit number of steps when searching DAG " "for predecessor nodes"))
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition: APFloat.h:1117
opStatus divide(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1205
void copySign(const APFloat &RHS)
Definition: APFloat.h:1299
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5465
opStatus subtract(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1187
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: APFloat.h:1423
opStatus add(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1178
bool isFinite() const
Definition: APFloat.h:1445
bool isNaN() const
Definition: APFloat.h:1438
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition: APFloat.h:1329
opStatus multiply(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1196
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
Definition: APFloat.h:1232
bool isZero() const
Definition: APFloat.h:1436
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1135
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1321
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:1095
opStatus mod(const APFloat &RHS)
Definition: APFloat.h:1223
bool isPosZero() const
Definition: APFloat.h:1451
bool isNegZero() const
Definition: APFloat.h:1452
void changeSign()
Definition: APFloat.h:1294
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Definition: APFloat.h:1106
bool isInfinity() const
Definition: APFloat.h:1437
Class for arbitrary precision integers.
Definition: APInt.h:78
APInt umul_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1945
APInt usub_sat(const APInt &RHS) const
Definition: APInt.cpp:2029
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1547
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1407
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:986
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:229
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1520
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
Definition: APInt.h:1392
unsigned popcount() const
Count the number of bits set.
Definition: APInt.h:1649
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
Definition: APInt.h:1386
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
Definition: APInt.cpp:612
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1007
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1492
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:910
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1330
APInt abs() const
Get the absolute value.
Definition: APInt.h:1773
APInt sadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2000
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:371
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1182
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:380
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition: APInt.cpp:1640
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1468
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1111
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:209
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:329
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1618
void clearAllBits()
Set every bit to 0.
Definition: APInt.h:1397
APInt rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
Definition: APInt.cpp:1128
APInt reverseBits() const
Definition: APInt.cpp:741
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:834
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition: APInt.h:1166
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1618
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
Definition: APInt.h:1607
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1577
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:624
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:219
APInt sshl_sat(const APInt &RHS) const
Definition: APInt.cpp:2060
APInt ushl_sat(const APInt &RHS) const
Definition: APInt.cpp:2074
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1015
APInt rotl(unsigned rotateAmt) const
Rotate left by rotateAmt.
Definition: APInt.cpp:1115
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:370
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
Definition: APInt.h:1417
unsigned logBase2() const
Definition: APInt.h:1739
APInt uadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2010
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:827
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1319
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
Definition: APInt.cpp:1710
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:334
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1150
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:959
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1367
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:873
APInt byteSwap() const
Definition: APInt.cpp:719
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1257
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:440
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:306
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:200
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
Definition: APInt.h:1389
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:455
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition: APInt.h:1237
bool isOne() const
Determine if this is a value of 1.
Definition: APInt.h:389
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:286
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:239
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:858
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:851
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1221
APInt ssub_sat(const APInt &RHS) const
Definition: APInt.cpp:2019
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
Recycle small arrays allocated from a BumpPtrAllocator.
Definition: ArrayRecycler.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:157
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:168
iterator begin() const
Definition: ArrayRef.h:156
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:163
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:198
This is an SDNode representing atomic operations.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
BitVector & reset()
Definition: BitVector.h:392
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
void clear()
clear - Removes all bits from the bitvector.
Definition: BitVector.h:335
BitVector & set()
Definition: BitVector.h:351
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:188
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
The address of a basic block.
Definition: Constants.h:893
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
std::optional< std::pair< APInt, APInt > > isConstantSequence() const
If this BuildVector is constant and represents the numerical series "<a, a+n, a+2n,...
SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
Definition: Allocator.h:148
void Reset()
Deallocate all but the current slab and reset the current pointer to the beginning of it,...
Definition: Allocator.h:123
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:271
const APFloat & getValue() const
Definition: Constants.h:315
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:151
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:148
bool isMachineConstantPoolEntry() const
This class represents a range of values.
Definition: ConstantRange.h:47
ConstantRange multiply(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
KnownBits toKnownBits() const
Return known bits for values in this range.
ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition: Constant.h:42
Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
Definition: Constants.cpp:1708
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:435
DWARF expression.
static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:197
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:851
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:843
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:743
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:847
A debug info location.
Definition: DebugLoc.h:33
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
Implements a dense probed hash-table based set.
Definition: DenseSet.h:278
const char * getSymbol() const
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Definition: FoldingSet.h:327
MachineBasicBlock * MBB
MBB - The current block.
Data structure describing the variable locations in a function.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:704
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:353
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:369
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
Definition: GlobalValue.h:263
unsigned getAddressSpace() const
Definition: GlobalValue.h:205
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
This class is used to form a handle around another node that is persistent and is updated across invo...
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
constexpr bool isValid() const
Definition: LowLevelType.h:145
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate the offet and size that ar...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:1069
Machine Value Type.
SimpleValueType SimpleTy
static MVT getIntegerVT(unsigned BitWidth)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Abstract base class for all machine specific constantpool value subclasses.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
This class contains meta information specific to a module.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *MMO)
MachineMemOperand * MMO
Memory reference information.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:228
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:310
The optimization diagnostic interface.
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:686
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Definition: DerivedTypes.h:703
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Definition: PointerUnion.h:142
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Keeps track of dbg_value information through SDISel.
Definition: SelectionDAG.h:162
BumpPtrAllocator & getAlloc()
Definition: SelectionDAG.h:191
void add(SDDbgValue *V, bool isParameter)
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node) const
Definition: SelectionDAG.h:197
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isDivergent() const
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
iterator_range< use_iterator > uses()
MemSDNodeBitfields MemSDNodeBits
void Profile(FoldingSetNodeID &ID) const
Gather unique data for the node.
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
void DropOperands()
Release the operands and set this node to have zero operands.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
SDNode * getUser()
This returns the SDNode that contains this Use.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual bool isTargetMemoryOpcode(unsigned Opcode) const
Returns true if a node with the given target-specific opcode has a memory operand.
virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const
Emit target-specific code that performs a memset.
virtual SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memmove.
virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memcpy.
SDNodeFlags getFlags() const
Definition: SelectionDAG.h:387
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:228
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:575
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:497
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
void updateDivergence(SDNode *N)
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
static unsigned getHasPredecessorMaxSteps()
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
bool cannotBeOrderedNegativeFP(SDValue Op) const
Test whether the given float value is known to be positive.
SDValue getRegister(Register Reg, EVT VT)
bool calculateDivergence(SDNode *N)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
std::optional< uint64_t > getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:501
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:456
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:395
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:555
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:854
SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
bool isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:495
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, const SDLoc &DLoc)
Helper used to make a call to a library function that has one argument of pointer type.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:888
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
SDValue getPartialReduceAdd(SDLoc DL, EVT ReducedTy, SDValue Op1, SDValue Op2)
Create the DAG equivalent of vector_partial_reduce where Op1 and Op2 are its operands and ReducedTY i...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:496
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
Definition: SelectionDAG.h:567
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
Definition: SelectionDAG.h:563
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:698
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:502
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:490
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
std::optional< uint64_t > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:871
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing=false)
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
bool isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth=0) const
Test if the given fp value is known to be an integer power-of-2, either positive or negative.
OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
bool expandMultipleResultFPLibCall(RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={})
Expands a node with multiple results to an FP or vector libcall.
std::optional< uint64_t > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVMContext * getContext() const
Definition: SelectionDAG.h:508
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:584
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
std::optional< ConstantRange > getValidShiftAmountRange(SDValue V, const APInt &DemandedElts, unsigned Depth) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
std::optional< bool > isBoolConstant(SDValue N, bool AllowTruncation=false) const
Check if a value \op N is a constant using the target's BooleanContent for its type.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:578
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:904
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
bool isADDLike(SDValue Op, bool NoWrap=false) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:558
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
size_type size() const
Definition: SmallPtrSet.h:94
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:363
bool erase(PtrType Ptr)
Remove pointer from the set.
Definition: SmallPtrSet.h:401
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:452
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:384
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:519
bool empty() const
Definition: SmallVector.h:81
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
void assign(size_type NumElts, ValueParamT Elt)
Definition: SmallVector.h:704
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:937
void reserve(size_type N)
Definition: SmallVector.h:663
iterator erase(const_iterator CI)
Definition: SmallVector.h:737
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:683
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:144
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
BooleanContent
Enum that describes how the target represents true/false values.
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
virtual void verifyTargetSDNode(const SDNode *N) const
Check the given SDNode. Aborts if it is invalid.
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
TargetOptions Options
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:568
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:345
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:270
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void set(Value *Val)
Definition: Value.h:886
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
Provides info so a possible vectorization of a function can be computed.
bool isMasked() const
StringRef getVectorFnName() const
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:213
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:193
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:232
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
Definition: TypeSize.h:179
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:239
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt mulhu(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on zero-extended operands.
Definition: APInt.cpp:3104
const APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
Definition: APInt.h:2242
APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
Definition: APInt.cpp:3091
APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
Definition: APInt.cpp:3081
const APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
Definition: APInt.h:2237
APInt mulhs(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on sign-extended operands.
Definition: APInt.cpp:3096
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition: APInt.cpp:2982
APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
Definition: APInt.cpp:3076
APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
Definition: APInt.cpp:3086
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:780
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:753
@ TargetConstantPool
Definition: ISDOpcodes.h:174
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1242
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:491
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1347
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1417
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
Definition: ISDOpcodes.h:1360
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1450
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:512
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:257
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1340
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:574
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
Definition: ISDOpcodes.h:1131
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:744
@ TargetBlockAddress
Definition: ISDOpcodes.h:176
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1342
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1312
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1343
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:276
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:502
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1102
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:814
@ ATOMIC_LOAD_USUB_COND
Definition: ISDOpcodes.h:1351
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:498
@ FATAN2
FATAN2 - atan2, inspired by libm.
Definition: ISDOpcodes.h:999
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:205
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1325
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:841
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:558
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1435
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1439
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:717
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:871
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1449
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:492
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:964
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1338
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:954
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:236
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1339
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:997
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1490
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1345
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
Definition: ISDOpcodes.h:1238
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1173
@ ATOMIC_LOAD_USUB_SAT
Definition: ISDOpcodes.h:1352
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:805
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:685
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:635
@ TargetExternalSymbol
Definition: ISDOpcodes.h:175
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1432
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:752
@ TargetJumpTable
Definition: ISDOpcodes.h:173
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
Definition: ISDOpcodes.h:183
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1292
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1436
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:788
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:981
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1346
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1148
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:334
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1341
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:661
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:515
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:356
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:757
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1308
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:218
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1451
@ RegisterMask
Definition: ISDOpcodes.h:75
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:229
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:642
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1348
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:215
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:330
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:170
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1444
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:674
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:735
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1337
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:615
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1336
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:588
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:1044
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:550
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:209
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:811
@ TargetConstantFP
Definition: ISDOpcodes.h:165
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:907
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:772
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1407
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1319
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1344
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:1031
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:366
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:338
@ TargetFrameIndex
Definition: ISDOpcodes.h:172
@ ConstantPool
Definition: ISDOpcodes.h:82
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:860
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:849
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:697
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1377
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:939
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:766
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:310
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
Definition: ISDOpcodes.h:1372
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1262
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1452
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:973
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1350
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1334
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:480
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1050
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1335
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:887
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:164
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:485
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:709
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:190
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1078
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1402
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:705
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:680
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1433
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:286
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition: ISDOpcodes.h:651
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:223
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:539
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:627
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1333
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1004
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:920
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition: ISDOpcodes.h:669
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:882
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:958
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Definition: ISDOpcodes.h:1481
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:906
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1440
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:817
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1217
@ BlockAddress
Definition: ISDOpcodes.h:84
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1418
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:794
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1349
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:508
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:347
@ AssertZext
Definition: ISDOpcodes.h:62
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
Definition: ISDOpcodes.h:1211
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1083
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
Definition: ISDOpcodes.h:1055
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition: ISDOpcodes.h:692
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:320
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:198
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:171
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:530
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
bool isExtOpcode(unsigned Opcode)
Definition: ISDOpcodes.h:1677
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed)
Returns true if the specified node is a vector where all elements can be truncated to the specified e...
bool matchUnaryPredicateImpl(SDValue Op, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1664
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
Definition: ISDOpcodes.h:1669
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
std::optional< unsigned > getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1568
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
NodeType getInverseMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns ISD::(U|S)MAX and ISD::(U|S)MIN,...
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1555
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1606
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1586
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:903
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:147
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:353
@ Offset
Definition: DWP.cpp:480
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:361
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1739
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:255
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition: Utils.cpp:1565
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2448
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void * PointerTy
Definition: GenericValue.h:21
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition: bit.h:317
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition: APFloat.h:1599
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2115
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:657
MaybeAlign getAlign(const Function &F, unsigned Index)
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition: Utils.cpp:1547
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition: APFloat.h:1521
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1746
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 maximumNumber semantics.
Definition: APFloat.h:1558
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:340
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:291
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1664
LLVM_READONLY APFloat minimumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimumNumber semantics.
Definition: APFloat.h:1585
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1753
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Other
Any other memory.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 minimumNumber semantics.
Definition: APFloat.h:1544
@ Mul
Product of integers.
bool isNullConstantOrUndef(SDValue V)
Returns true if V is a constant integer zero or an UNDEF node.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1841
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:217
bool funcReturnsFirstArgOfCall(const CallInst &CI)
Returns true if the parent of CI returns CI's first argument after calling CI.
Definition: Analysis.cpp:715
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1903
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:581
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:208
void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition: APFloat.h:1572
LLVM_READONLY APFloat maximumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximumNumber semantics.
Definition: APFloat.h:1612
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:382
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
Definition: Metadata.h:780
MDNode * TBAA
The tag for type-based alias analysis.
Definition: Metadata.h:777
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:265
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition: APFloat.h:287
static constexpr roundingMode rmTowardNegative
Definition: APFloat.h:300
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:297
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:301
static const fltSemantics & IEEEquad() LLVM_READNONE
Definition: APFloat.cpp:267
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:266
static const fltSemantics & IEEEhalf() LLVM_READNONE
Definition: APFloat.cpp:263
static constexpr roundingMode rmTowardPositive
Definition: APFloat.h:299
static const fltSemantics & BFloat() LLVM_READNONE
Definition: APFloat.cpp:264
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:313
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
Extended Value Type.
Definition: ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:390
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:137
intptr_t getRawBits() const
Definition: ValueTypes.h:507
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:279
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:295
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:147
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:345
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:368
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:354
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:380
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:311
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
bool isFixedLengthVector() const
Definition: ValueTypes.h:181
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:318
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:287
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:251
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:210
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:174
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:323
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition: ValueTypes.h:142
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
Definition: ValueTypes.cpp:320
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:331
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:303
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:448
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:152
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition: KnownBits.h:293
KnownBits sextInReg(unsigned SrcBitWidth) const
Return known bits for a in-register sign extension of the value we're tracking.
Definition: KnownBits.cpp:158
static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
Definition: KnownBits.cpp:909
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
Definition: KnownBits.h:247
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
Definition: KnownBits.cpp:211
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:100
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:79
void makeNonNegative()
Make this value non-negative.
Definition: KnownBits.h:116
static KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
Definition: KnownBits.cpp:774
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:234
static KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
Definition: KnownBits.cpp:428
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
Definition: KnownBits.cpp:1049
bool isUnknown() const
Returns true if we don't know any bits.
Definition: KnownBits.h:65
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:266
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
Definition: KnownBits.cpp:496
void makeNegative()
Make this value negative.
Definition: KnownBits.h:111
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition: KnownBits.h:153
KnownBits byteSwap() const
Definition: KnownBits.h:471
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:281
void setAllZero()
Make all bits known to be zero and discard any previous information.
Definition: KnownBits.h:85
KnownBits reverseBits() const
Definition: KnownBits.h:475
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
Definition: KnownBits.h:225
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:43
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
Definition: KnownBits.cpp:187
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition: KnownBits.h:164
void resetAll()
Resets the known state of all bits.
Definition: KnownBits.h:73
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
Definition: KnownBits.h:313
static KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
Definition: KnownBits.cpp:370
bool isNonZero() const
Returns true if this value is known to be non-zero.
Definition: KnownBits.h:103
static KnownBits abdu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for abdu(LHS, RHS).
Definition: KnownBits.cpp:228
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
Definition: KnownBits.h:217
static KnownBits avgFloorU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorU.
Definition: KnownBits.cpp:792
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition: KnownBits.h:303
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition: KnownBits.h:172
static KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
Definition: KnownBits.cpp:146
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
Definition: KnownBits.h:188
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:137
static KnownBits abds(KnownBits LHS, KnownBits RHS)
Compute known bits for abds(LHS, RHS).
Definition: KnownBits.cpp:247
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
Definition: KnownBits.cpp:215
static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
Definition: KnownBits.cpp:901
static KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
Definition: KnownBits.cpp:1066
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
Definition: KnownBits.cpp:1009
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
Definition: KnownBits.cpp:60
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
Definition: KnownBits.h:106
static KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
Definition: KnownBits.cpp:953
static KnownBits avgFloorS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorS.
Definition: KnownBits.cpp:788
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
Definition: KnownBits.h:318
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:97
static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
Definition: KnownBits.cpp:53
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition: KnownBits.h:272
void insertBits(const KnownBits &SubBits, unsigned BitPosition)
Insert the bits from a smaller known bits starting at bitPosition.
Definition: KnownBits.h:211
static KnownBits avgCeilU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilU.
Definition: KnownBits.cpp:800
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
Definition: KnownBits.cpp:804
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
Definition: KnownBits.h:159
KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
Definition: KnownBits.cpp:550
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
Definition: KnownBits.cpp:285
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
Definition: KnownBits.cpp:205
static KnownBits avgCeilS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilS.
Definition: KnownBits.cpp:796
This class contains a discriminated union of information about pointers in memory operands,...
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNonNeg() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned int NumVTs
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
Definition: SelectionDAG.h:315
DAGUpdateListener *const Next
Definition: SelectionDAG.h:316
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)