LLVM 24.0.0git
SelectionDAG.cpp
Go to the documentation of this file.
1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SDNodeDbgValue.h"
15#include "SelectionDAGOptions.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/BitVector.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
23#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/Twine.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/Constants.h"
55#include "llvm/IR/DataLayout.h"
57#include "llvm/IR/DebugLoc.h"
59#include "llvm/IR/Function.h"
60#include "llvm/IR/GlobalValue.h"
62#include "llvm/IR/Metadata.h"
63#include "llvm/IR/Type.h"
67#include "llvm/Support/Debug.h"
77#include <algorithm>
78#include <cassert>
79#include <cstdint>
80#include <cstdlib>
81#include <limits>
82#include <optional>
83#include <string>
84#include <utility>
85#include <vector>
86
87using namespace llvm;
88using namespace llvm::SDPatternMatch;
89
90/// makeVTList - Return an instance of the SDVTList struct initialized with the
91/// specified members.
92static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
93 SDVTList Res = {VTs, NumVTs};
94 return Res;
95}
96
97// Default null implementations of the callbacks.
101
102void SelectionDAG::DAGNodeDeletedListener::anchor() {}
103void SelectionDAG::DAGNodeInsertedListener::anchor() {}
104
105#define DEBUG_TYPE "selectiondag"
106
108 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
109}
110
112 return getSelectionDAGOptions().has_predecessor_max_steps;
113}
114
115//===----------------------------------------------------------------------===//
116// ConstantFPSDNode Class
117//===----------------------------------------------------------------------===//
118
119/// isExactlyValue - We don't rely on operator== working on double values, as
120/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
121/// As such, this method can be used to do an exact bit-for-bit comparison of
122/// two floating point values.
124 return getValueAPF().bitwiseIsEqual(V);
125}
126
128 const APFloat& Val) {
129 assert(VT.isFloatingPoint() && "Can only convert between FP types");
130
131 // convert modifies in place, so make a copy.
132 APFloat Val2 = APFloat(Val);
133 bool losesInfo;
135 &losesInfo);
136 return !losesInfo;
137}
138
139//===----------------------------------------------------------------------===//
140// ISD Namespace
141//===----------------------------------------------------------------------===//
142
144 if (V.getOpcode() == ISD::SPLAT_VECTOR) {
145 if (auto OptAPInt = V.getOperand(0)->bitcastToAPInt()) {
146 unsigned EltSize =
147 V.getValueType().getVectorElementType().getSizeInBits();
148 SplatVal = OptAPInt->trunc(EltSize);
149 return true;
150 }
151 }
152
153 auto *BV = dyn_cast<BuildVectorSDNode>(V);
154 if (!BV)
155 return false;
156
157 APInt SplatUndef;
158 unsigned SplatBitSize;
159 bool HasUndefs;
160 unsigned EltSize = V.getValueType().getVectorElementType().getSizeInBits();
161 // Endianness does not matter here. We are checking for a splat given the
162 // element size of the vector, and if we find such a splat for little endian
163 // layout, then that should be valid also for big endian (as the full vector
164 // size is known to be a multiple of the element size).
165 const bool IsBigEndian = false;
166 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
167 EltSize, IsBigEndian) &&
168 EltSize == SplatBitSize;
169}
170
171// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
172// specializations of the more general isConstantSplatVector()?
173
174bool ISD::isConstantSplatVectorAllOnes(SDValue V, bool BuildVectorOnly) {
175 // Look through a bit convert.
176 while (V.getOpcode() == ISD::BITCAST)
177 V = V.getOperand(0);
178
179 if (!BuildVectorOnly && V.getOpcode() == ISD::SPLAT_VECTOR) {
180 APInt SplatVal;
181 return isConstantSplatVector(V, SplatVal) && SplatVal.isAllOnes();
182 }
183
184 if (V.getOpcode() != ISD::BUILD_VECTOR)
185 return false;
186
187 unsigned i = 0, e = V.getNumOperands();
188
189 // Skip over all of the undef values.
190 while (i != e && V.getOperand(i).isUndef())
191 ++i;
192
193 // Do not accept an all-undef vector.
194 if (i == e) return false;
195
196 // Do not accept build_vectors that aren't all constants or which have non-~0
197 // elements. We have to be a bit careful here, as the type of the constant
198 // may not be the same as the type of the vector elements due to type
199 // legalization (the elements are promoted to a legal type for the target and
200 // a vector of a type may be legal when the base element type is not).
201 // We only want to check enough bits to cover the vector elements, because
202 // we care if the resultant vector is all ones, not whether the individual
203 // constants are.
204 SDValue NotZero = V.getOperand(i);
205 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
206 unsigned EltSize = V.getValueType().getScalarSizeInBits();
207 if (OptAPInt->countr_one() < EltSize)
208 return false;
209 } else
210 return false;
211
212 // Okay, we have at least one ~0 value, check to see if the rest match or are
213 // undefs. Even with the above element type twiddling, this should be OK, as
214 // the same type legalization should have applied to all the elements.
215 for (++i; i != e; ++i)
216 if (V.getOperand(i) != NotZero && !V.getOperand(i).isUndef())
217 return false;
218 return true;
219}
220
221bool ISD::isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly) {
222 // Look through a bit convert.
223 while (V.getOpcode() == ISD::BITCAST)
224 V = V.getOperand(0);
225
226 if (!BuildVectorOnly && V.getOpcode() == ISD::SPLAT_VECTOR) {
227 APInt SplatVal;
228 return isConstantSplatVector(V, SplatVal) && SplatVal.isZero();
229 }
230
231 if (V.getOpcode() != ISD::BUILD_VECTOR)
232 return false;
233
234 bool IsAllUndef = true;
235 for (const SDValue &Op : V->op_values()) {
236 if (Op.isUndef())
237 continue;
238 IsAllUndef = false;
239 // Do not accept build_vectors that aren't all constants or which have non-0
240 // elements. We have to be a bit careful here, as the type of the constant
241 // may not be the same as the type of the vector elements due to type
242 // legalization (the elements are promoted to a legal type for the target
243 // and a vector of a type may be legal when the base element type is not).
244 // We only want to check enough bits to cover the vector elements, because
245 // we care if the resultant vector is all zeros, not whether the individual
246 // constants are.
247 if (auto OptAPInt = Op->bitcastToAPInt()) {
248 unsigned EltSize = V.getValueType().getScalarSizeInBits();
249 if (OptAPInt->countr_zero() < EltSize)
250 return false;
251 } else
252 return false;
253 }
254
255 // Do not accept an all-undef vector.
256 if (IsAllUndef)
257 return false;
258 return true;
259}
260
262 return isConstantSplatVectorAllOnes(V, /*BuildVectorOnly*/ true);
263}
264
266 return isConstantSplatVectorAllZeros(V, /*BuildVectorOnly*/ true);
267}
268
270 if (V.getOpcode() != ISD::BUILD_VECTOR)
271 return false;
272
273 for (const SDValue &Op : V->op_values()) {
274 if (Op.isUndef())
275 continue;
277 return false;
278 }
279 return true;
280}
281
283 if (V.getOpcode() != ISD::BUILD_VECTOR)
284 return false;
285
286 for (const SDValue &Op : V->op_values()) {
287 if (Op.isUndef())
288 continue;
290 return false;
291 }
292 return true;
293}
294
295bool ISD::isVectorShrinkable(SDValue V, unsigned NewEltSize, bool Signed) {
296 assert(V.getValueType().isVector() && "Expected a vector!");
297
298 unsigned EltSize = V.getValueType().getScalarSizeInBits();
299 if (EltSize <= NewEltSize)
300 return false;
301
302 if (V.getOpcode() == ISD::ZERO_EXTEND) {
303 return (V.getOperand(0).getValueType().getScalarSizeInBits() <=
304 NewEltSize) &&
305 !Signed;
306 }
307 if (V.getOpcode() == ISD::SIGN_EXTEND) {
308 return (V.getOperand(0).getValueType().getScalarSizeInBits() <=
309 NewEltSize) &&
310 Signed;
311 }
312 if (V.getOpcode() != ISD::BUILD_VECTOR)
313 return false;
314
315 for (const SDValue &Op : V->op_values()) {
316 if (Op.isUndef())
317 continue;
319 return false;
320
321 APInt C = Op->getAsAPIntVal().trunc(EltSize);
322 if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)
323 return false;
324 if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)
325 return false;
326 }
327
328 return true;
329}
330
332 // Return false if the node has no operands.
333 // This is "logically inconsistent" with the definition of "all" but
334 // is probably the desired behavior.
335 if (V.getNumOperands() == 0)
336 return false;
337 return all_of(V->op_values(), [](SDValue Op) { return Op.isUndef(); });
338}
339
341 return V.getOpcode() == ISD::FREEZE && V.getOperand(0).isUndef();
342}
343
344template <typename ConstNodeType>
346 std::function<bool(ConstNodeType *)> Match,
347 bool AllowUndefs, bool AllowTruncation) {
348 // FIXME: Add support for scalar UNDEF cases?
349 if (auto *C = dyn_cast<ConstNodeType>(Op))
350 return Match(C);
351
352 // FIXME: Add support for vector UNDEF cases?
353 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
354 ISD::SPLAT_VECTOR != Op.getOpcode())
355 return false;
356
357 if (ISD::SPLAT_VECTOR == Op.getOpcode() && !DemandedElts)
358 return true;
359
360 EVT SVT = Op.getValueType().getScalarType();
361 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
362 if (ISD::SPLAT_VECTOR != Op.getOpcode() && !DemandedElts[i])
363 continue;
364
365 if (AllowUndefs && Op.getOperand(i).isUndef()) {
366 if (!Match(nullptr))
367 return false;
368 continue;
369 }
370
371 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
373 !Match(Cst))
374 return false;
375 }
376 return true;
377}
378// Build used template types.
380 SDValue, const APInt &, std::function<bool(ConstantSDNode *)>, bool, bool);
382 SDValue, const APInt &, std::function<bool(ConstantFPSDNode *)>, bool,
383 bool);
384
386 SDValue LHS, SDValue RHS, const APInt &DemandedElts,
387 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
388 bool AllowUndefs, bool AllowTypeMismatch) {
389 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
390 return false;
391
392 // TODO: Add support for scalar UNDEF cases?
393 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
394 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
395 return Match(LHSCst, RHSCst);
396
397 // TODO: Add support for vector UNDEF cases?
398 if (LHS.getOpcode() != RHS.getOpcode() ||
399 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
400 LHS.getOpcode() != ISD::SPLAT_VECTOR))
401 return false;
402
403 if (ISD::SPLAT_VECTOR == LHS.getOpcode() && !DemandedElts)
404 return true;
405
406 EVT SVT = LHS.getValueType().getScalarType();
407 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
408 if (ISD::SPLAT_VECTOR != LHS.getOpcode() && !DemandedElts[i])
409 continue;
410 SDValue LHSOp = LHS.getOperand(i);
411 SDValue RHSOp = RHS.getOperand(i);
412 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
413 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
414 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
415 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
416 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
417 return false;
418 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
419 LHSOp.getValueType() != RHSOp.getValueType()))
420 return false;
421 if (!Match(LHSCst, RHSCst))
422 return false;
423 }
424 return true;
425}
426
428 switch (MinMaxOpc) {
429 default:
430 llvm_unreachable("unrecognized opcode");
431 case ISD::UMIN:
432 return ISD::UMAX;
433 case ISD::UMAX:
434 return ISD::UMIN;
435 case ISD::SMIN:
436 return ISD::SMAX;
437 case ISD::SMAX:
438 return ISD::SMIN;
439 }
440}
441
443 switch (MinMaxOpc) {
444 default:
445 llvm_unreachable("unrecognized min/max opcode");
446 case ISD::SMIN:
447 return ISD::UMIN;
448 case ISD::SMAX:
449 return ISD::UMAX;
450 case ISD::UMIN:
451 return ISD::SMIN;
452 case ISD::UMAX:
453 return ISD::SMAX;
454 }
455}
456
458 switch (VecReduceOpcode) {
459 default:
460 llvm_unreachable("Expected VECREDUCE opcode");
463 case ISD::VP_REDUCE_FADD:
464 case ISD::VP_REDUCE_SEQ_FADD:
465 return ISD::FADD;
468 case ISD::VP_REDUCE_FMUL:
469 case ISD::VP_REDUCE_SEQ_FMUL:
470 return ISD::FMUL;
472 case ISD::VP_REDUCE_ADD:
473 return ISD::ADD;
475 case ISD::VP_REDUCE_MUL:
476 return ISD::MUL;
478 case ISD::VP_REDUCE_AND:
479 return ISD::AND;
481 case ISD::VP_REDUCE_OR:
482 return ISD::OR;
484 case ISD::VP_REDUCE_XOR:
485 return ISD::XOR;
487 case ISD::VP_REDUCE_SMAX:
488 return ISD::SMAX;
490 case ISD::VP_REDUCE_SMIN:
491 return ISD::SMIN;
493 case ISD::VP_REDUCE_UMAX:
494 return ISD::UMAX;
496 case ISD::VP_REDUCE_UMIN:
497 return ISD::UMIN;
499 case ISD::VP_REDUCE_FMAX:
500 return ISD::FMAXNUM;
502 case ISD::VP_REDUCE_FMIN:
503 return ISD::FMINNUM;
505 case ISD::VP_REDUCE_FMAXIMUM:
506 return ISD::FMAXIMUM;
508 case ISD::VP_REDUCE_FMINIMUM:
509 return ISD::FMINIMUM;
511 return ISD::FMAXIMUMNUM;
513 return ISD::FMINIMUMNUM;
514 }
515}
516
518 switch (MaskedOpc) {
519 case ISD::MASKED_UDIV:
520 return ISD::UDIV;
521 case ISD::MASKED_SDIV:
522 return ISD::SDIV;
523 case ISD::MASKED_UREM:
524 return ISD::UREM;
525 case ISD::MASKED_SREM:
526 return ISD::SREM;
527 default:
528 llvm_unreachable("Expected masked binop opcode");
529 }
530}
531
532bool ISD::isVPOpcode(unsigned Opcode) {
533 switch (Opcode) {
534 default:
535 return false;
536#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
537 case ISD::VPSD: \
538 return true;
539#include "llvm/IR/VPIntrinsics.def"
540 }
541}
542
543bool ISD::isVPBinaryOp(unsigned Opcode) {
544 switch (Opcode) {
545 default:
546 break;
547#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
548#define VP_PROPERTY_BINARYOP return true;
549#define END_REGISTER_VP_SDNODE(VPSD) break;
550#include "llvm/IR/VPIntrinsics.def"
551 }
552 return false;
553}
554
555bool ISD::isVPReduction(unsigned Opcode) {
556 switch (Opcode) {
557 default:
558 return false;
559 case ISD::VP_REDUCE_ADD:
560 case ISD::VP_REDUCE_MUL:
561 case ISD::VP_REDUCE_AND:
562 case ISD::VP_REDUCE_OR:
563 case ISD::VP_REDUCE_XOR:
564 case ISD::VP_REDUCE_SMAX:
565 case ISD::VP_REDUCE_SMIN:
566 case ISD::VP_REDUCE_UMAX:
567 case ISD::VP_REDUCE_UMIN:
568 case ISD::VP_REDUCE_FMAX:
569 case ISD::VP_REDUCE_FMIN:
570 case ISD::VP_REDUCE_FMAXIMUM:
571 case ISD::VP_REDUCE_FMINIMUM:
572 case ISD::VP_REDUCE_FADD:
573 case ISD::VP_REDUCE_FMUL:
574 case ISD::VP_REDUCE_SEQ_FADD:
575 case ISD::VP_REDUCE_SEQ_FMUL:
576 return true;
577 }
578}
579
580/// The operand position of the vector mask.
581std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
582 switch (Opcode) {
583 default:
584 return std::nullopt;
585#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
586 case ISD::VPSD: \
587 return MASKPOS;
588#include "llvm/IR/VPIntrinsics.def"
589 }
590}
591
592/// The operand position of the explicit vector length parameter.
593std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
594 switch (Opcode) {
595 default:
596 return std::nullopt;
597#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
598 case ISD::VPSD: \
599 return EVLPOS;
600#include "llvm/IR/VPIntrinsics.def"
601 }
602}
603
604std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
605 bool hasFPExcept) {
606 // FIXME: Return strict opcodes in case of fp exceptions.
607 switch (VPOpcode) {
608 default:
609 return std::nullopt;
610#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
611#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
612#define END_REGISTER_VP_SDNODE(VPOPC) break;
613#include "llvm/IR/VPIntrinsics.def"
614 }
615 return std::nullopt;
616}
617
618std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
619 switch (Opcode) {
620 default:
621 return std::nullopt;
622#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
623#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
624#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
625#include "llvm/IR/VPIntrinsics.def"
626 }
627}
628
630 switch (ExtType) {
631 case ISD::EXTLOAD:
632 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
633 case ISD::SEXTLOAD:
634 return ISD::SIGN_EXTEND;
635 case ISD::ZEXTLOAD:
636 return ISD::ZERO_EXTEND;
637 default:
638 break;
639 }
640
641 llvm_unreachable("Invalid LoadExtType");
642}
643
645 // To perform this operation, we just need to swap the L and G bits of the
646 // operation.
647 unsigned OldL = (Operation >> 2) & 1;
648 unsigned OldG = (Operation >> 1) & 1;
649 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
650 (OldL << 1) | // New G bit
651 (OldG << 2)); // New L bit.
652}
653
655 unsigned Operation = Op;
656 if (isIntegerLike)
657 Operation ^= 7; // Flip L, G, E bits, but not U.
658 else
659 Operation ^= 15; // Flip all of the condition bits.
660
662 Operation &= ~8; // Don't let N and U bits get set.
663
664 return ISD::CondCode(Operation);
665}
666
670
672 bool isIntegerLike) {
673 return getSetCCInverseImpl(Op, isIntegerLike);
674}
675
676/// For an integer comparison, return 1 if the comparison is a signed operation
677/// and 2 if the result is an unsigned comparison. Return zero if the operation
678/// does not depend on the sign of the input (setne and seteq).
679static int isSignedOp(ISD::CondCode Opcode) {
680 switch (Opcode) {
681 default: llvm_unreachable("Illegal integer setcc operation!");
682 case ISD::SETEQ:
683 case ISD::SETNE: return 0;
684 case ISD::SETLT:
685 case ISD::SETLE:
686 case ISD::SETGT:
687 case ISD::SETGE: return 1;
688 case ISD::SETULT:
689 case ISD::SETULE:
690 case ISD::SETUGT:
691 case ISD::SETUGE: return 2;
692 }
693}
694
696 EVT Type) {
697 bool IsInteger = Type.isInteger();
698 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
699 // Cannot fold a signed integer setcc with an unsigned integer setcc.
700 return ISD::SETCC_INVALID;
701
702 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
703
704 // If the N and U bits get set, then the resultant comparison DOES suddenly
705 // care about orderedness, and it is true when ordered.
706 if (Op > ISD::SETTRUE2)
707 Op &= ~16; // Clear the U bit if the N bit is set.
708
709 // Canonicalize illegal integer setcc's.
710 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
711 Op = ISD::SETNE;
712
713 return ISD::CondCode(Op);
714}
715
717 EVT Type) {
718 bool IsInteger = Type.isInteger();
719 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
720 // Cannot fold a signed setcc with an unsigned setcc.
721 return ISD::SETCC_INVALID;
722
723 // Combine all of the condition bits.
724 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
725
726 // Canonicalize illegal integer setcc's.
727 if (IsInteger) {
728 switch (Result) {
729 default: break;
730 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
731 case ISD::SETOEQ: // SETEQ & SETU[LG]E
732 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
733 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
734 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
735 }
736 }
737
738 return Result;
739}
740
741//===----------------------------------------------------------------------===//
742// SDNode Key Support
743//===----------------------------------------------------------------------===//
744
745/// If this is an SDNode with special info, add this info to the NodeID data.
746/// MorphNodeTo passes the opcode \p N is becoming, so \p Opc may differ from
747/// N->getOpcode() and a case may only cast to a class \p N already is.
748static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N,
749 unsigned Opc) {
750 // Machine nodes are uniqued by opcode, value types and operands alone, as in
751 // getMachineNode; MorphNodeTo's clearMemRefs overwrites the fields below.
753 return;
754
755 switch (Opc) {
758 case ISD::MCSymbol:
759 llvm_unreachable("Should only be used on nodes with operands");
760 default: break; // Normal nodes don't need extra info.
762 case ISD::Constant: {
764 ID.AddPointer(C->getConstantIntValue());
765 ID.AddBoolean(C->isOpaque());
766 break;
767 }
769 case ISD::ConstantFP:
770 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
771 break;
777 ID.AddPointer(GA->getGlobal());
778 ID.AddInteger(GA->getOffset());
779 ID.AddInteger(GA->getTargetFlags());
780 break;
781 }
782 case ISD::BasicBlock:
783 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
784 break;
785 case ISD::EH_LABEL:
787 ID.AddPointer(cast<LabelSDNode>(N)->getLabel());
788 break;
790 ID.AddPointer(cast<DeactivationSymbolSDNode>(N)->getGlobal());
791 break;
792 case ISD::Register:
793 ID.AddInteger(cast<RegisterSDNode>(N)->getReg().id());
794 break;
796 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
797 break;
798 case ISD::SRCVALUE:
799 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
800 break;
801 case ISD::FrameIndex:
803 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
804 break;
806 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
807 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
808 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
809 break;
810 case ISD::JumpTable:
812 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
813 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
814 break;
818 ID.AddInteger(CP->getAlign().value());
819 ID.AddInteger(CP->getOffset());
822 else
823 ID.AddPointer(CP->getConstVal());
824 ID.AddInteger(CP->getTargetFlags());
825 break;
826 }
827 case ISD::TargetIndex: {
829 ID.AddInteger(TI->getIndex());
830 ID.AddInteger(TI->getOffset());
831 ID.AddInteger(TI->getTargetFlags());
832 break;
833 }
834 case ISD::LOAD: {
835 const LoadSDNode *LD = cast<LoadSDNode>(N);
836 ID.AddInteger(LD->getMemoryVT().getRawBits());
837 ID.AddInteger(LD->getRawSubclassData());
838 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
839 ID.AddInteger(LD->getMemOperand()->getFlags());
840 break;
841 }
842 case ISD::STORE: {
843 const StoreSDNode *ST = cast<StoreSDNode>(N);
844 ID.AddInteger(ST->getMemoryVT().getRawBits());
845 ID.AddInteger(ST->getRawSubclassData());
846 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
847 ID.AddInteger(ST->getMemOperand()->getFlags());
848 break;
849 }
850 case ISD::VP_LOAD: {
851 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
852 ID.AddInteger(ELD->getMemoryVT().getRawBits());
853 ID.AddInteger(ELD->getRawSubclassData());
854 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
855 ID.AddInteger(ELD->getMemOperand()->getFlags());
856 break;
857 }
858 case ISD::VP_LOAD_FF: {
859 const auto *LD = cast<VPLoadFFSDNode>(N);
860 ID.AddInteger(LD->getMemoryVT().getRawBits());
861 ID.AddInteger(LD->getRawSubclassData());
862 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
863 ID.AddInteger(LD->getMemOperand()->getFlags());
864 break;
865 }
866 case ISD::VP_STORE: {
867 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
868 ID.AddInteger(EST->getMemoryVT().getRawBits());
869 ID.AddInteger(EST->getRawSubclassData());
870 ID.AddInteger(EST->getPointerInfo().getAddrSpace());
871 ID.AddInteger(EST->getMemOperand()->getFlags());
872 break;
873 }
874 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
876 ID.AddInteger(SLD->getMemoryVT().getRawBits());
877 ID.AddInteger(SLD->getRawSubclassData());
878 ID.AddInteger(SLD->getPointerInfo().getAddrSpace());
879 break;
880 }
881 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
883 ID.AddInteger(SST->getMemoryVT().getRawBits());
884 ID.AddInteger(SST->getRawSubclassData());
885 ID.AddInteger(SST->getPointerInfo().getAddrSpace());
886 break;
887 }
888 case ISD::VP_GATHER: {
890 ID.AddInteger(EG->getMemoryVT().getRawBits());
891 ID.AddInteger(EG->getRawSubclassData());
892 ID.AddInteger(EG->getPointerInfo().getAddrSpace());
893 ID.AddInteger(EG->getMemOperand()->getFlags());
894 break;
895 }
896 case ISD::VP_SCATTER: {
898 ID.AddInteger(ES->getMemoryVT().getRawBits());
899 ID.AddInteger(ES->getRawSubclassData());
900 ID.AddInteger(ES->getPointerInfo().getAddrSpace());
901 ID.AddInteger(ES->getMemOperand()->getFlags());
902 break;
903 }
904 case ISD::MLOAD: {
906 ID.AddInteger(MLD->getMemoryVT().getRawBits());
907 ID.AddInteger(MLD->getRawSubclassData());
908 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
909 ID.AddInteger(MLD->getMemOperand()->getFlags());
910 break;
911 }
912 case ISD::MSTORE: {
914 ID.AddInteger(MST->getMemoryVT().getRawBits());
915 ID.AddInteger(MST->getRawSubclassData());
916 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
917 ID.AddInteger(MST->getMemOperand()->getFlags());
918 break;
919 }
920 case ISD::MGATHER: {
922 ID.AddInteger(MG->getMemoryVT().getRawBits());
923 ID.AddInteger(MG->getRawSubclassData());
924 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
925 ID.AddInteger(MG->getMemOperand()->getFlags());
926 break;
927 }
928 case ISD::MSCATTER:
932 const MemSDNode *M = cast<MemSDNode>(N);
933 ID.AddInteger(M->getMemoryVT().getRawBits());
934 ID.AddInteger(M->getRawSubclassData());
935 ID.AddInteger(M->getPointerInfo().getAddrSpace());
936 ID.AddInteger(M->getMemOperand()->getFlags());
937 break;
938 }
939 case ISD::VECTOR_SHUFFLE: {
940 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
941 for (int M : Mask)
942 ID.AddInteger(M);
943 break;
944 }
945 case ISD::ADDRSPACECAST: {
947 ID.AddInteger(ASC->getSrcAddressSpace());
948 ID.AddInteger(ASC->getDestAddressSpace());
949 break;
950 }
952 case ISD::BlockAddress: {
954 ID.AddPointer(BA->getBlockAddress());
955 ID.AddInteger(BA->getOffset());
956 ID.AddInteger(BA->getTargetFlags());
957 break;
958 }
959 case ISD::AssertAlign:
960 ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());
961 break;
962 case ISD::PREFETCH:
965 // Handled by MemIntrinsicSDNode check after the switch.
966 break;
968 ID.AddPointer(cast<MDNodeSDNode>(N)->getMD());
969 break;
970 } // end switch (Opc)
971
972 if (auto *AT = dyn_cast<AtomicSDNode>(N)) {
973 ID.AddInteger(AT->getMemoryVT().getRawBits());
974 ID.AddInteger(AT->getRawSubclassData());
975 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
976 ID.AddInteger(AT->getMemOperand()->getFlags());
977 }
978
979 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
980 // to check.
981 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {
982 ID.AddInteger(MN->getRawSubclassData());
983 ID.AddInteger(MN->getMemoryVT().getRawBits());
984 for (const MachineMemOperand *MMO : MN->memoperands()) {
985 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
986 ID.AddInteger(MMO->getFlags());
987 }
988 }
989}
990
992 : Opcode(N.getOpcode()), VTs(N.getVTList().VTs),
993 OpStorage(N.op_begin(), N.op_end()) {
994 Ops = OpStorage;
995 AddNodeIDCustom(Tail, &N, N.getOpcode());
996}
997
998#ifndef NDEBUG
999/// Serialize a node the way SDNodes were uniqued before SDNodeKey, to
1000/// cross-check the typed comparison against. Deliberately reads the node
1001/// rather than going through SDNodeKey, so the key constructor is checked too
1002/// and not used as its own oracle.
1003static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1004 ID.AddInteger(N->getOpcode());
1005 ID.AddPointer(N->getVTList().VTs);
1006 for (const SDUse &Op : N->ops()) {
1007 ID.AddPointer(Op.getNode());
1008 ID.AddInteger(Op.getResNo());
1009 }
1010 AddNodeIDCustom(ID, N, N->getOpcode());
1011}
1012
1014 ID.AddInteger(Key.Opcode);
1015 ID.AddPointer(Key.VTs);
1016 for (const SDValue &Op : Key.Ops) {
1017 ID.AddPointer(Op.getNode());
1018 ID.AddInteger(Op.getResNo());
1019 }
1020 ID.AddNodeID(Key.Tail);
1021}
1022#endif
1023
1024static bool keyMatches(const SDNodeKey &Key, const SDNode &N) {
1025 if (N.getOpcode() != Key.Opcode || N.getVTList().VTs != Key.VTs)
1026 return false;
1027 if (!llvm::equal(N.ops(), Key.Ops))
1028 return false;
1029 // Build only what AddNodeIDCustom would have added, which is nothing for
1030 // most opcodes, rather than the whole profile.
1031 FoldingSetNodeID Tail;
1032 AddNodeIDCustom(Tail, &N, N.getOpcode());
1033 return Tail == Key.Tail;
1034}
1035
1037 bool Result = keyMatches(Key, N);
1038
1039#ifndef NDEBUG
1040 FoldingSetNodeID NodeProfile, KeyProfile;
1041 AddNodeIDNode(NodeProfile, &N);
1042 AddNodeIDNode(KeyProfile, Key);
1043 assert((NodeProfile == KeyProfile) == Result &&
1044 "SDNodeKey equality disagrees with profile");
1045#endif
1046 return Result;
1047}
1048
1049//===----------------------------------------------------------------------===//
1050// SelectionDAG Class
1051//===----------------------------------------------------------------------===//
1052
1053/// doNotCSE - Return true if CSE should not be performed for this node.
1054static bool doNotCSE(SDNode *N) {
1055 if (N->getValueType(0) == MVT::Glue)
1056 return true; // Never CSE anything that produces a glue result.
1057
1058 switch (N->getOpcode()) {
1059 default: break;
1060 case ISD::HANDLENODE:
1061 case ISD::EH_LABEL:
1062 return true; // Never CSE these nodes.
1063 }
1064
1065 // Check that remaining values produced are not flags.
1066 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1067 if (N->getValueType(i) == MVT::Glue)
1068 return true; // Never CSE anything that produces a glue result.
1069
1070 return false;
1071}
1072
1073/// Construct a DemandedElts mask which demands all elements of \p V.
1074/// If \p V is not a fixed-length vector, then this will return a single bit.
1076 EVT VT = V.getValueType();
1077 // Since the number of lanes in a scalable vector is unknown at compile time,
1078 // we track one bit which is implicitly broadcast to all lanes. This means
1079 // that all lanes in a scalable vector are considered demanded.
1081 : APInt(1, 1);
1082}
1083
1084/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1085/// SelectionDAG.
1087 // Create a dummy node (which is not added to allnodes), that adds a reference
1088 // to the root node, preventing it from being deleted.
1089 HandleSDNode Dummy(getRoot());
1090
1091 SmallVector<SDNode*, 128> DeadNodes;
1092
1093 // Add all obviously-dead nodes to the DeadNodes worklist.
1094 for (SDNode &Node : allnodes())
1095 if (Node.use_empty())
1096 DeadNodes.push_back(&Node);
1097
1098 RemoveDeadNodes(DeadNodes);
1099
1100 // If the root changed (e.g. it was a dead load, update the root).
1101 setRoot(Dummy.getValue());
1102}
1103
1104/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1105/// given list, and any nodes that become unreachable as a result.
1107
1108 // Process the worklist, deleting the nodes and adding their uses to the
1109 // worklist.
1110 while (!DeadNodes.empty()) {
1111 SDNode *N = DeadNodes.pop_back_val();
1112 // Skip to next node if we've already managed to delete the node. This could
1113 // happen if replacing a node causes a node previously added to the node to
1114 // be deleted.
1115 if (N->getOpcode() == ISD::DELETED_NODE)
1116 continue;
1117
1118 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1119 DUL->NodeDeleted(N, nullptr);
1120
1121 // Take the node out of the appropriate CSE map.
1122 RemoveNodeFromCSEMaps(N);
1123
1124 // Next, brutally remove the operand list. This is safe to do, as there are
1125 // no cycles in the graph.
1126 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1127 SDUse &Use = *I++;
1128 SDNode *Operand = Use.getNode();
1129 Use.set(SDValue());
1130
1131 // Now that we removed this operand, see if there are no uses of it left.
1132 if (Operand->use_empty())
1133 DeadNodes.push_back(Operand);
1134 }
1135
1136 DeallocateNode(N);
1137 }
1138}
1139
1141 SmallVector<SDNode*, 16> DeadNodes(1, N);
1142
1143 // Create a dummy node that adds a reference to the root node, preventing
1144 // it from being deleted. (This matters if the root is an operand of the
1145 // dead node.)
1146 HandleSDNode Dummy(getRoot());
1147
1148 RemoveDeadNodes(DeadNodes);
1149}
1150
1152 // First take this out of the appropriate CSE map.
1153 RemoveNodeFromCSEMaps(N);
1154
1155 // Finally, remove uses due to operands of this node, remove from the
1156 // AllNodes list, and delete the node.
1157 DeleteNodeNotInCSEMaps(N);
1158}
1159
1160void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1161 assert(N->getIterator() != AllNodes.begin() &&
1162 "Cannot delete the entry node!");
1163 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1164
1165 // Drop all of the operands and decrement used node's use counts.
1166 N->DropOperands();
1167
1168 DeallocateNode(N);
1169}
1170
1171void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1172 assert(!(V->isVariadic() && isParameter));
1173 if (isParameter)
1174 ByvalParmDbgValues.push_back(V);
1175 else
1176 DbgValues.push_back(V);
1177 for (const SDNode *Node : V->getSDNodes())
1178 if (Node)
1179 DbgValMap[Node].push_back(V);
1180}
1181
1183 DbgValMapType::iterator I = DbgValMap.find(Node);
1184 if (I == DbgValMap.end())
1185 return;
1186 for (auto &Val: I->second)
1187 Val->setIsInvalidated();
1188 DbgValMap.erase(I);
1189}
1190
1191void SelectionDAG::DeallocateNode(SDNode *N) {
1192 // If we have operands, deallocate them.
1194
1195 NodeAllocator.Deallocate(AllNodes.remove(N));
1196
1197 // Set the opcode to DELETED_NODE to help catch bugs when node
1198 // memory is reallocated.
1199 // FIXME: There are places in SDag that have grown a dependency on the opcode
1200 // value in the released node.
1201 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1202 N->NodeType = ISD::DELETED_NODE;
1203
1204 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1205 // them and forget about that node.
1206 DbgInfo->erase(N);
1207
1208 // Invalidate extra info.
1209 SDEI.erase(N);
1210}
1211
1212#ifndef NDEBUG
1213/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1214void SelectionDAG::verifyNode(SDNode *N) const {
1215 switch (N->getOpcode()) {
1216 default:
1217 if (N->isTargetOpcode())
1219 break;
1220 case ISD::BUILD_PAIR: {
1221 EVT VT = N->getValueType(0);
1222 assert(N->getNumValues() == 1 && "Too many results!");
1223 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1224 "Wrong return type!");
1225 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1226 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1227 "Mismatched operand types!");
1228 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1229 "Wrong operand type!");
1230 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1231 "Wrong return type size");
1232 break;
1233 }
1234 case ISD::BUILD_VECTOR: {
1235 assert(N->getNumValues() == 1 && "Too many results!");
1236 assert(N->getValueType(0).isVector() && "Wrong return type!");
1237 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1238 "Wrong number of operands!");
1239 EVT EltVT = N->getValueType(0).getVectorElementType();
1240 for (const SDUse &Op : N->ops()) {
1241 assert((Op.getValueType() == EltVT ||
1242 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1243 EltVT.bitsLE(Op.getValueType()))) &&
1244 "Wrong operand type!");
1245 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1246 "Operands must all have the same type");
1247 }
1248 break;
1249 }
1250 case ISD::SADDO:
1251 case ISD::UADDO:
1252 case ISD::SSUBO:
1253 case ISD::USUBO:
1254 assert(N->getNumValues() == 2 && "Wrong number of results!");
1255 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1256 "Invalid add/sub overflow op!");
1257 assert(N->getVTList().VTs[0].isInteger() &&
1258 N->getVTList().VTs[1].isInteger() &&
1259 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1260 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1261 "Binary operator types must match!");
1262 break;
1263 }
1264}
1265#endif // NDEBUG
1266
1267/// Insert a newly allocated node into the DAG.
1268///
1269/// Handles insertion into the all nodes list and CSE map, as well as
1270/// verification and other common operations when a new node is allocated.
1271void SelectionDAG::InsertNode(SDNode *N) {
1272 AllNodes.push_back(N);
1273#ifndef NDEBUG
1274 N->PersistentId = NextPersistentId++;
1275 verifyNode(N);
1276#endif
1277 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1278 DUL->NodeInserted(N);
1279}
1280
1281/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1282/// correspond to it. This is useful when we're about to delete or repurpose
1283/// the node. We don't want future request for structurally identical nodes
1284/// to return N anymore.
1285bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1286 bool Erased = false;
1287 switch (N->getOpcode()) {
1288 case ISD::HANDLENODE: return false; // noop.
1289 case ISD::CONDCODE:
1290 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1291 "Cond code doesn't exist!");
1292 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1293 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1294 break;
1296 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1297 break;
1299 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1300 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1301 ESN->getSymbol(), ESN->getTargetFlags()));
1302 break;
1303 }
1304 case ISD::MCSymbol: {
1305 auto *MCSN = cast<MCSymbolSDNode>(N);
1306 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1307 break;
1308 }
1309 case ISD::VALUETYPE: {
1310 EVT VT = cast<VTSDNode>(N)->getVT();
1311 if (VT.isExtended()) {
1312 Erased = ExtendedValueTypeNodes.erase(VT);
1313 } else {
1314 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1315 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1316 }
1317 break;
1318 }
1319 default:
1320 // Remove it from the CSE Map.
1321 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1322 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1323 Erased = CSEMap.erase(N);
1324 break;
1325 }
1326#ifndef NDEBUG
1327 // Verify that the node was actually in one of the CSE maps, unless it has a
1328 // glue result (which cannot be CSE'd) or is one of the special cases that are
1329 // not subject to CSE.
1330 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1331 !N->isMachineOpcode() && !doNotCSE(N)) {
1332 N->dump(this);
1333 dbgs() << "\n";
1334 llvm_unreachable("Node is not in map!");
1335 }
1336#endif
1337 return Erased;
1338}
1339
1340/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1341/// maps and modified in place. Add it back to the CSE maps, unless an identical
1342/// node already exists, in which case transfer all its users to the existing
1343/// node. This transfer can potentially trigger recursive merging.
1344void
1345SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1346 // For node types that aren't CSE'd, just act as if no identical node
1347 // already exists.
1348 if (!doNotCSE(N)) {
1349 SDNode *Existing = CSEMap.getOrInsert(N);
1350 if (Existing != N) {
1351 // If there was already an existing matching node, use ReplaceAllUsesWith
1352 // to replace the dead one with the existing one. This can cause
1353 // recursive merging of other unrelated nodes down the line.
1354 Existing->intersectFlagsWith(N->getFlags());
1355 if (auto *MemNode = dyn_cast<MemSDNode>(Existing)) {
1357 cast<MemSDNode>(N)->memoperands();
1358 // Range and cache hint metadata are not part of the DAG CSE key because
1359 // we prefer to CSE even when metadata does not match. Merge potentially
1360 // differing metadata conservatively.
1361 MemNode->refineMMOMetadata(NewMMOs);
1362 }
1363 ReplaceAllUsesWith(N, Existing);
1364
1365 // N is now dead. Inform the listeners and delete it.
1366 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1367 DUL->NodeDeleted(N, Existing);
1368 DeleteNodeNotInCSEMaps(N);
1369 return;
1370 }
1371 }
1372
1373 // If the node doesn't already exist, we updated it. Inform listeners.
1374 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1375 DUL->NodeUpdated(N);
1376}
1377
1378/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1379/// were replaced with those specified. If this node is never memoized,
1380/// return null, otherwise return a pointer to the slot it would take. If a
1381/// node already exists with these operands, the slot will be non-null.
1382SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1383 FoldingSetInsertToken &InsertToken) {
1384 if (doNotCSE(N))
1385 return nullptr;
1386
1387 SDValue Ops[] = { Op };
1388 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1389 AddNodeIDCustom(ID.Tail, N, N->getOpcode());
1390 SDNode *Node = lookupNode(ID, SDLoc(N), InsertToken);
1391 if (Node)
1392 Node->intersectFlagsWith(N->getFlags());
1393 return Node;
1394}
1395
1396/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1397/// were replaced with those specified. If this node is never memoized,
1398/// return null, otherwise return a pointer to the slot it would take. If a
1399/// node already exists with these operands, the slot will be non-null.
1400SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1401 FoldingSetInsertToken &InsertToken) {
1402 if (doNotCSE(N))
1403 return nullptr;
1404
1405 SDValue Ops[] = { Op1, Op2 };
1406 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1407 AddNodeIDCustom(ID.Tail, N, N->getOpcode());
1408 SDNode *Node = lookupNode(ID, SDLoc(N), InsertToken);
1409 if (Node)
1410 Node->intersectFlagsWith(N->getFlags());
1411 return Node;
1412}
1413
1414/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1415/// were replaced with those specified. If this node is never memoized,
1416/// return null, otherwise return a pointer to the slot it would take. If a
1417/// node already exists with these operands, the slot will be non-null.
1418SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1419 FoldingSetInsertToken &InsertToken) {
1420 if (doNotCSE(N))
1421 return nullptr;
1422
1423 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1424 AddNodeIDCustom(ID.Tail, N, N->getOpcode());
1425 SDNode *Node = lookupNode(ID, SDLoc(N), InsertToken);
1426 if (Node)
1427 Node->intersectFlagsWith(N->getFlags());
1428 return Node;
1429}
1430
1432 Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)
1433 : VT.getTypeForEVT(*getContext());
1434
1435 return getDataLayout().getABITypeAlign(Ty);
1436}
1437
1438// EntryNode could meaningfully have debug info if we can find it...
1440 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1441 getVTList(MVT::Other, MVT::Glue)),
1442 Root(getEntryNode()) {
1443 InsertNode(&EntryNode);
1444 DbgInfo = new SDDbgInfo();
1445}
1446
1448 const TargetLibraryInfo *LibraryInfo,
1449 const LibcallLoweringInfo *LibcallsInfo,
1450 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1451 BlockFrequencyInfo *BFIin,
1452 FunctionVarLocs const *VarLocs) {
1453 MF = &NewMF;
1456 LibInfo = LibraryInfo;
1457 Libcalls = LibcallsInfo;
1458 Context = &MF->getFunction().getContext();
1459 DAGOpts = &Context->getLibraryOptions().get<SelectionDAGOptions>();
1460 UA = NewUA;
1461 PSI = PSIin;
1462 BFI = BFIin;
1463 FnVarLocs = VarLocs;
1464}
1465
1467 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1468 allnodes_clear();
1469 OperandRecycler.clear(OperandAllocator);
1470 delete DbgInfo;
1471}
1472
1474 return llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1475}
1476
1478 return FLI->MBB->getBasicBlock();
1479}
1480
1481void SelectionDAG::allnodes_clear() {
1482 assert(&*AllNodes.begin() == &EntryNode);
1483 AllNodes.remove(AllNodes.begin());
1484 while (!AllNodes.empty())
1485 DeallocateNode(&AllNodes.front());
1486#ifndef NDEBUG
1487 NextPersistentId = 0;
1488#endif
1489}
1490
1491SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key,
1492 FoldingSetInsertToken &InsertToken) {
1493 SDNode *N = CSEMap.lookup(Key, InsertToken);
1494 if (N) {
1495 switch (N->getOpcode()) {
1496 default: break;
1497 case ISD::Constant:
1498 case ISD::ConstantFP:
1499 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1500 "debug location. Use another overload.");
1501 }
1502 }
1503 return N;
1504}
1505
1506SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key, const SDLoc &DL,
1507 FoldingSetInsertToken &InsertToken) {
1508 SDNode *N = CSEMap.lookup(Key, InsertToken);
1509 if (N) {
1510 switch (N->getOpcode()) {
1511 case ISD::Constant:
1512 case ISD::ConstantFP:
1513 // Erase debug location from the node if the node is used at several
1514 // different places. Do not propagate one location to all uses as it
1515 // will cause a worse single stepping debugging experience.
1516 if (N->getDebugLoc() != DL.getDebugLoc())
1517 N->setDebugLoc(DebugLoc());
1518 break;
1519 default:
1520 // When the node's point of use is located earlier in the instruction
1521 // sequence than its prior point of use, update its debug info to the
1522 // earlier location.
1523 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1524 N->setDebugLoc(DL.getDebugLoc());
1525 break;
1526 }
1527 }
1528 return N;
1529}
1530
1532 allnodes_clear();
1533 OperandRecycler.clear(OperandAllocator);
1534 OperandAllocator.Reset();
1535 CSEMap.clear();
1536
1537 ExtendedValueTypeNodes.clear();
1538 ExternalSymbols.clear();
1539 TargetExternalSymbols.clear();
1540 MCSymbols.clear();
1541 SDEI.clear();
1542 llvm::fill(CondCodeNodes, nullptr);
1543 llvm::fill(ValueTypeNodes, nullptr);
1544
1545 EntryNode.UseList = nullptr;
1546 InsertNode(&EntryNode);
1547 Root = getEntryNode();
1548 DbgInfo->clear();
1549}
1550
1552 return VT.bitsGT(Op.getValueType())
1553 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1554 : getNode(ISD::FP_ROUND, DL, VT, Op,
1555 getIntPtrConstant(0, DL, /*isTarget=*/true));
1556}
1557
1558std::pair<SDValue, SDValue>
1560 const SDLoc &DL, EVT VT) {
1561 assert(!VT.bitsEq(Op.getValueType()) &&
1562 "Strict no-op FP extend/round not allowed.");
1563 SDValue Res =
1564 VT.bitsGT(Op.getValueType())
1565 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1566 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1567 {Chain, Op, getIntPtrConstant(0, DL, /*isTarget=*/true)});
1568
1569 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1570}
1571
1573 return VT.bitsGT(Op.getValueType()) ?
1574 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1575 getNode(ISD::TRUNCATE, DL, VT, Op);
1576}
1577
1579 return VT.bitsGT(Op.getValueType()) ?
1580 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1581 getNode(ISD::TRUNCATE, DL, VT, Op);
1582}
1583
1585 return VT.bitsGT(Op.getValueType()) ?
1586 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1587 getNode(ISD::TRUNCATE, DL, VT, Op);
1588}
1589
1591 EVT VT) {
1592 assert(!VT.isVector());
1593 auto Type = Op.getValueType();
1594 SDValue DestOp;
1595 if (Type == VT)
1596 return Op;
1597 auto Size = Op.getValueSizeInBits();
1598 DestOp = getBitcast(EVT::getIntegerVT(*Context, Size), Op);
1599 if (DestOp.getValueType() == VT)
1600 return DestOp;
1601
1602 return getAnyExtOrTrunc(DestOp, DL, VT);
1603}
1604
1606 EVT OpVT) {
1607 if (VT.bitsLE(Op.getValueType()))
1608 return getNode(ISD::TRUNCATE, SL, VT, Op);
1609
1610 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1611 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1612}
1613
1615 EVT OpVT = Op.getValueType();
1616 assert(VT.isInteger() && OpVT.isInteger() &&
1617 "Cannot getZeroExtendInReg FP types");
1618 assert(VT.isVector() == OpVT.isVector() &&
1619 "getZeroExtendInReg type should be vector iff the operand "
1620 "type is vector!");
1621 assert((!VT.isVector() ||
1623 "Vector element counts must match in getZeroExtendInReg");
1624 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1625 if (OpVT == VT)
1626 return Op;
1627 // TODO: Use computeKnownBits instead of AssertZext.
1628 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Op.getOperand(1))
1629 ->getVT()
1630 .getScalarType()
1631 .bitsLE(VT.getScalarType()))
1632 return Op;
1634 VT.getScalarSizeInBits());
1635 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1636}
1637
1639 // Only unsigned pointer semantics are supported right now. In the future this
1640 // might delegate to TLI to check pointer signedness.
1641 return getZExtOrTrunc(Op, DL, VT);
1642}
1643
1645 // Only unsigned pointer semantics are supported right now. In the future this
1646 // might delegate to TLI to check pointer signedness.
1647 return getZeroExtendInReg(Op, DL, VT);
1648}
1649
1651 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);
1652}
1653
1654/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1656 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1657}
1658
1660 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1661 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1662}
1663
1665 EVT OpVT) {
1666 if (!V)
1667 return getConstant(0, DL, VT);
1668
1669 switch (TLI->getBooleanContents(OpVT)) {
1672 return getConstant(1, DL, VT);
1674 return getAllOnesConstant(DL, VT);
1675 }
1676 llvm_unreachable("Unexpected boolean content enum!");
1677}
1678
1680 bool isT, bool isO) {
1681 return getConstant(APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1682 DL, VT, isT, isO);
1683}
1684
1686 bool isT, bool isO) {
1687 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1688}
1689
1691 EVT VT, bool isT, bool isO) {
1692 assert(VT.isInteger() && "Cannot create FP integer constant!");
1693
1694 EVT EltVT = VT.getScalarType();
1695 const ConstantInt *Elt = &Val;
1696
1697 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1698 // to-be-splatted scalar ConstantInt.
1699 if (isa<VectorType>(Elt->getType()))
1700 Elt = ConstantInt::get(*getContext(), Elt->getValue());
1701
1702 // In some cases the vector type is legal but the element type is illegal and
1703 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1704 // inserted value (the type does not need to match the vector element type).
1705 // Any extra bits introduced will be truncated away.
1706 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1708 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1709 APInt NewVal;
1710 if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))
1711 NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());
1712 else
1713 NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1714 Elt = ConstantInt::get(*getContext(), NewVal);
1715 }
1716 // In other cases the element type is illegal and needs to be expanded, for
1717 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1718 // the value into n parts and use a vector type with n-times the elements.
1719 // Then bitcast to the type requested.
1720 // Legalizing constants too early makes the DAGCombiner's job harder so we
1721 // only legalize if the DAG tells us we must produce legal types.
1722 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1723 TLI->getTypeAction(*getContext(), EltVT) ==
1725 const APInt &NewVal = Elt->getValue();
1726 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1727 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1728
1729 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1730 if (VT.isScalableVector() ||
1731 TLI->isOperationLegal(ISD::SPLAT_VECTOR, VT)) {
1732 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1733 "Can only handle an even split!");
1734 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1735
1736 SmallVector<SDValue, 2> ScalarParts;
1737 for (unsigned i = 0; i != Parts; ++i)
1738 ScalarParts.push_back(getConstant(
1739 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1740 ViaEltVT, isT, isO));
1741
1742 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1743 }
1744
1745 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1746 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1747
1748 // Check the temporary vector is the correct size. If this fails then
1749 // getTypeToTransformTo() probably returned a type whose size (in bits)
1750 // isn't a power-of-2 factor of the requested type size.
1751 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1752
1753 SmallVector<SDValue, 2> EltParts;
1754 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1755 EltParts.push_back(getConstant(
1756 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1757 ViaEltVT, isT, isO));
1758
1759 // EltParts is currently in little endian order. If we actually want
1760 // big-endian order then reverse it now.
1761 if (getDataLayout().isBigEndian())
1762 std::reverse(EltParts.begin(), EltParts.end());
1763
1764 // The elements must be reversed when the element order is different
1765 // to the endianness of the elements (because the BITCAST is itself a
1766 // vector shuffle in this situation). However, we do not need any code to
1767 // perform this reversal because getConstant() is producing a vector
1768 // splat.
1769 // This situation occurs in MIPS MSA.
1770
1772 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1773 llvm::append_range(Ops, EltParts);
1774
1775 SDValue V =
1776 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1777 return V;
1778 }
1779
1780 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1781 "APInt size does not match type size!");
1782 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1783 SDVTList VTs = getVTList(EltVT);
1784 SDNodeKey ID(Opc, VTs, {});
1785 ID.AddPointer(Elt);
1786 ID.AddBoolean(isO);
1787 FoldingSetInsertToken InsertToken;
1788 SDNode *N = nullptr;
1789 if ((N = lookupNode(ID, DL, InsertToken)))
1790 if (!VT.isVector())
1791 return SDValue(N, 0);
1792
1793 if (!N) {
1794 N = newSDNode<ConstantSDNode>(isT, isO, Elt, VTs);
1795 if (!isT)
1796 N->setDebugLoc(DL.getDebugLoc());
1797 CSEMap.insert(N, InsertToken);
1798 InsertNode(N);
1799 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1800 }
1801
1802 SDValue Result(N, 0);
1803 if (VT.isVector())
1804 Result = getSplat(VT, DL, Result);
1805 return Result;
1806}
1807
1809 bool isT, bool isO) {
1810 unsigned Size = VT.getScalarSizeInBits();
1811 return getConstant(APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1812}
1813
1815 bool IsOpaque) {
1817 IsTarget, IsOpaque);
1818}
1819
1821 bool isTarget) {
1822 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1823}
1824
1826 const SDLoc &DL) {
1827 assert(VT.isInteger() && "Shift amount is not an integer type!");
1828 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout());
1829 return getConstant(Val, DL, ShiftVT);
1830}
1831
1833 const SDLoc &DL) {
1834 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1835 return getShiftAmountConstant(Val.getZExtValue(), VT, DL);
1836}
1837
1839 bool isTarget) {
1840 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1841}
1842
1844 bool isTarget) {
1845 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1846}
1847
1849 EVT VT, bool isTarget) {
1850 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1851
1852 EVT EltVT = VT.getScalarType();
1853 const ConstantFP *Elt = &V;
1854
1855 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1856 // the to-be-splatted scalar ConstantFP.
1857 if (isa<VectorType>(Elt->getType()))
1858 Elt = ConstantFP::get(*getContext(), Elt->getValue());
1859
1860 // Do the map lookup using the actual bit pattern for the floating point
1861 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1862 // we don't have issues with SNANs.
1863 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1864 SDVTList VTs = getVTList(EltVT);
1865 SDNodeKey ID(Opc, VTs, {});
1866 ID.AddPointer(Elt);
1867 FoldingSetInsertToken InsertToken;
1868 SDNode *N = nullptr;
1869 if ((N = lookupNode(ID, DL, InsertToken)))
1870 if (!VT.isVector())
1871 return SDValue(N, 0);
1872
1873 if (!N) {
1874 N = newSDNode<ConstantFPSDNode>(isTarget, Elt, VTs);
1875 CSEMap.insert(N, InsertToken);
1876 InsertNode(N);
1877 }
1878
1879 SDValue Result(N, 0);
1880 if (VT.isVector())
1881 Result = getSplat(VT, DL, Result);
1882 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1883 return Result;
1884}
1885
1887 bool isTarget) {
1888 EVT EltVT = VT.getScalarType();
1889 if (EltVT == MVT::f32)
1890 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1891 if (EltVT == MVT::f64)
1892 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1893 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1894 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1895 bool Ignored;
1896 APFloat APF = APFloat(Val);
1898 &Ignored);
1899 return getConstantFP(APF, DL, VT, isTarget);
1900 }
1901 llvm_unreachable("Unsupported type in getConstantFP");
1902}
1903
1905 EVT VT, int64_t Offset, bool isTargetGA,
1906 unsigned TargetFlags) {
1907 assert((TargetFlags == 0 || isTargetGA) &&
1908 "Cannot set target flags on target-independent globals");
1909
1910 // Truncate (with sign-extension) the offset value to the pointer size.
1912 if (BitWidth < 64)
1914
1915 unsigned Opc;
1916 if (GV->isThreadLocal())
1918 else
1920
1921 SDVTList VTs = getVTList(VT);
1922 SDNodeKey ID(Opc, VTs, {});
1923 ID.AddPointer(GV);
1924 ID.AddInteger(Offset);
1925 ID.AddInteger(TargetFlags);
1926 FoldingSetInsertToken InsertToken;
1927 if (SDNode *E = lookupNode(ID, DL, InsertToken))
1928 return SDValue(E, 0);
1929
1930 auto *N = newSDNode<GlobalAddressSDNode>(
1931 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VTs, Offset, TargetFlags);
1932 CSEMap.insert(N, InsertToken);
1933 InsertNode(N);
1934 return SDValue(N, 0);
1935}
1936
1938 SDVTList VTs = getVTList(MVT::Untyped);
1940 ID.AddPointer(GV);
1941 FoldingSetInsertToken InsertToken;
1942 if (SDNode *E = lookupNode(ID, SDLoc(), InsertToken))
1943 return SDValue(E, 0);
1944
1945 auto *N = newSDNode<DeactivationSymbolSDNode>(GV, VTs);
1946 CSEMap.insert(N, InsertToken);
1947 InsertNode(N);
1948 return SDValue(N, 0);
1949}
1950
1951SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1952 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1953 SDVTList VTs = getVTList(VT);
1954 SDNodeKey ID(Opc, VTs, {});
1955 ID.AddInteger(FI);
1956 FoldingSetInsertToken InsertToken;
1957 if (SDNode *E = lookupNode(ID, InsertToken))
1958 return SDValue(E, 0);
1959
1960 auto *N = newSDNode<FrameIndexSDNode>(FI, VTs, isTarget);
1961 CSEMap.insert(N, InsertToken);
1962 InsertNode(N);
1963 return SDValue(N, 0);
1964}
1965
1966SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1967 unsigned TargetFlags) {
1968 assert((TargetFlags == 0 || isTarget) &&
1969 "Cannot set target flags on target-independent jump tables");
1970 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1971 SDVTList VTs = getVTList(VT);
1972 SDNodeKey ID(Opc, VTs, {});
1973 ID.AddInteger(JTI);
1974 ID.AddInteger(TargetFlags);
1975 FoldingSetInsertToken InsertToken;
1976 if (SDNode *E = lookupNode(ID, InsertToken))
1977 return SDValue(E, 0);
1978
1979 auto *N = newSDNode<JumpTableSDNode>(JTI, VTs, isTarget, TargetFlags);
1980 CSEMap.insert(N, InsertToken);
1981 InsertNode(N);
1982 return SDValue(N, 0);
1983}
1984
1986 const SDLoc &DL) {
1988 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Other, Chain,
1989 getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));
1990}
1991
1993 MaybeAlign Alignment, int Offset,
1994 bool isTarget, unsigned TargetFlags) {
1995 assert((TargetFlags == 0 || isTarget) &&
1996 "Cannot set target flags on target-independent globals");
1997 if (!Alignment)
1998 Alignment = shouldOptForSize()
1999 ? getDataLayout().getABITypeAlign(C->getType())
2000 : getDataLayout().getPrefTypeAlign(C->getType());
2001 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2002 SDVTList VTs = getVTList(VT);
2003 SDNodeKey ID(Opc, VTs, {});
2004 ID.AddInteger(Alignment->value());
2005 ID.AddInteger(Offset);
2006 ID.AddPointer(C);
2007 ID.AddInteger(TargetFlags);
2008 FoldingSetInsertToken InsertToken;
2009 if (SDNode *E = lookupNode(ID, InsertToken))
2010 return SDValue(E, 0);
2011
2012 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VTs, Offset, *Alignment,
2013 TargetFlags);
2014 CSEMap.insert(N, InsertToken);
2015 InsertNode(N);
2016 SDValue V = SDValue(N, 0);
2017 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
2018 return V;
2019}
2020
2022 MaybeAlign Alignment, int Offset,
2023 bool isTarget, unsigned TargetFlags) {
2024 assert((TargetFlags == 0 || isTarget) &&
2025 "Cannot set target flags on target-independent globals");
2026 if (!Alignment)
2027 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
2028 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2029 SDVTList VTs = getVTList(VT);
2030 SDNodeKey ID(Opc, VTs, {});
2031 ID.AddInteger(Alignment->value());
2032 ID.AddInteger(Offset);
2033 C->addSelectionDAGCSEId(ID.Tail);
2034 ID.AddInteger(TargetFlags);
2035 FoldingSetInsertToken InsertToken;
2036 if (SDNode *E = lookupNode(ID, InsertToken))
2037 return SDValue(E, 0);
2038
2039 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VTs, Offset, *Alignment,
2040 TargetFlags);
2041 CSEMap.insert(N, InsertToken);
2042 InsertNode(N);
2043 return SDValue(N, 0);
2044}
2045
2047 SDNodeKey ID(ISD::BasicBlock, getVTList(MVT::Other), {});
2048 ID.AddPointer(MBB);
2049 FoldingSetInsertToken InsertToken;
2050 if (SDNode *E = lookupNode(ID, InsertToken))
2051 return SDValue(E, 0);
2052
2053 auto *N = newSDNode<BasicBlockSDNode>(MBB);
2054 CSEMap.insert(N, InsertToken);
2055 InsertNode(N);
2056 return SDValue(N, 0);
2057}
2058
2060 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2061 ValueTypeNodes.size())
2062 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
2063
2064 SDNode *&N = VT.isExtended() ?
2065 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2066
2067 if (N) return SDValue(N, 0);
2068 N = newSDNode<VTSDNode>(VT);
2069 InsertNode(N);
2070 return SDValue(N, 0);
2071}
2072
2074 SDNode *&N = ExternalSymbols[Sym];
2075 if (N) return SDValue(N, 0);
2076 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, getVTList(VT));
2077 InsertNode(N);
2078 return SDValue(N, 0);
2079}
2080
2081SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2083 return getExternalSymbol(SymName.data(), VT);
2084}
2085
2087 SDNode *&N = MCSymbols[Sym];
2088 if (N)
2089 return SDValue(N, 0);
2090 N = newSDNode<MCSymbolSDNode>(Sym, getVTList(VT));
2091 InsertNode(N);
2092 return SDValue(N, 0);
2093}
2094
2096 unsigned TargetFlags) {
2097 SDNode *&N =
2098 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2099 if (N) return SDValue(N, 0);
2100 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, getVTList(VT));
2101 InsertNode(N);
2102 return SDValue(N, 0);
2103}
2104
2106 EVT VT, unsigned TargetFlags) {
2108 return getTargetExternalSymbol(SymName.data(), VT, TargetFlags);
2109}
2110
2112 if ((unsigned)Cond >= CondCodeNodes.size())
2113 CondCodeNodes.resize(Cond+1);
2114
2115 if (!CondCodeNodes[Cond]) {
2116 auto *N = newSDNode<CondCodeSDNode>(Cond);
2117 CondCodeNodes[Cond] = N;
2118 InsertNode(N);
2119 }
2120
2121 return SDValue(CondCodeNodes[Cond], 0);
2122}
2123
2125 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2126 "APInt size does not match type size!");
2127
2128 if (MulImm == 0)
2129 return getConstant(0, DL, VT);
2130
2131 const MachineFunction &MF = getMachineFunction();
2132 const Function &F = MF.getFunction();
2133 ConstantRange CR = getVScaleRange(&F, 64);
2134 if (const APInt *C = CR.getSingleElement())
2135 return getConstant(MulImm * C->getZExtValue(), DL, VT);
2136
2137 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
2138}
2139
2140/// \returns a value of type \p VT that represents the runtime value of \p
2141/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2142/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2143/// or TypeSize.
2144template <typename Ty>
2146 EVT VT, Ty Quantity) {
2147 if (Quantity.isScalable())
2148 return DAG.getVScale(
2149 DL, VT, APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2150
2151 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2152}
2153
2155 ElementCount EC) {
2156 return getFixedOrScalableQuantity(*this, DL, VT, EC);
2157}
2158
2160 return getFixedOrScalableQuantity(*this, DL, VT, TS);
2161}
2162
2164 ElementCount EC) {
2165 EVT IdxVT = TLI->getVectorIdxTy(getDataLayout());
2166 EVT MaskVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), DataVT);
2167 return getNode(ISD::GET_ACTIVE_LANE_MASK, DL, MaskVT,
2168 getConstant(0, DL, IdxVT), getElementCount(DL, IdxVT, EC));
2169}
2170
2172 APInt One(ResVT.getScalarSizeInBits(), 1);
2173 return getStepVector(DL, ResVT, One);
2174}
2175
2177 const APInt &StepVal) {
2178 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2179 if (ResVT.isScalableVector())
2180 return getNode(
2181 ISD::STEP_VECTOR, DL, ResVT,
2182 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
2183
2184 SmallVector<SDValue, 16> OpsStepConstants;
2185 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2186 OpsStepConstants.push_back(
2187 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
2188 return getBuildVector(ResVT, DL, OpsStepConstants);
2189}
2190
2191/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2192/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2197
2199 SDValue N2, ArrayRef<int> Mask) {
2200 assert(VT.getVectorNumElements() == Mask.size() &&
2201 "Must have the same number of vector elements as mask elements!");
2202 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2203 "Invalid VECTOR_SHUFFLE");
2204
2205 // Canonicalize shuffle undef, undef -> undef
2206 if (N1.isUndef() && N2.isUndef()) {
2207 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2208 return getPOISON(VT);
2209 return getUNDEF(VT);
2210 }
2211
2212 // Validate that all indices in Mask are within the range of the elements
2213 // input to the shuffle.
2214 int NElts = Mask.size();
2215 assert(llvm::all_of(Mask,
2216 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2217 "Index out of range");
2218
2219 // Copy the mask so we can do any needed cleanup.
2220 SmallVector<int, 8> MaskVec(Mask);
2221
2222 // Canonicalize shuffle v, v -> v, poison
2223 if (N1 == N2) {
2224 N2 = getPOISON(VT);
2225 for (int i = 0; i != NElts; ++i)
2226 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2227 }
2228
2229 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2230 if (N1.isUndef())
2231 commuteShuffle(N1, N2, MaskVec);
2232
2233 if (TLI->hasVectorBlend()) {
2234 // If shuffling a splat, try to blend the splat instead. We do this here so
2235 // that even when this arises during lowering we don't have to re-handle it.
2236 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2237 BitVector UndefElements;
2238 SDValue Splat = BV->getSplatValue(&UndefElements);
2239 if (!Splat)
2240 return;
2241
2242 for (int i = 0; i < NElts; ++i) {
2243 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2244 continue;
2245
2246 // If this input comes from undef, mark it as such.
2247 if (UndefElements[MaskVec[i] - Offset]) {
2248 MaskVec[i] = -1;
2249 continue;
2250 }
2251
2252 // If we can blend a non-undef lane, use that instead.
2253 if (!UndefElements[i])
2254 MaskVec[i] = i + Offset;
2255 }
2256 };
2257 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2258 BlendSplat(N1BV, 0);
2259 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2260 BlendSplat(N2BV, NElts);
2261 }
2262
2263 // Canonicalize all index into lhs, -> shuffle lhs, poison
2264 // Canonicalize all index into rhs, -> shuffle rhs, poison
2265 bool AllLHS = true, AllRHS = true;
2266 bool N2Undef = N2.isUndef();
2267 for (int i = 0; i != NElts; ++i) {
2268 if (MaskVec[i] >= NElts) {
2269 if (N2Undef)
2270 MaskVec[i] = -1;
2271 else
2272 AllLHS = false;
2273 } else if (MaskVec[i] >= 0) {
2274 AllRHS = false;
2275 }
2276 }
2277 if (AllLHS && AllRHS)
2278 return getPOISON(VT);
2279 if (AllLHS && !N2Undef)
2280 N2 = getPOISON(VT);
2281 if (AllRHS) {
2282 N1 = getPOISON(VT);
2283 commuteShuffle(N1, N2, MaskVec);
2284 }
2285 // Reset our undef status after accounting for the mask.
2286 N2Undef = N2.isUndef();
2287 // Re-check whether both sides ended up undef.
2288 if (N1.isUndef() && N2Undef) {
2289 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2290 return getPOISON(VT);
2291 return getUNDEF(VT);
2292 }
2293
2294 // If Identity shuffle return that node.
2295 bool Identity = true, AllSame = true;
2296 for (int i = 0; i != NElts; ++i) {
2297 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2298 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2299 }
2300 if (Identity && NElts)
2301 return N1;
2302
2303 // Shuffling a constant splat doesn't change the result.
2304 if (N2Undef) {
2305 SDValue V = N1;
2306
2307 // Look through any bitcasts. We check that these don't change the number
2308 // (and size) of elements and just changes their types.
2309 while (V.getOpcode() == ISD::BITCAST)
2310 V = V->getOperand(0);
2311
2312 // A splat should always show up as a build vector node.
2313 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2314 BitVector UndefElements;
2315 SDValue Splat = BV->getSplatValue(&UndefElements);
2316 // If this is a splat of an undef, shuffling it is also undef.
2317 if (Splat && Splat.isUndef())
2318 return Splat.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
2319
2320 bool SameNumElts =
2321 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2322
2323 // We only have a splat which can skip shuffles if there is a splatted
2324 // value and no undef lanes rearranged by the shuffle.
2325 if (Splat && UndefElements.none()) {
2326 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2327 // number of elements match or the value splatted is a zero constant.
2328 if (SameNumElts || isNullConstant(Splat))
2329 return N1;
2330 }
2331
2332 // If the shuffle itself creates a splat, build the vector directly.
2333 if (AllSame && SameNumElts) {
2334 EVT BuildVT = BV->getValueType(0);
2335 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
2336 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
2337
2338 // We may have jumped through bitcasts, so the type of the
2339 // BUILD_VECTOR may not match the type of the shuffle.
2340 if (BuildVT != VT)
2341 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
2342 return NewBV;
2343 }
2344 }
2345 }
2346
2347 SDVTList VTs = getVTList(VT);
2348 SDValue Ops[2] = { N1, N2 };
2350 for (int i = 0; i != NElts; ++i)
2351 ID.AddInteger(MaskVec[i]);
2352
2353 FoldingSetInsertToken InsertToken;
2354 if (SDNode *E = lookupNode(ID, dl, InsertToken))
2355 return SDValue(E, 0);
2356
2357 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2358 // SDNode doesn't have access to it. This memory will be "leaked" when
2359 // the node is deallocated, but recovered when the NodeAllocator is released.
2360 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
2361 llvm::copy(MaskVec, MaskAlloc);
2362
2363 auto *N = newSDNode<ShuffleVectorSDNode>(VTs, dl.getIROrder(),
2364 dl.getDebugLoc(), MaskAlloc);
2365 createOperands(N, Ops);
2366
2367 CSEMap.insert(N, InsertToken);
2368 InsertNode(N);
2369 SDValue V = SDValue(N, 0);
2370 NewSDValueDbgMsg(V, "Creating new node: ", this);
2371 return V;
2372}
2373
2375 EVT VT = SV.getValueType(0);
2376 SmallVector<int, 8> MaskVec(SV.getMask());
2378
2379 SDValue Op0 = SV.getOperand(0);
2380 SDValue Op1 = SV.getOperand(1);
2381 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
2382}
2383
2385 SDVTList VTs = getVTList(VT);
2386 SDNodeKey ID(ISD::Register, VTs, {});
2387 ID.AddInteger(Reg.id());
2388 FoldingSetInsertToken InsertToken;
2389 if (SDNode *E = lookupNode(ID, InsertToken))
2390 return SDValue(E, 0);
2391
2392 auto *N = newSDNode<RegisterSDNode>(Reg, VTs);
2393 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2394 CSEMap.insert(N, InsertToken);
2395 InsertNode(N);
2396 return SDValue(N, 0);
2397}
2398
2400 SDNodeKey ID(ISD::RegisterMask, getVTList(MVT::Untyped), {});
2401 ID.AddPointer(RegMask);
2402 FoldingSetInsertToken InsertToken;
2403 if (SDNode *E = lookupNode(ID, InsertToken))
2404 return SDValue(E, 0);
2405
2406 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2407 CSEMap.insert(N, InsertToken);
2408 InsertNode(N);
2409 return SDValue(N, 0);
2410}
2411
2413 MCSymbol *Label) {
2414 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2415}
2416
2417SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2418 SDValue Root, MCSymbol *Label) {
2419 SDValue Ops[] = { Root };
2420 SDNodeKey ID(Opcode, getVTList(MVT::Other), Ops);
2421 ID.AddPointer(Label);
2422 FoldingSetInsertToken InsertToken;
2423 if (SDNode *E = lookupNode(ID, InsertToken))
2424 return SDValue(E, 0);
2425
2426 auto *N =
2427 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2428 createOperands(N, Ops);
2429
2430 CSEMap.insert(N, InsertToken);
2431 InsertNode(N);
2432 return SDValue(N, 0);
2433}
2434
2436 int64_t Offset, bool isTarget,
2437 unsigned TargetFlags) {
2438 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2439 SDVTList VTs = getVTList(VT);
2440
2441 SDNodeKey ID(Opc, VTs, {});
2442 ID.AddPointer(BA);
2443 ID.AddInteger(Offset);
2444 ID.AddInteger(TargetFlags);
2445 FoldingSetInsertToken InsertToken;
2446 if (SDNode *E = lookupNode(ID, InsertToken))
2447 return SDValue(E, 0);
2448
2449 auto *N = newSDNode<BlockAddressSDNode>(Opc, VTs, BA, Offset, TargetFlags);
2450 CSEMap.insert(N, InsertToken);
2451 InsertNode(N);
2452 return SDValue(N, 0);
2453}
2454
2456 SDNodeKey ID(ISD::SRCVALUE, getVTList(MVT::Other), {});
2457 ID.AddPointer(V);
2458
2459 FoldingSetInsertToken InsertToken;
2460 if (SDNode *E = lookupNode(ID, InsertToken))
2461 return SDValue(E, 0);
2462
2463 auto *N = newSDNode<SrcValueSDNode>(V);
2464 CSEMap.insert(N, InsertToken);
2465 InsertNode(N);
2466 return SDValue(N, 0);
2467}
2468
2470 SDNodeKey ID(ISD::MDNODE_SDNODE, getVTList(MVT::Other), {});
2471 ID.AddPointer(MD);
2472
2473 FoldingSetInsertToken InsertToken;
2474 if (SDNode *E = lookupNode(ID, InsertToken))
2475 return SDValue(E, 0);
2476
2477 auto *N = newSDNode<MDNodeSDNode>(MD);
2478 CSEMap.insert(N, InsertToken);
2479 InsertNode(N);
2480 return SDValue(N, 0);
2481}
2482
2484 if (VT == V.getValueType())
2485 return V;
2486
2487 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2488}
2489
2491 unsigned SrcAS, unsigned DestAS,
2492 const SDNodeFlags Flags) {
2493 SDVTList VTs = getVTList(VT);
2494 SDValue Ops[] = {Ptr};
2496 ID.AddInteger(SrcAS);
2497 ID.AddInteger(DestAS);
2498
2499 FoldingSetInsertToken InsertToken;
2500 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
2501 E->intersectFlagsWith(Flags);
2502 return SDValue(E, 0);
2503 }
2504
2505 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2506 VTs, SrcAS, DestAS);
2507 N->setFlags(Flags);
2508 createOperands(N, Ops);
2509
2510 CSEMap.insert(N, InsertToken);
2511 InsertNode(N);
2512 return SDValue(N, 0);
2513}
2514
2516 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2517}
2518
2520 UndefPoisonKind Kind) {
2521 if (isGuaranteedNotToBeUndefOrPoison(V, DemandedElts, Kind))
2522 return V;
2523 return getFreeze(V);
2524}
2525
2526/// getShiftAmountOperand - Return the specified value casted to
2527/// the target's desired shift amount type.
2529 EVT OpTy = Op.getValueType();
2530 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2531 if (OpTy == ShTy || OpTy.isVector()) return Op;
2532
2533 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2534}
2535
2537 SDLoc dl(Node);
2539 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2540 EVT VT = Node->getValueType(0);
2541 SDValue Tmp1 = Node->getOperand(0);
2542 SDValue Tmp2 = Node->getOperand(1);
2543 const MaybeAlign MA(Node->getConstantOperandVal(3));
2544
2545 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2546 Tmp2, MachinePointerInfo(V));
2547 SDValue VAList = VAListLoad;
2548
2549 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2550 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2551 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2552
2553 VAList = getNode(
2554 ISD::AND, dl, VAList.getValueType(), VAList,
2555 getSignedConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2556 }
2557
2558 // Increment the pointer, VAList, to the next vaarg
2559 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2560 getConstant(getDataLayout().getTypeAllocSize(
2561 VT.getTypeForEVT(*getContext())),
2562 dl, VAList.getValueType()));
2563 // Store the incremented VAList to the legalized pointer
2564 Tmp1 =
2565 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2566 // Load the actual argument out of the pointer VAList
2567 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2568}
2569
2571 SDLoc dl(Node);
2573 // This defaults to loading a pointer from the input and storing it to the
2574 // output, returning the chain.
2575 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2576 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2577 SDValue Tmp1 =
2578 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2579 Node->getOperand(2), MachinePointerInfo(VS));
2580 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2581 MachinePointerInfo(VD));
2582}
2583
2585 const DataLayout &DL = getDataLayout();
2586 Type *Ty = VT.getTypeForEVT(*getContext());
2587 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2588
2589 if (TLI->isTypeLegal(VT) || !VT.isVector())
2590 return RedAlign;
2591
2592 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2593 const Align StackAlign = TFI->getStackAlign();
2594
2595 // See if we can choose a smaller ABI alignment in cases where it's an
2596 // illegal vector type that will get broken down.
2597 if (RedAlign > StackAlign) {
2598 EVT IntermediateVT;
2599 MVT RegisterVT;
2600 unsigned NumIntermediates;
2601 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2602 NumIntermediates, RegisterVT);
2603 Ty = IntermediateVT.getTypeForEVT(*getContext());
2604 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2605 if (RedAlign2 < RedAlign)
2606 RedAlign = RedAlign2;
2607
2608 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2609 // If the stack is not realignable, the alignment should be limited to the
2610 // StackAlignment
2611 RedAlign = std::min(RedAlign, StackAlign);
2612 }
2613
2614 return RedAlign;
2615}
2616
2618 MachineFrameInfo &MFI = MF->getFrameInfo();
2619 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2620 int StackID = 0;
2621 if (Bytes.isScalable())
2622 StackID = TFI->getStackIDForScalableVectors();
2623 // The stack id gives an indication of whether the object is scalable or
2624 // not, so it's safe to pass in the minimum size here.
2625 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,
2626 false, nullptr, StackID);
2627 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2628}
2629
2631 Type *Ty = VT.getTypeForEVT(*getContext());
2632 Align StackAlign =
2633 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2634 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2635}
2636
2638 TypeSize VT1Size = VT1.getStoreSize();
2639 TypeSize VT2Size = VT2.getStoreSize();
2640 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2641 "Don't know how to choose the maximum size when creating a stack "
2642 "temporary");
2643 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2644 ? VT1Size
2645 : VT2Size;
2646
2647 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2648 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2649 const DataLayout &DL = getDataLayout();
2650 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2651 return CreateStackTemporary(Bytes, Align);
2652}
2653
2655 const SDLoc &DL, SDValue Chain) {
2656 EVT SrcVT = SrcOp.getValueType();
2657 Type *DestType = DestVT.getTypeForEVT(*getContext());
2658 Align DestAlign = getDataLayout().getPrefTypeAlign(DestType);
2659
2660 // Create the stack frame object.
2661 Align SrcAlign =
2663 SDValue FIPtr = CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
2664
2665 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
2666 int SPFI = StackPtrFI->getIndex();
2667 MachinePointerInfo PtrInfo =
2669
2670 // Emit a store to the stack slot. Use a truncstore if the input value is
2671 // later than DestVT.
2672 SDValue Store;
2673
2674 if (SrcVT.bitsGT(SlotVT))
2675 Store = getTruncStore(Chain, DL, SrcOp, FIPtr, PtrInfo, SlotVT, SrcAlign);
2676 else {
2677 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
2678 Store = getStore(Chain, DL, SrcOp, FIPtr, PtrInfo, SrcAlign);
2679 }
2680
2681 // Result is a load from the stack slot.
2682 if (SlotVT.bitsEq(DestVT))
2683 return getLoad(DestVT, DL, Store, FIPtr, PtrInfo, DestAlign);
2684
2685 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
2686 return getExtLoad(ISD::EXTLOAD, DL, DestVT, Store, FIPtr, PtrInfo, SlotVT,
2687 DestAlign);
2688}
2689
2691 ISD::CondCode Cond, const SDLoc &dl,
2692 SDNodeFlags Flags) {
2693 EVT OpVT = N1.getValueType();
2694
2695 auto GetUndefBooleanConstant = [&]() {
2696 if (VT.getScalarType() == MVT::i1 ||
2697 TLI->getBooleanContents(OpVT) ==
2699 return getUNDEF(VT);
2700 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2701 // so we cannot use getUNDEF(). Return zero instead.
2702 return getConstant(0, dl, VT);
2703 };
2704
2705 // These setcc operations always fold.
2706 switch (Cond) {
2707 default: break;
2708 case ISD::SETFALSE:
2709 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2710 case ISD::SETTRUE:
2711 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2712
2713 case ISD::SETOEQ:
2714 case ISD::SETOGT:
2715 case ISD::SETOGE:
2716 case ISD::SETOLT:
2717 case ISD::SETOLE:
2718 case ISD::SETONE:
2719 case ISD::SETO:
2720 case ISD::SETUO:
2721 case ISD::SETUEQ:
2722 case ISD::SETUNE:
2723 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2724 break;
2725 }
2726
2727 if (OpVT.isInteger()) {
2728 // For EQ and NE, we can always pick a value for the undef to make the
2729 // predicate pass or fail, so we can return undef.
2730 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2731 // icmp eq/ne X, undef -> undef.
2732 if ((N1.isUndef() || N2.isUndef()) &&
2733 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2734 return GetUndefBooleanConstant();
2735
2736 // If both operands are undef, we can return undef for int comparison.
2737 // icmp undef, undef -> undef.
2738 if (N1.isUndef() && N2.isUndef())
2739 return GetUndefBooleanConstant();
2740
2741 // icmp X, X -> true/false
2742 // icmp X, undef -> true/false because undef could be X.
2743 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2744 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2745 }
2746
2747 if (ConstantSDNode *N2C = isConstOrConstSplat(N2)) {
2748 const APInt &C2 = N2C->getAPIntValue();
2749 if (ConstantSDNode *N1C = isConstOrConstSplat(N1)) {
2750 const APInt &C1 = N1C->getAPIntValue();
2751
2753 dl, VT, OpVT);
2754 }
2755 }
2756
2757 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2758 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2759
2760 if (N1CFP && N2CFP) {
2761 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2762 switch (Cond) {
2763 default: break;
2764 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2765 return GetUndefBooleanConstant();
2766 [[fallthrough]];
2767 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2768 OpVT);
2769 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2770 return GetUndefBooleanConstant();
2771 [[fallthrough]];
2773 R==APFloat::cmpLessThan, dl, VT,
2774 OpVT);
2775 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2776 return GetUndefBooleanConstant();
2777 [[fallthrough]];
2778 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2779 OpVT);
2780 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2781 return GetUndefBooleanConstant();
2782 [[fallthrough]];
2784 VT, OpVT);
2785 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2786 return GetUndefBooleanConstant();
2787 [[fallthrough]];
2789 R==APFloat::cmpEqual, dl, VT,
2790 OpVT);
2791 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2792 return GetUndefBooleanConstant();
2793 [[fallthrough]];
2795 R==APFloat::cmpEqual, dl, VT, OpVT);
2796 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2797 OpVT);
2798 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2799 OpVT);
2801 R==APFloat::cmpEqual, dl, VT,
2802 OpVT);
2803 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2804 OpVT);
2806 R==APFloat::cmpLessThan, dl, VT,
2807 OpVT);
2809 R==APFloat::cmpUnordered, dl, VT,
2810 OpVT);
2812 VT, OpVT);
2813 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2814 OpVT);
2815 }
2816 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2817 // Ensure that the constant occurs on the RHS.
2819 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2820 return SDValue();
2821 return getSetCC(dl, VT, N2, N1, SwappedCond, /*Chain=*/{},
2822 /*IsSignaling=*/false, Flags);
2823 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2824 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2825 // If an operand is known to be a nan (or undef that could be a nan), we can
2826 // fold it.
2827 // Choosing NaN for the undef will always make unordered comparison succeed
2828 // and ordered comparison fails.
2829 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2830 switch (ISD::getUnorderedFlavor(Cond)) {
2831 default:
2832 llvm_unreachable("Unknown flavor!");
2833 case 0: // Known false.
2834 return getBoolConstant(false, dl, VT, OpVT);
2835 case 1: // Known true.
2836 return getBoolConstant(true, dl, VT, OpVT);
2837 case 2: // Undefined.
2838 return GetUndefBooleanConstant();
2839 }
2840 }
2841
2842 // Could not fold it.
2843 return SDValue();
2844}
2845
2846/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2847/// use this predicate to simplify operations downstream.
2849 unsigned BitWidth = Op.getScalarValueSizeInBits();
2851}
2852
2853// TODO: Should have argument to specify if sign bit of nan is ignorable.
2855 if (Depth >= MaxRecursionDepth)
2856 return false; // Limit search depth.
2857
2858 unsigned Opc = Op.getOpcode();
2859 switch (Opc) {
2860 case ISD::FABS:
2861 return true;
2862 case ISD::AssertNoFPClass: {
2863 FPClassTest NoFPClass =
2864 static_cast<FPClassTest>(Op.getConstantOperandVal(1));
2865
2866 const FPClassTest TestMask = fcNan | fcNegative;
2867 return (NoFPClass & TestMask) == TestMask;
2868 }
2869 case ISD::ARITH_FENCE:
2870 return SignBitIsZeroFP(Op.getOperand(0), Depth + 1);
2871 case ISD::FEXP:
2872 case ISD::FEXP2:
2873 case ISD::FEXP10:
2874 return Op->getFlags().hasNoNaNs();
2875 case ISD::FMINNUM:
2876 case ISD::FMINNUM_IEEE:
2877 case ISD::FMINIMUM:
2878 case ISD::FMINIMUMNUM:
2879 return SignBitIsZeroFP(Op.getOperand(1), Depth + 1) &&
2880 SignBitIsZeroFP(Op.getOperand(0), Depth + 1);
2881 case ISD::FMAXNUM:
2882 case ISD::FMAXNUM_IEEE:
2883 case ISD::FMAXIMUM:
2884 case ISD::FMAXIMUMNUM:
2885 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2886 // is sufficient.
2887 return SignBitIsZeroFP(Op.getOperand(1), Depth + 1) &&
2888 SignBitIsZeroFP(Op.getOperand(0), Depth + 1);
2889 default:
2890 return false;
2891 }
2892
2893 llvm_unreachable("covered opcode switch");
2894}
2895
2896/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2897/// this predicate to simplify operations downstream. Mask is known to be zero
2898/// for bits that V cannot have.
2900 unsigned Depth) const {
2901 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2902}
2903
2904/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2905/// DemandedElts. We use this predicate to simplify operations downstream.
2906/// Mask is known to be zero for bits that V cannot have.
2908 const APInt &DemandedElts,
2909 unsigned Depth) const {
2910 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2911}
2912
2913/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2914/// DemandedElts. We use this predicate to simplify operations downstream.
2916 unsigned Depth /* = 0 */) const {
2917 return computeKnownBits(V, DemandedElts, Depth).isZero();
2918}
2919
2920/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2922 unsigned Depth) const {
2923 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2924}
2925
2927 const APInt &DemandedElts,
2928 unsigned Depth) const {
2929 EVT VT = Op.getValueType();
2930 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2931
2932 unsigned NumElts = VT.getVectorNumElements();
2933 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2934
2935 APInt KnownZeroElements = APInt::getZero(NumElts);
2936 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2937 if (!DemandedElts[EltIdx])
2938 continue; // Don't query elements that are not demanded.
2939 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
2940 if (MaskedVectorIsZero(Op, Mask, Depth))
2941 KnownZeroElements.setBit(EltIdx);
2942 }
2943 return KnownZeroElements;
2944}
2945
2946/// isSplatValue - Return true if the vector V has the same value
2947/// across all DemandedElts. For scalable vectors, we don't know the
2948/// number of lanes at compile time. Instead, we use a 1 bit APInt
2949/// to represent a conservative value for all lanes; that is, that
2950/// one bit value is implicitly splatted across all lanes.
2951bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2952 APInt &UndefElts, unsigned Depth) const {
2953 unsigned Opcode = V.getOpcode();
2954 EVT VT = V.getValueType();
2955 assert(VT.isVector() && "Vector type expected");
2956 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2957 "scalable demanded bits are ignored");
2958
2959 if (!DemandedElts)
2960 return false; // No demanded elts, better to assume we don't know anything.
2961
2962 if (Depth >= MaxRecursionDepth)
2963 return false; // Limit search depth.
2964
2965 // Deal with some common cases here that work for both fixed and scalable
2966 // vector types.
2967 switch (Opcode) {
2968 case ISD::SPLAT_VECTOR:
2969 UndefElts = V.getOperand(0).isUndef()
2970 ? APInt::getAllOnes(DemandedElts.getBitWidth())
2971 : APInt(DemandedElts.getBitWidth(), 0);
2972 return true;
2973 case ISD::ADD:
2974 case ISD::SUB:
2975 case ISD::AND:
2976 case ISD::XOR:
2977 case ISD::OR: {
2978 APInt UndefLHS, UndefRHS;
2979 SDValue LHS = V.getOperand(0);
2980 SDValue RHS = V.getOperand(1);
2981 // Only recognize splats with the same demanded undef elements for both
2982 // operands, otherwise we might fail to handle binop-specific undef
2983 // handling.
2984 // e.g. (and undef, 0) -> 0 etc.
2985 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2986 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1) &&
2987 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2988 UndefElts = UndefLHS | UndefRHS;
2989 return true;
2990 }
2991 return false;
2992 }
2993 case ISD::ABS:
2995 case ISD::TRUNCATE:
2996 case ISD::SIGN_EXTEND:
2997 case ISD::ZERO_EXTEND:
2998 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2999 default:
3000 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3001 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3002 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,
3003 Depth);
3004 break;
3005 }
3006
3007 // We don't support other cases than those above for scalable vectors at
3008 // the moment.
3009 if (VT.isScalableVector())
3010 return false;
3011
3012 unsigned NumElts = VT.getVectorNumElements();
3013 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3014 UndefElts = APInt::getZero(NumElts);
3015
3016 switch (Opcode) {
3017 case ISD::BUILD_VECTOR: {
3018 SDValue Scl;
3019 for (unsigned i = 0; i != NumElts; ++i) {
3020 SDValue Op = V.getOperand(i);
3021 if (Op.isUndef()) {
3022 UndefElts.setBit(i);
3023 continue;
3024 }
3025 if (!DemandedElts[i])
3026 continue;
3027 if (Scl && Scl != Op)
3028 return false;
3029 Scl = Op;
3030 }
3031 return true;
3032 }
3033 case ISD::VECTOR_SHUFFLE: {
3034 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3035 APInt DemandedLHS = APInt::getZero(NumElts);
3036 APInt DemandedRHS = APInt::getZero(NumElts);
3037 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
3038 for (int i = 0; i != (int)NumElts; ++i) {
3039 int M = Mask[i];
3040 if (M < 0) {
3041 UndefElts.setBit(i);
3042 continue;
3043 }
3044 if (!DemandedElts[i])
3045 continue;
3046 if (M < (int)NumElts)
3047 DemandedLHS.setBit(M);
3048 else
3049 DemandedRHS.setBit(M - NumElts);
3050 }
3051
3052 // If we aren't demanding either op, assume there's no splat.
3053 // If we are demanding both ops, assume there's no splat.
3054 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3055 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3056 return false;
3057
3058 // See if the demanded elts of the source op is a splat or we only demand
3059 // one element, which should always be a splat.
3060 // TODO: Handle source ops splats with undefs.
3061 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3062 APInt SrcUndefs;
3063 return (SrcElts.popcount() == 1) ||
3064 (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&
3065 (SrcElts & SrcUndefs).isZero());
3066 };
3067 if (!DemandedLHS.isZero())
3068 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
3069 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
3070 }
3072 // Offset the demanded elts by the subvector index.
3073 SDValue Src = V.getOperand(0);
3074 // We don't support scalable vectors at the moment.
3075 if (Src.getValueType().isScalableVector())
3076 return false;
3077 uint64_t Idx = V.getConstantOperandVal(1);
3078 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3079 APInt UndefSrcElts;
3080 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3081 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
3082 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
3083 return true;
3084 }
3085 break;
3086 }
3090 // Widen the demanded elts by the src element count.
3091 SDValue Src = V.getOperand(0);
3092 // We don't support scalable vectors at the moment.
3093 if (Src.getValueType().isScalableVector())
3094 return false;
3095 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3096 APInt UndefSrcElts;
3097 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
3098 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
3099 UndefElts = UndefSrcElts.trunc(NumElts);
3100 return true;
3101 }
3102 break;
3103 }
3104 case ISD::BITCAST: {
3105 SDValue Src = V.getOperand(0);
3106 EVT SrcVT = Src.getValueType();
3107 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3108 unsigned BitWidth = VT.getScalarSizeInBits();
3109
3110 // Ignore bitcasts from unsupported types.
3111 // TODO: Add fp support?
3112 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3113 break;
3114
3115 // Bitcast 'small element' vector to 'large element' vector.
3116 if ((BitWidth % SrcBitWidth) == 0) {
3117 // See if each sub element is a splat.
3118 unsigned Scale = BitWidth / SrcBitWidth;
3119 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3120 APInt ScaledDemandedElts =
3121 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3122 for (unsigned I = 0; I != Scale; ++I) {
3123 APInt SubUndefElts;
3124 APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);
3125 APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);
3126 SubDemandedElts &= ScaledDemandedElts;
3127 if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))
3128 return false;
3129 // TODO: Add support for merging sub undef elements.
3130 if (!SubUndefElts.isZero())
3131 return false;
3132 }
3133 return true;
3134 }
3135 break;
3136 }
3137 }
3138
3139 return false;
3140}
3141
3142/// Helper wrapper to main isSplatValue function.
3143bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3144 EVT VT = V.getValueType();
3145 assert(VT.isVector() && "Vector type expected");
3146
3147 APInt UndefElts;
3148 // Since the number of lanes in a scalable vector is unknown at compile time,
3149 // we track one bit which is implicitly broadcast to all lanes. This means
3150 // that all lanes in a scalable vector are considered demanded.
3151 APInt DemandedElts
3153 return isSplatValue(V, DemandedElts, UndefElts) &&
3154 (AllowUndefs || !UndefElts);
3155}
3156
3159
3160 EVT VT = V.getValueType();
3161 unsigned Opcode = V.getOpcode();
3162 switch (Opcode) {
3163 default: {
3164 APInt UndefElts;
3165 // Since the number of lanes in a scalable vector is unknown at compile time,
3166 // we track one bit which is implicitly broadcast to all lanes. This means
3167 // that all lanes in a scalable vector are considered demanded.
3168 APInt DemandedElts
3170
3171 if (isSplatValue(V, DemandedElts, UndefElts)) {
3172 if (VT.isScalableVector()) {
3173 // DemandedElts and UndefElts are ignored for scalable vectors, since
3174 // the only supported cases are SPLAT_VECTOR nodes.
3175 SplatIdx = 0;
3176 } else {
3177 // Handle case where all demanded elements are UNDEF.
3178 if (DemandedElts.isSubsetOf(UndefElts)) {
3179 SplatIdx = 0;
3180 return getUNDEF(VT);
3181 }
3182 SplatIdx = (UndefElts & DemandedElts).countr_one();
3183 }
3184 return V;
3185 }
3186 break;
3187 }
3188 case ISD::SPLAT_VECTOR:
3189 SplatIdx = 0;
3190 return V;
3191 case ISD::VECTOR_SHUFFLE: {
3192 assert(!VT.isScalableVector());
3193 // Check if this is a shuffle node doing a splat.
3194 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3195 // getTargetVShiftNode currently struggles without the splat source.
3196 auto *SVN = cast<ShuffleVectorSDNode>(V);
3197 if (!SVN->isSplat())
3198 break;
3199 int Idx = SVN->getSplatIndex();
3200 int NumElts = V.getValueType().getVectorNumElements();
3201 SplatIdx = Idx % NumElts;
3202 return V.getOperand(Idx / NumElts);
3203 }
3204 }
3205
3206 return SDValue();
3207}
3208
3210 int SplatIdx;
3211 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3212 EVT SVT = SrcVector.getValueType().getScalarType();
3213 EVT LegalSVT = SVT;
3214 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
3215 if (!SVT.isInteger())
3216 return SDValue();
3217 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
3218 if (LegalSVT.bitsLT(SVT))
3219 return SDValue();
3220 }
3221 return getExtractVectorElt(SDLoc(V), LegalSVT, SrcVector, SplatIdx);
3222 }
3223 return SDValue();
3224}
3225
3226std::optional<ConstantRange>
3228 unsigned Depth) const {
3229 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3230 V.getOpcode() == ISD::SRA) &&
3231 "Unknown shift node");
3232 // Shifting more than the bitwidth is not valid.
3233 unsigned BitWidth = V.getScalarValueSizeInBits();
3234
3235 if (auto *Cst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
3236 const APInt &ShAmt = Cst->getAPIntValue();
3237 if (ShAmt.uge(BitWidth))
3238 return std::nullopt;
3239 return ConstantRange(ShAmt);
3240 }
3241
3242 if (auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1))) {
3243 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3244 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3245 if (!DemandedElts[i])
3246 continue;
3247 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3248 if (!SA) {
3249 MinAmt = MaxAmt = nullptr;
3250 break;
3251 }
3252 const APInt &ShAmt = SA->getAPIntValue();
3253 if (ShAmt.uge(BitWidth))
3254 return std::nullopt;
3255 if (!MinAmt || MinAmt->ugt(ShAmt))
3256 MinAmt = &ShAmt;
3257 if (!MaxAmt || MaxAmt->ult(ShAmt))
3258 MaxAmt = &ShAmt;
3259 }
3260 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3261 "Failed to find matching min/max shift amounts");
3262 if (MinAmt && MaxAmt)
3263 return ConstantRange(*MinAmt, *MaxAmt + 1);
3264 }
3265
3266 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3267 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3268 KnownBits KnownAmt = computeKnownBits(V.getOperand(1), DemandedElts, Depth);
3269 if (KnownAmt.getMaxValue().ult(BitWidth))
3270 return ConstantRange::fromKnownBits(KnownAmt, /*IsSigned=*/false);
3271
3272 return std::nullopt;
3273}
3274
3275std::optional<unsigned>
3277 unsigned Depth) const {
3278 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3279 V.getOpcode() == ISD::SRA) &&
3280 "Unknown shift node");
3281 if (std::optional<ConstantRange> AmtRange =
3282 getValidShiftAmountRange(V, DemandedElts, Depth))
3283 if (const APInt *ShAmt = AmtRange->getSingleElement())
3284 return ShAmt->getZExtValue();
3285 return std::nullopt;
3286}
3287
3288std::optional<unsigned>
3290 APInt DemandedElts = getDemandAllEltsMask(V);
3291 return getValidShiftAmount(V, DemandedElts, Depth);
3292}
3293
3294std::optional<unsigned>
3296 unsigned Depth) const {
3297 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3298 V.getOpcode() == ISD::SRA) &&
3299 "Unknown shift node");
3300 if (std::optional<ConstantRange> AmtRange =
3301 getValidShiftAmountRange(V, DemandedElts, Depth))
3302 return AmtRange->getUnsignedMin().getZExtValue();
3303 return std::nullopt;
3304}
3305
3306std::optional<unsigned>
3308 APInt DemandedElts = getDemandAllEltsMask(V);
3309 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3310}
3311
3312std::optional<unsigned>
3314 unsigned Depth) const {
3315 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3316 V.getOpcode() == ISD::SRA) &&
3317 "Unknown shift node");
3318 if (std::optional<ConstantRange> AmtRange =
3319 getValidShiftAmountRange(V, DemandedElts, Depth))
3320 return AmtRange->getUnsignedMax().getZExtValue();
3321 return std::nullopt;
3322}
3323
3324std::optional<unsigned>
3326 APInt DemandedElts = getDemandAllEltsMask(V);
3327 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3328}
3329
3330/// Determine which bits of Op are known to be either zero or one and return
3331/// them in Known. For vectors, the known bits are those that are shared by
3332/// every vector element.
3334 APInt DemandedElts = getDemandAllEltsMask(Op);
3335 return computeKnownBits(Op, DemandedElts, Depth);
3336}
3337
3338/// Determine which bits of Op are known to be either zero or one and return
3339/// them in Known. The DemandedElts argument allows us to only collect the known
3340/// bits that are shared by the requested vector elements.
3342 unsigned Depth) const {
3343 unsigned BitWidth = Op.getScalarValueSizeInBits();
3344
3345 KnownBits Known(BitWidth); // Don't know anything.
3346
3347 if (auto OptAPInt = Op->bitcastToAPInt()) {
3348 // We know all of the bits for a constant!
3349 APInt V = *std::move(OptAPInt);
3350
3351 // Swap the low-order and high-order double of a ppc_fp128 when casting to
3352 // i128, see #44482.
3353 //
3354 // A ppc_fp128 is two doubles, with the high-order double stored at the
3355 // lower address. Reading that as an integer therefore puts the high-order
3356 // double in the high 64 bits on big-endian targets and in the low 64 bits
3357 // on little-endian targets.
3358 //
3359 // But APFloat::bitcastToAPInt is endianness-agnostic and always places the
3360 // high-order double in the low 64 bits. Hence the two doubles must be
3361 // flipped on big-endian targets.
3362 if (getDataLayout().isBigEndian() && Op.getValueType() == MVT::ppcf128)
3363 V = V.rotl(64);
3364
3365 return KnownBits::makeConstant(V);
3366 }
3367
3368 if (Depth >= MaxRecursionDepth)
3369 return Known; // Limit search depth.
3370
3371 KnownBits Known2;
3372 unsigned NumElts = DemandedElts.getBitWidth();
3373 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3374 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3375 assert((!Op.getValueType().isFixedLengthVector() ||
3376 NumElts == Op.getValueType().getVectorNumElements()) &&
3377 "Unexpected vector size");
3378
3379 if (!DemandedElts)
3380 return Known; // No demanded elts, better to assume we don't know anything.
3381
3382 unsigned Opcode = Op.getOpcode();
3383 switch (Opcode) {
3384 case ISD::FREEZE: {
3385 if (isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,
3387 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3388 break;
3389 }
3390 case ISD::MERGE_VALUES:
3391 return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,
3392 Depth + 1);
3393 case ISD::SPLAT_VECTOR: {
3394 SDValue SrcOp = Op.getOperand(0);
3395 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3396 "Expected SPLAT_VECTOR implicit truncation");
3397 // Implicitly truncate the bits to match the official semantics of
3398 // SPLAT_VECTOR.
3400 break;
3401 }
3403 unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();
3404 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3405 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3406 for (auto [I, SrcOp] : enumerate(Op->ops())) {
3407 Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);
3408 }
3409 break;
3410 }
3411 case ISD::STEP_VECTOR: {
3412 const APInt &Step = Op.getConstantOperandAPInt(0);
3413
3414 if (Step.isPowerOf2())
3415 Known.Zero.setLowBits(Step.logBase2());
3416
3418
3419 if (!isUIntN(BitWidth, Op.getValueType().getVectorMinNumElements()))
3420 break;
3421 const APInt MinNumElts =
3422 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3423
3424 bool Overflow;
3425 const APInt MaxNumElts = getVScaleRange(&F, BitWidth)
3427 .umul_ov(MinNumElts, Overflow);
3428 if (Overflow)
3429 break;
3430
3431 const APInt MaxValue = (MaxNumElts - 1).umul_ov(Step, Overflow);
3432 if (Overflow)
3433 break;
3434
3435 Known.Zero.setHighBits(MaxValue.countl_zero());
3436 break;
3437 }
3438 case ISD::BUILD_VECTOR:
3439 assert(!Op.getValueType().isScalableVector());
3440 // Collect the known bits that are shared by every demanded vector element.
3441 Known.setAllConflict();
3442 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3443 if (!DemandedElts[i])
3444 continue;
3445
3446 SDValue SrcOp = Op.getOperand(i);
3447 if (SrcOp.getOpcode() == ISD::POISON)
3448 continue;
3449
3450 Known2 = computeKnownBits(SrcOp, Depth + 1);
3451
3452 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3453 if (SrcOp.getValueSizeInBits() != BitWidth) {
3454 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3455 "Expected BUILD_VECTOR implicit truncation");
3456 Known2 = Known2.trunc(BitWidth);
3457 }
3458
3459 // Known bits are the values that are shared by every demanded element.
3460 Known = Known.intersectWith(Known2);
3461
3462 // If we don't know any bits, early out.
3463 if (Known.isUnknown())
3464 break;
3465 }
3466
3467 // If every demanded element was poison, we know nothing.
3468 if (Known.hasConflict())
3469 Known.resetAll();
3470 break;
3471 case ISD::VECTOR_COMPRESS: {
3472 SDValue Vec = Op.getOperand(0);
3473 SDValue PassThru = Op.getOperand(2);
3474 Known = computeKnownBits(PassThru, DemandedElts, Depth + 1);
3475 // If we don't know any bits, early out.
3476 if (Known.isUnknown())
3477 break;
3478 Known2 = computeKnownBits(Vec, Depth + 1);
3479 Known = Known.intersectWith(Known2);
3480 break;
3481 }
3482 case ISD::VECTOR_SHUFFLE: {
3483 assert(!Op.getValueType().isScalableVector());
3484 // Collect the known bits that are shared by every vector element referenced
3485 // by the shuffle.
3486 APInt DemandedLHS, DemandedRHS;
3488 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3489 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
3490 DemandedLHS, DemandedRHS))
3491 break;
3492
3493 // Known bits are the values that are shared by every demanded element.
3494 Known.setAllConflict();
3495 if (!!DemandedLHS) {
3496 SDValue LHS = Op.getOperand(0);
3497 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
3498 Known = Known.intersectWith(Known2);
3499 }
3500 // If we don't know any bits, early out.
3501 if (Known.isUnknown())
3502 break;
3503 if (!!DemandedRHS) {
3504 SDValue RHS = Op.getOperand(1);
3505 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
3506 Known = Known.intersectWith(Known2);
3507 }
3508 break;
3509 }
3510 case ISD::VSCALE: {
3512 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
3514 break;
3515 }
3516 case ISD::CONCAT_VECTORS: {
3517 if (Op.getValueType().isScalableVector())
3518 break;
3519 // Split DemandedElts and test each of the demanded subvectors.
3520 Known.setAllConflict();
3521 EVT SubVectorVT = Op.getOperand(0).getValueType();
3522 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3523 unsigned NumSubVectors = Op.getNumOperands();
3524 for (unsigned i = 0; i != NumSubVectors; ++i) {
3525 APInt DemandedSub =
3526 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
3527 if (!!DemandedSub) {
3528 SDValue Sub = Op.getOperand(i);
3529 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
3530 Known = Known.intersectWith(Known2);
3531 }
3532 // If we don't know any bits, early out.
3533 if (Known.isUnknown())
3534 break;
3535 }
3536 break;
3537 }
3538 case ISD::INSERT_SUBVECTOR: {
3539 if (Op.getValueType().isScalableVector())
3540 break;
3541 // Demand any elements from the subvector and the remainder from the src its
3542 // inserted into.
3543 SDValue Src = Op.getOperand(0);
3544 SDValue Sub = Op.getOperand(1);
3545 uint64_t Idx = Op.getConstantOperandVal(2);
3546 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3547 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3548 APInt DemandedSrcElts = DemandedElts;
3549 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
3550
3551 Known.setAllConflict();
3552 if (!!DemandedSubElts) {
3553 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
3554 if (Known.isUnknown())
3555 break; // early-out.
3556 }
3557 if (!!DemandedSrcElts) {
3558 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3559 Known = Known.intersectWith(Known2);
3560 }
3561 break;
3562 }
3564 // Offset the demanded elts by the subvector index.
3565 SDValue Src = Op.getOperand(0);
3566
3567 APInt DemandedSrcElts;
3568 if (Src.getValueType().isScalableVector())
3569 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3570 else {
3571 uint64_t Idx = Op.getConstantOperandVal(1);
3572 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3573 DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3574 }
3575 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3576 break;
3577 }
3578 case ISD::SCALAR_TO_VECTOR: {
3579 if (Op.getValueType().isScalableVector())
3580 break;
3581 // We know about scalar_to_vector as much as we know about it source,
3582 // which becomes the first element of otherwise unknown vector.
3583 if (DemandedElts != 1)
3584 break;
3585
3586 SDValue N0 = Op.getOperand(0);
3587 Known = computeKnownBits(N0, Depth + 1);
3588 if (N0.getValueSizeInBits() != BitWidth)
3589 Known = Known.trunc(BitWidth);
3590
3591 break;
3592 }
3593 case ISD::BITCAST: {
3594 if (Op.getValueType().isScalableVector())
3595 break;
3596
3597 SDValue N0 = Op.getOperand(0);
3598 EVT SubVT = N0.getValueType();
3599 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3600
3601 // Ignore bitcasts from unsupported types.
3602 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3603 break;
3604
3605 // Fast handling of 'identity' bitcasts.
3606 if (BitWidth == SubBitWidth) {
3607 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
3608 break;
3609 }
3610
3611 bool IsLE = getDataLayout().isLittleEndian();
3612
3613 // Bitcast 'small element' vector to 'large element' scalar/vector.
3614 if ((BitWidth % SubBitWidth) == 0) {
3615 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3616
3617 // Collect known bits for the (larger) output by collecting the known
3618 // bits from each set of sub elements and shift these into place.
3619 // We need to separately call computeKnownBits for each set of
3620 // sub elements as the knownbits for each is likely to be different.
3621 unsigned SubScale = BitWidth / SubBitWidth;
3622 APInt SubDemandedElts(NumElts * SubScale, 0);
3623 for (unsigned i = 0; i != NumElts; ++i)
3624 if (DemandedElts[i])
3625 SubDemandedElts.setBit(i * SubScale);
3626
3627 for (unsigned i = 0; i != SubScale; ++i) {
3628 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
3629 Depth + 1);
3630 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3631 Known.insertBits(Known2, SubBitWidth * Shifts);
3632 }
3633 }
3634
3635 // Bitcast 'large element' scalar/vector to 'small element' vector.
3636 if ((SubBitWidth % BitWidth) == 0) {
3637 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3638
3639 // Collect known bits for the (smaller) output by collecting the known
3640 // bits from the overlapping larger input elements and extracting the
3641 // sub sections we actually care about.
3642 unsigned SubScale = SubBitWidth / BitWidth;
3643 APInt SubDemandedElts =
3644 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
3645 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
3646
3647 Known.setAllConflict();
3648 for (unsigned i = 0; i != NumElts; ++i)
3649 if (DemandedElts[i]) {
3650 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3651 unsigned Offset = (Shifts % SubScale) * BitWidth;
3652 Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));
3653 // If we don't know any bits, early out.
3654 if (Known.isUnknown())
3655 break;
3656 }
3657 }
3658 break;
3659 }
3660 case ISD::AND:
3661 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3662 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3663
3664 Known &= Known2;
3665 break;
3666 case ISD::OR:
3667 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3668 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3669
3670 Known |= Known2;
3671 break;
3672 case ISD::XOR:
3673 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3674 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3675
3676 Known ^= Known2;
3677 break;
3678 case ISD::MUL: {
3679 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3680 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3681 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3682 // TODO: SelfMultiply can be poison, but not undef.
3683 if (SelfMultiply)
3684 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3685 Op.getOperand(0), DemandedElts, UndefPoisonKind::UndefOrPoison,
3686 Depth + 1);
3687 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3688
3689 // If the multiplication is known not to overflow, the product of a number
3690 // with itself is non-negative. Only do this if we didn't already computed
3691 // the opposite value for the sign bit.
3692 if (Op->getFlags().hasNoSignedWrap() &&
3693 Op.getOperand(0) == Op.getOperand(1) &&
3694 !Known.isNegative())
3695 Known.makeNonNegative();
3696 break;
3697 }
3698 case ISD::MULHU: {
3699 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3700 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3701 Known = KnownBits::mulhu(Known, Known2);
3702 break;
3703 }
3704 case ISD::MULHS: {
3705 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3706 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3707 Known = KnownBits::mulhs(Known, Known2);
3708 break;
3709 }
3710 case ISD::ABDU: {
3711 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3712 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3713 Known = KnownBits::abdu(Known, Known2);
3714 break;
3715 }
3716 case ISD::ABDS: {
3717 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3718 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3719 Known = KnownBits::abds(Known, Known2);
3720 unsigned SignBits1 =
3721 ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3722 if (SignBits1 == 1)
3723 break;
3724 unsigned SignBits0 =
3725 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3726 Known.Zero.setHighBits(std::min(SignBits0, SignBits1) - 1);
3727 break;
3728 }
3729 case ISD::UMUL_LOHI: {
3730 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3731 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3732 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3733 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3734 if (Op.getResNo() == 0)
3735 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3736 else
3737 Known = KnownBits::mulhu(Known, Known2);
3738 break;
3739 }
3740 case ISD::SMUL_LOHI: {
3741 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3742 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3743 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3744 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3745 if (Op.getResNo() == 0)
3746 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3747 else
3748 Known = KnownBits::mulhs(Known, Known2);
3749 break;
3750 }
3751 case ISD::AVGFLOORU: {
3752 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3753 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3754 Known = KnownBits::avgFloorU(Known, Known2);
3755 break;
3756 }
3757 case ISD::AVGCEILU: {
3758 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3759 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3760 Known = KnownBits::avgCeilU(Known, Known2);
3761 break;
3762 }
3763 case ISD::AVGFLOORS: {
3764 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3765 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3766 Known = KnownBits::avgFloorS(Known, Known2);
3767 break;
3768 }
3769 case ISD::AVGCEILS: {
3770 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3771 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3772 Known = KnownBits::avgCeilS(Known, Known2);
3773 break;
3774 }
3775 case ISD::AVGSUBFLOORU:
3776 case ISD::AVGSUBFLOORS: {
3777 bool IsSigned = Op.getOpcode() == ISD::AVGSUBFLOORS;
3778 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3779 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3780 // Subtract in i[N+1] so the difference can't overflow, then halve.
3781 Known = IsSigned ? Known.sext(BitWidth + 1) : Known.zext(BitWidth + 1);
3782 Known2 = IsSigned ? Known2.sext(BitWidth + 1) : Known2.zext(BitWidth + 1);
3783 Known = KnownBits::sub(Known, Known2);
3784 // The shifted in top bit is unknown, but it is truncated away.
3785 Known >>= 1;
3786 Known = Known.trunc(BitWidth);
3787 break;
3788 }
3789 case ISD::SELECT:
3790 case ISD::VSELECT:
3791 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3792 // If we don't know any bits, early out.
3793 if (Known.isUnknown())
3794 break;
3795 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3796
3797 // Only known if known in both the LHS and RHS.
3798 Known = Known.intersectWith(Known2);
3799 break;
3800 case ISD::SELECT_CC:
3801 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3802 // If we don't know any bits, early out.
3803 if (Known.isUnknown())
3804 break;
3805 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3806
3807 // Only known if known in both the LHS and RHS.
3808 Known = Known.intersectWith(Known2);
3809 break;
3810 case ISD::SMULO:
3811 case ISD::UMULO:
3812 if (Op.getResNo() != 1)
3813 break;
3814 // The boolean result conforms to getBooleanContents.
3815 // If we know the result of a setcc has the top bits zero, use this info.
3816 // We know that we have an integer-based boolean since these operations
3817 // are only available for integer.
3818 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3820 BitWidth > 1)
3821 Known.Zero.setBitsFrom(1);
3822 break;
3823 case ISD::SETCC:
3824 case ISD::SETCCCARRY:
3825 case ISD::STRICT_FSETCC:
3826 case ISD::STRICT_FSETCCS: {
3827 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3828 // If we know the result of a setcc has the top bits zero, use this info.
3829 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3831 BitWidth > 1)
3832 Known.Zero.setBitsFrom(1);
3833 break;
3834 }
3835 case ISD::SHL: {
3836 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3837 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3838
3839 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3840 bool NSW = Op->getFlags().hasNoSignedWrap();
3841
3842 bool ShAmtNonZero = Known2.isNonZero();
3843
3844 Known = KnownBits::shl(Known, Known2, NUW, NSW, ShAmtNonZero);
3845
3846 // Minimum shift low bits are known zero.
3847 if (std::optional<unsigned> ShMinAmt =
3848 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))
3849 Known.Zero.setLowBits(*ShMinAmt);
3850 break;
3851 }
3852 case ISD::SRL:
3853 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3854 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3855 Known = KnownBits::lshr(Known, Known2, /*ShAmtNonZero=*/false,
3856 Op->getFlags().hasExact());
3857
3858 // Minimum shift high bits are known zero.
3859 if (std::optional<unsigned> ShMinAmt =
3860 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))
3861 Known.Zero.setHighBits(*ShMinAmt);
3862 break;
3863 case ISD::SRA:
3864 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3865 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3866 Known = KnownBits::ashr(Known, Known2, /*ShAmtNonZero=*/false,
3867 Op->getFlags().hasExact());
3868 break;
3869 case ISD::ROTL:
3870 case ISD::ROTR:
3871 if (ConstantSDNode *C =
3872 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3873 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3874
3875 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3876
3877 // Canonicalize to ROTR.
3878 if (Opcode == ISD::ROTL && Amt != 0)
3879 Amt = BitWidth - Amt;
3880
3881 Known.Zero = Known.Zero.rotr(Amt);
3882 Known.One = Known.One.rotr(Amt);
3883 }
3884 break;
3885 case ISD::FSHL:
3886 case ISD::FSHR:
3887 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3888 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3889
3890 // For fshl, 0-shift returns the 1st arg.
3891 // For fshr, 0-shift returns the 2nd arg.
3892 if (Amt == 0) {
3893 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3894 DemandedElts, Depth + 1);
3895 break;
3896 }
3897
3898 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3899 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3900 const APInt ShAmt(BitWidth, Amt);
3901 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3902 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3903 Known = Opcode == ISD::FSHL ? KnownBits::fshl(Known, Known2, ShAmt)
3904 : KnownBits::fshr(Known, Known2, ShAmt);
3905 }
3906 break;
3907 case ISD::SHL_PARTS:
3908 case ISD::SRA_PARTS:
3909 case ISD::SRL_PARTS: {
3910 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3911
3912 // Collect lo/hi source values and concatenate.
3913 unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();
3914 unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();
3915 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3916 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3917 Known = Known2.concat(Known);
3918
3919 // Collect shift amount.
3920 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3921
3922 if (Opcode == ISD::SHL_PARTS)
3923 Known = KnownBits::shl(Known, Known2);
3924 else if (Opcode == ISD::SRA_PARTS)
3925 Known = KnownBits::ashr(Known, Known2);
3926 else // if (Opcode == ISD::SRL_PARTS)
3927 Known = KnownBits::lshr(Known, Known2);
3928
3929 // TODO: Minimum shift low/high bits are known zero.
3930
3931 if (Op.getResNo() == 0)
3932 Known = Known.extractBits(LoBits, 0);
3933 else
3934 Known = Known.extractBits(HiBits, LoBits);
3935 break;
3936 }
3938 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3939 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3940 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3941 break;
3942 }
3943 case ISD::CTTZ:
3944 case ISD::CTTZ_ZERO_POISON: {
3945 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3946 // If we have a known 1, its position is our upper bound.
3947 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3948 unsigned LowBits = llvm::bit_width(PossibleTZ);
3949 Known.Zero.setBitsFrom(LowBits);
3950 break;
3951 }
3952 case ISD::CTLZ:
3953 case ISD::CTLZ_ZERO_POISON: {
3954 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3955 // If we have a known 1, its position is our upper bound.
3956 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3957 unsigned LowBits = llvm::bit_width(PossibleLZ);
3958 Known.Zero.setBitsFrom(LowBits);
3959 break;
3960 }
3961 case ISD::CTLS: {
3962 unsigned MinRedundantSignBits =
3963 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1) - 1;
3964 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3966 Known = Range.toKnownBits();
3967 break;
3968 }
3969 case ISD::CTPOP: {
3970 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3971 // If we know some of the bits are zero, they can't be one.
3972 unsigned PossibleOnes = Known2.countMaxPopulation();
3973 Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));
3974 break;
3975 }
3976 case ISD::PARITY: {
3977 // Parity returns 0 everywhere but the LSB.
3978 Known.Zero.setBitsFrom(1);
3979 break;
3980 }
3981 case ISD::PDEP: {
3982 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3983 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3984 Known = KnownBits::pdep(Known2, Known);
3985 break;
3986 }
3987 case ISD::PEXT: {
3988 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3989 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3990 Known = KnownBits::pext(Known2, Known);
3991 break;
3992 }
3993 case ISD::CLMUL: {
3994 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3995 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3996 Known = KnownBits::clmul(Known, Known2);
3997 break;
3998 }
3999 case ISD::MGATHER:
4000 case ISD::MLOAD: {
4001 ISD::LoadExtType ETy =
4002 (Opcode == ISD::MGATHER)
4003 ? cast<MaskedGatherSDNode>(Op)->getExtensionType()
4004 : cast<MaskedLoadSDNode>(Op)->getExtensionType();
4005 if (ETy == ISD::ZEXTLOAD) {
4006 EVT MemVT = cast<MemSDNode>(Op)->getMemoryVT();
4007 KnownBits Known0(MemVT.getScalarSizeInBits());
4008 return Known0.zext(BitWidth);
4009 }
4010 break;
4011 }
4012 case ISD::LOAD: {
4014 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
4015 if (ISD::isNON_EXTLoad(LD) && Cst) {
4016 // Determine any common known bits from the loaded constant pool value.
4017 Type *CstTy = Cst->getType();
4018 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
4019 !Op.getValueType().isScalableVector()) {
4020 // If its a vector splat, then we can (quickly) reuse the scalar path.
4021 // NOTE: We assume all elements match and none are UNDEF.
4022 if (CstTy->isVectorTy()) {
4023 if (const Constant *Splat = Cst->getSplatValue()) {
4024 Cst = Splat;
4025 CstTy = Cst->getType();
4026 }
4027 }
4028 // TODO - do we need to handle different bitwidths?
4029 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4030 // Iterate across all vector elements finding common known bits.
4031 Known.setAllConflict();
4032 for (unsigned i = 0; i != NumElts; ++i) {
4033 if (!DemandedElts[i])
4034 continue;
4035 if (Constant *Elt = Cst->getAggregateElement(i)) {
4036 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4037 const APInt &Value = CInt->getValue();
4038 Known.One &= Value;
4039 Known.Zero &= ~Value;
4040 continue;
4041 }
4042 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4043 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4044 Known.One &= Value;
4045 Known.Zero &= ~Value;
4046 continue;
4047 }
4048 }
4049 Known.One.clearAllBits();
4050 Known.Zero.clearAllBits();
4051 break;
4052 }
4053 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4054 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
4055 Known = KnownBits::makeConstant(CInt->getValue());
4056 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
4057 Known =
4058 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
4059 }
4060 }
4061 }
4062 } else if (Op.getResNo() == 0) {
4063 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4064 KnownBits KnownScalarMemory(ScalarMemorySize);
4065 if (const MDNode *MD = LD->getRanges())
4066 computeKnownBitsFromRangeMetadata(*MD, KnownScalarMemory);
4067
4068 // Extend the Known bits from memory to the size of the scalar result.
4069 if (ISD::isZEXTLoad(Op.getNode()))
4070 Known = KnownScalarMemory.zext(BitWidth);
4071 else if (ISD::isSEXTLoad(Op.getNode()))
4072 Known = KnownScalarMemory.sext(BitWidth);
4073 else if (ISD::isEXTLoad(Op.getNode()))
4074 Known = KnownScalarMemory.anyext(BitWidth);
4075 else
4076 Known = KnownScalarMemory;
4077 assert(Known.getBitWidth() == BitWidth);
4078 return Known;
4079 }
4080 break;
4081 }
4083 if (Op.getValueType().isScalableVector())
4084 break;
4085 EVT InVT = Op.getOperand(0).getValueType();
4086 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
4087 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
4088 Known = Known.zext(BitWidth);
4089 break;
4090 }
4091 case ISD::ZERO_EXTEND: {
4092 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4093 Known = Known.zext(BitWidth);
4094 break;
4095 }
4097 if (Op.getValueType().isScalableVector())
4098 break;
4099 EVT InVT = Op.getOperand(0).getValueType();
4100 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
4101 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
4102 // If the sign bit is known to be zero or one, then sext will extend
4103 // it to the top bits, else it will just zext.
4104 Known = Known.sext(BitWidth);
4105 break;
4106 }
4107 case ISD::SIGN_EXTEND: {
4108 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4109 // If the sign bit is known to be zero or one, then sext will extend
4110 // it to the top bits, else it will just zext.
4111 Known = Known.sext(BitWidth);
4112 break;
4113 }
4115 if (Op.getValueType().isScalableVector())
4116 break;
4117 EVT InVT = Op.getOperand(0).getValueType();
4118 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
4119 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
4120 Known = Known.anyext(BitWidth);
4121 break;
4122 }
4123 case ISD::ANY_EXTEND: {
4124 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4125 Known = Known.anyext(BitWidth);
4126 break;
4127 }
4128 case ISD::TRUNCATE: {
4129 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4130 Known = Known.trunc(BitWidth);
4131 break;
4132 }
4133 case ISD::TRUNCATE_SSAT_S: {
4134 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4135 Known = Known.truncSSat(BitWidth);
4136 break;
4137 }
4138 case ISD::TRUNCATE_SSAT_U: {
4139 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4140 Known = Known.truncSSatU(BitWidth);
4141 break;
4142 }
4143 case ISD::TRUNCATE_USAT_U: {
4144 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4145 Known = Known.truncUSat(BitWidth);
4146 break;
4147 }
4148 case ISD::AssertZext: {
4149 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4151 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4152 Known.Zero |= (~InMask);
4153 Known.One &= (~Known.Zero);
4154 break;
4155 }
4156 case ISD::AssertAlign: {
4157 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
4158 assert(LogOfAlign != 0);
4159
4160 // TODO: Should use maximum with source
4161 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4162 // well as clearing one bits.
4163 Known.Zero.setLowBits(LogOfAlign);
4164 Known.One.clearLowBits(LogOfAlign);
4165 break;
4166 }
4167 case ISD::AssertNoFPClass: {
4168 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4169
4170 FPClassTest NoFPClass =
4171 static_cast<FPClassTest>(Op.getConstantOperandVal(1));
4172 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4173 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4174 // Cannot be negative.
4175 Known.makeNonNegative();
4176 }
4177
4178 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4179 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4180 // Cannot be positive.
4181 Known.makeNegative();
4182 }
4183
4184 break;
4185 }
4186 case ISD::FABS:
4187 // fabs clears the sign bit
4188 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4189 Known.Zero.setSignBit();
4190 Known.One.clearSignBit();
4191 break;
4192 case ISD::FGETSIGN:
4193 // All bits are zero except the low bit.
4194 Known.Zero.setBitsFrom(1);
4195 break;
4196 case ISD::ADD: {
4197 SDNodeFlags Flags = Op.getNode()->getFlags();
4198 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4199 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4200 bool SelfAdd = Op.getOperand(0) == Op.getOperand(1) &&
4202 Op.getOperand(0), DemandedElts,
4204 Known = KnownBits::add(Known, Known2, Flags.hasNoSignedWrap(),
4205 Flags.hasNoUnsignedWrap(), SelfAdd);
4206 break;
4207 }
4208 case ISD::SUB: {
4209 SDNodeFlags Flags = Op.getNode()->getFlags();
4210 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4211 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4212 Known = KnownBits::sub(Known, Known2, Flags.hasNoSignedWrap(),
4213 Flags.hasNoUnsignedWrap());
4214 break;
4215 }
4216 case ISD::USUBO:
4217 case ISD::SSUBO:
4218 case ISD::USUBO_CARRY:
4219 case ISD::SSUBO_CARRY:
4220 if (Op.getResNo() == 1) {
4221 // If we know the result of a setcc has the top bits zero, use this info.
4222 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
4224 BitWidth > 1)
4225 Known.Zero.setBitsFrom(1);
4226 break;
4227 }
4228 [[fallthrough]];
4229 case ISD::SUBC: {
4230 assert(Op.getResNo() == 0 &&
4231 "We only compute knownbits for the difference here.");
4232
4233 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4234 KnownBits Borrow(1);
4235 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4236 Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
4237 // Borrow has bit width 1
4238 Borrow = Borrow.trunc(1);
4239 } else {
4240 Borrow.setAllZero();
4241 }
4242
4243 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4244 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4245 Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
4246 break;
4247 }
4248 case ISD::UADDO:
4249 case ISD::SADDO:
4250 case ISD::UADDO_CARRY:
4251 case ISD::SADDO_CARRY:
4252 if (Op.getResNo() == 1) {
4253 // If we know the result of a setcc has the top bits zero, use this info.
4254 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
4256 BitWidth > 1)
4257 Known.Zero.setBitsFrom(1);
4258 break;
4259 }
4260 [[fallthrough]];
4261 case ISD::ADDC:
4262 case ISD::ADDE: {
4263 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4264
4265 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4266 KnownBits Carry(1);
4267 if (Opcode == ISD::ADDE)
4268 // Can't track carry from glue, set carry to unknown.
4269 Carry.resetAll();
4270 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4271 Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
4272 // Carry has bit width 1
4273 Carry = Carry.trunc(1);
4274 } else {
4275 Carry.setAllZero();
4276 }
4277
4278 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4279 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4280 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
4281 break;
4282 }
4283 case ISD::UDIV: {
4284 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4285 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4286 Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());
4287 break;
4288 }
4289 case ISD::SDIV: {
4290 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4291 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4292 Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());
4293 break;
4294 }
4295 case ISD::SREM: {
4296 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4297 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4298 Known = KnownBits::srem(Known, Known2);
4299 break;
4300 }
4301 case ISD::UREM: {
4302 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4303 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4304 Known = KnownBits::urem(Known, Known2);
4305 break;
4306 }
4307 case ISD::EXTRACT_ELEMENT: {
4308 Known = computeKnownBits(Op.getOperand(0), Depth+1);
4309 const unsigned Index = Op.getConstantOperandVal(1);
4310 const unsigned EltBitWidth = Op.getValueSizeInBits();
4311
4312 Known = Known.extractBits(EltBitWidth, Index * EltBitWidth);
4313 break;
4314 }
4316 SDValue InVec = Op.getOperand(0);
4317 SDValue EltNo = Op.getOperand(1);
4318 EVT VecVT = InVec.getValueType();
4319 // computeKnownBits not yet implemented for scalable vectors.
4320 if (VecVT.isScalableVector())
4321 break;
4322 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4323 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4324
4325 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4326 // anything about the extended bits.
4327 if (BitWidth > EltBitWidth)
4328 Known = Known.trunc(EltBitWidth);
4329
4330 // If we know the element index, just demand that vector element, else for
4331 // an unknown element index, ignore DemandedElts and demand them all.
4332 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4333 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4334 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4335 DemandedSrcElts =
4336 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4337
4338 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
4339 if (BitWidth > EltBitWidth)
4340 Known = Known.anyext(BitWidth);
4341 break;
4342 }
4343 case ISD::BUILD_PAIR: {
4344 // Operand 0 is the low half and operand 1 the high half,
4345 // KnownBits::concat places its argument in the low bits.
4346 Known = computeKnownBits(Op.getOperand(0), Depth + 1);
4347 Known2 = computeKnownBits(Op.getOperand(1), Depth + 1);
4348 Known = Known2.concat(Known);
4349 break;
4350 }
4352 if (Op.getValueType().isScalableVector())
4353 break;
4354
4355 // If we know the element index, split the demand between the
4356 // source vector and the inserted element, otherwise assume we need
4357 // the original demanded vector elements and the value.
4358 SDValue InVec = Op.getOperand(0);
4359 SDValue InVal = Op.getOperand(1);
4360 SDValue EltNo = Op.getOperand(2);
4361 bool DemandedVal = true;
4362 APInt DemandedVecElts = DemandedElts;
4363 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4364 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4365 unsigned EltIdx = CEltNo->getZExtValue();
4366 DemandedVal = !!DemandedElts[EltIdx];
4367 DemandedVecElts.clearBit(EltIdx);
4368 }
4369 Known.setAllConflict();
4370 if (DemandedVal) {
4371 Known2 = computeKnownBits(InVal, Depth + 1);
4372 Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));
4373 }
4374 if (!!DemandedVecElts) {
4375 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
4376 Known = Known.intersectWith(Known2);
4377 }
4378 break;
4379 }
4380 case ISD::BITREVERSE: {
4381 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4382 Known = Known2.reverseBits();
4383 break;
4384 }
4385 case ISD::BSWAP: {
4386 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4387 Known = Known2.byteSwap();
4388 break;
4389 }
4390 case ISD::ABS:
4391 case ISD::ABS_MIN_POISON: {
4392 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4393 Known = Known2.abs();
4394 Known.Zero.setHighBits(
4395 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1) - 1);
4396 break;
4397 }
4398 case ISD::USUBSAT: {
4399 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4400 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4401 Known = KnownBits::usub_sat(Known, Known2);
4402 break;
4403 }
4404 case ISD::UMIN: {
4405 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4406 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4407 Known = KnownBits::umin(Known, Known2);
4408 break;
4409 }
4410 case ISD::UMAX: {
4411 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4412 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4413 Known = KnownBits::umax(Known, Known2);
4414 break;
4415 }
4416 case ISD::SMIN:
4417 case ISD::SMAX: {
4418 // If we have a clamp pattern, we know that the number of sign bits will be
4419 // the minimum of the clamp min/max range.
4420 bool IsMax = (Opcode == ISD::SMAX);
4421 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4422 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4423 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4424 CstHigh =
4425 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4426 if (CstLow && CstHigh) {
4427 if (!IsMax)
4428 std::swap(CstLow, CstHigh);
4429
4430 const APInt &ValueLow = CstLow->getAPIntValue();
4431 const APInt &ValueHigh = CstHigh->getAPIntValue();
4432 if (ValueLow.sle(ValueHigh)) {
4433 unsigned LowSignBits = ValueLow.getNumSignBits();
4434 unsigned HighSignBits = ValueHigh.getNumSignBits();
4435 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
4436 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4437 Known.One.setHighBits(MinSignBits);
4438 break;
4439 }
4440 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4441 Known.Zero.setHighBits(MinSignBits);
4442 break;
4443 }
4444 }
4445 }
4446
4447 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4448 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4449 if (IsMax)
4450 Known = KnownBits::smax(Known, Known2);
4451 else
4452 Known = KnownBits::smin(Known, Known2);
4453
4454 // For SMAX, if CstLow is non-negative we know the result will be
4455 // non-negative and thus all sign bits are 0.
4456 // TODO: There's an equivalent of this for smin with negative constant for
4457 // known ones.
4458 if (IsMax && CstLow) {
4459 const APInt &ValueLow = CstLow->getAPIntValue();
4460 if (ValueLow.isNonNegative()) {
4461 unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4462 Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));
4463 }
4464 }
4465
4466 break;
4467 }
4468 case ISD::UINT_TO_FP: {
4469 Known.makeNonNegative();
4470 break;
4471 }
4472 case ISD::SINT_TO_FP: {
4473 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4474 if (Known2.isNonNegative())
4475 Known.makeNonNegative();
4476 else if (Known2.isNegative())
4477 Known.makeNegative();
4478 break;
4479 }
4480 case ISD::FP_TO_UINT_SAT: {
4481 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4482 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4484 break;
4485 }
4486 case ISD::ATOMIC_LOAD: {
4487 // If we are looking at the loaded value.
4488 if (Op.getResNo() == 0) {
4489 auto *AT = cast<AtomicSDNode>(Op);
4490 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4491 KnownBits KnownScalarMemory(ScalarMemorySize);
4492 if (const MDNode *MD = AT->getRanges())
4493 computeKnownBitsFromRangeMetadata(*MD, KnownScalarMemory);
4494
4495 switch (AT->getExtensionType()) {
4496 case ISD::ZEXTLOAD:
4497 Known = KnownScalarMemory.zext(BitWidth);
4498 break;
4499 case ISD::SEXTLOAD:
4500 Known = KnownScalarMemory.sext(BitWidth);
4501 break;
4502 case ISD::EXTLOAD:
4503 switch (TLI->getExtendForAtomicOps()) {
4504 case ISD::ZERO_EXTEND:
4505 Known = KnownScalarMemory.zext(BitWidth);
4506 break;
4507 case ISD::SIGN_EXTEND:
4508 Known = KnownScalarMemory.sext(BitWidth);
4509 break;
4510 default:
4511 Known = KnownScalarMemory.anyext(BitWidth);
4512 break;
4513 }
4514 break;
4515 case ISD::NON_EXTLOAD:
4516 Known = KnownScalarMemory;
4517 break;
4518 }
4519 assert(Known.getBitWidth() == BitWidth);
4520 }
4521 break;
4522 }
4524 if (Op.getResNo() == 1) {
4525 // The boolean result conforms to getBooleanContents.
4526 // If we know the result of a setcc has the top bits zero, use this info.
4527 // We know that we have an integer-based boolean since these operations
4528 // are only available for integer.
4529 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
4531 BitWidth > 1)
4532 Known.Zero.setBitsFrom(1);
4533 break;
4534 }
4535 [[fallthrough]];
4537 case ISD::ATOMIC_SWAP:
4548 case ISD::ATOMIC_LOAD_UMAX: {
4549 // If we are looking at the loaded value.
4550 if (Op.getResNo() == 0) {
4551 auto *AT = cast<AtomicSDNode>(Op);
4552 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4553
4554 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4555 Known.Zero.setBitsFrom(MemBits);
4556 }
4557 break;
4558 }
4559 case ISD::FrameIndex:
4560 case ISD::TargetFrameIndex: {
4561 const MachineFunction &MF = getMachineFunction();
4562 int FrameIdx = cast<FrameIndexSDNode>(Op)->getIndex();
4563 TLI->computeKnownBitsForStackObjectPointer(
4564 Known, MF, MF.getFrameInfo().getObjectAlign(FrameIdx));
4565 break;
4566 }
4567 case ISD::VP_LOAD_FF: {
4568 if (Op.getResNo() != 1)
4569 break;
4570 // The second result of vp.load.ff is an unsigned value that is less than or
4571 // equal to the EVL operand.
4572 KnownBits VLKB =
4573 computeKnownBits(Op.getOperand(3), DemandedElts, Depth + 1);
4574 // The new VL is also bounded by the largest vector length.
4575 EVT ResVT = Op->getValueType(0);
4576 auto ResKB = KnownBits::makeConstant(
4578 if (ResVT.isScalableVector()) {
4580 ResKB = KnownBits::mul(getVScaleRange(&F, BitWidth).toKnownBits(), ResKB);
4581 }
4582 Known.Zero.setHighBits(KnownBits::umin(VLKB, ResKB).countMinLeadingZeros());
4583 break;
4584 }
4585
4586 default:
4587 if (Opcode < ISD::BUILTIN_OP_END)
4588 break;
4589 [[fallthrough]];
4593 // Allow the target to implement this method for its nodes.
4594 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
4595 break;
4596 }
4597
4598 return Known;
4599}
4600
4601/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4614
4617 // X + 0 never overflow
4618 if (isNullConstant(N1))
4619 return OFK_Never;
4620
4621 // If both operands each have at least two sign bits, the addition
4622 // cannot overflow.
4623 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4624 return OFK_Never;
4625
4626 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4627 return OFK_Sometime;
4628}
4629
4632 // X + 0 never overflow
4633 if (isNullConstant(N1))
4634 return OFK_Never;
4635
4636 // mulhi + 1 never overflow
4637 KnownBits N1Known = computeKnownBits(N1);
4638 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4639 N1Known.getMaxValue().ult(2))
4640 return OFK_Never;
4641
4642 KnownBits N0Known = computeKnownBits(N0);
4643 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4644 N0Known.getMaxValue().ult(2))
4645 return OFK_Never;
4646
4647 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4648 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4649 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4650 return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));
4651}
4652
4655 // X - 0 never overflow
4656 if (isNullConstant(N1))
4657 return OFK_Never;
4658
4659 // If both operands each have at least two sign bits, the subtraction
4660 // cannot overflow.
4661 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4662 return OFK_Never;
4663
4664 KnownBits N0Known = computeKnownBits(N0);
4665 KnownBits N1Known = computeKnownBits(N1);
4666 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);
4667 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);
4668 return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));
4669}
4670
4673 // X - 0 never overflow
4674 if (isNullConstant(N1))
4675 return OFK_Never;
4676
4677 ConstantRange N0Range =
4678 computeConstantRangeIncludingKnownBits(N0, /*ForSigned=*/false);
4679 ConstantRange N1Range =
4680 computeConstantRangeIncludingKnownBits(N1, /*ForSigned=*/false);
4681 return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));
4682}
4683
4686 // X * 0 and X * 1 never overflow.
4687 if (isNullConstant(N1) || isOneConstant(N1))
4688 return OFK_Never;
4689
4692 return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));
4693}
4694
4697 // X * 0 and X * 1 never overflow.
4698 if (isNullConstant(N1) || isOneConstant(N1))
4699 return OFK_Never;
4700
4701 // Get the size of the result.
4702 unsigned BitWidth = N0.getScalarValueSizeInBits();
4703
4704 // Sum of the sign bits.
4705 unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);
4706
4707 // If we have enough sign bits, then there's no overflow.
4708 if (SignBits > BitWidth + 1)
4709 return OFK_Never;
4710
4711 if (SignBits == BitWidth + 1) {
4712 // The overflow occurs when the true multiplication of the
4713 // the operands is the minimum negative number.
4714 KnownBits N0Known = computeKnownBits(N0);
4715 KnownBits N1Known = computeKnownBits(N1);
4716 // If one of the operands is non-negative, then there's no
4717 // overflow.
4718 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4719 return OFK_Never;
4720 }
4721
4722 return OFK_Sometime;
4723}
4724
4726 unsigned Depth) const {
4727 APInt DemandedElts = getDemandAllEltsMask(Op);
4728 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4729}
4730
4732 const APInt &DemandedElts,
4733 bool ForSigned,
4734 unsigned Depth) const {
4735 EVT VT = Op.getValueType();
4736 unsigned BitWidth = VT.getScalarSizeInBits();
4737
4738 if (Depth >= MaxRecursionDepth)
4739 return ConstantRange::getFull(BitWidth);
4740
4741 if (ConstantSDNode *C = isConstOrConstSplat(Op, DemandedElts))
4742 return ConstantRange(C->getAPIntValue());
4743
4744 unsigned Opcode = Op.getOpcode();
4745 switch (Opcode) {
4746 case ISD::VSCALE: {
4748 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
4749 return getVScaleRange(&F, BitWidth).multiply(Multiplier);
4750 }
4751 default:
4752 break;
4753 }
4754
4755 return ConstantRange::getFull(BitWidth);
4756}
4757
4760 unsigned Depth) const {
4761 APInt DemandedElts = getDemandAllEltsMask(Op);
4762 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4763 Depth);
4764}
4765
4767 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4768 unsigned Depth) const {
4769 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4771 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4774 return CR1.intersectWith(CR2, RangeType);
4775}
4776
4778 unsigned Depth) const {
4779 APInt DemandedElts = getDemandAllEltsMask(Val);
4780 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4781}
4782
4784 const APInt &DemandedElts,
4785 bool OrZero, unsigned Depth) const {
4786 if (Depth >= MaxRecursionDepth)
4787 return false; // Limit search depth.
4788
4789 EVT OpVT = Val.getValueType();
4790 unsigned BitWidth = OpVT.getScalarSizeInBits();
4791 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4792 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4793 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4794 assert(
4795 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4796 "Unexpected vector size");
4797
4798 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4799 APInt V = C->getAPIntValue().zextOrTrunc(BitWidth);
4800 return (OrZero && V.isZero()) || V.isPowerOf2();
4801 };
4802
4803 // Is the constant a known power of 2 or zero?
4804 if (ISD::matchUnaryPredicate(Val, DemandedElts, IsPowerOfTwoOrZero,
4805 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
4806 return true;
4807
4808 switch (Val.getOpcode()) {
4810 SDValue InVec = Val.getOperand(0);
4811 SDValue EltNo = Val.getOperand(1);
4812 EVT VecVT = InVec.getValueType();
4813
4814 // Skip scalable vectors or implicit extensions.
4815 if (VecVT.isScalableVector() ||
4816 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4817 break;
4818
4819 // If we know the element index, just demand that vector element, else for
4820 // an unknown element index, ignore DemandedElts and demand them all.
4821 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4822 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4823 APInt DemandedSrcElts =
4824 ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)
4825 ? APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue())
4826 : APInt::getAllOnes(NumSrcElts);
4827 return isKnownToBeAPowerOfTwo(InVec, DemandedSrcElts, OrZero, Depth + 1);
4828 }
4829
4830 case ISD::AND: {
4831 // Looking for `x & -x` pattern:
4832 // If x == 0:
4833 // x & -x -> 0
4834 // If x != 0:
4835 // x & -x -> non-zero pow2
4836 // so if we find the pattern return whether we know `x` is non-zero.
4837 SDValue X, Z;
4838 if (sd_match(Val, m_And(m_Value(X), m_Neg(m_Deferred(X)))) ||
4839 (sd_match(Val, m_And(m_Value(X), m_Sub(m_Value(Z), m_Deferred(X)))) &&
4840 MaskedVectorIsZero(Z, DemandedElts, Depth + 1)))
4841 return OrZero || isKnownNeverZero(X, DemandedElts, Depth);
4842 break;
4843 }
4844
4845 case ISD::SHL: {
4846 // A left-shift of a constant one will have exactly one bit set because
4847 // shifting the bit off the end is undefined.
4848 auto *C = isConstOrConstSplat(Val.getOperand(0), DemandedElts);
4849 if (C && C->getAPIntValue() == 1)
4850 return true;
4851 return (OrZero || isKnownNeverZero(Val, DemandedElts, Depth)) &&
4852 isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4853 Depth + 1);
4854 }
4855
4856 case ISD::SRL: {
4857 // A logical right-shift of a constant sign-bit will have exactly
4858 // one bit set.
4859 auto *C = isConstOrConstSplat(Val.getOperand(0), DemandedElts);
4860 if (C && C->getAPIntValue().isSignMask())
4861 return true;
4862 return (OrZero || isKnownNeverZero(Val, DemandedElts, Depth)) &&
4863 isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4864 Depth + 1);
4865 }
4866
4867 case ISD::TRUNCATE:
4868 return (OrZero || isKnownNeverZero(Val, DemandedElts, Depth)) &&
4869 isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4870 Depth + 1);
4871
4872 case ISD::ROTL:
4873 case ISD::ROTR:
4874 return isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4875 Depth + 1);
4876 case ISD::BSWAP:
4877 case ISD::BITREVERSE:
4878 return isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4879 Depth + 1);
4880
4881 case ISD::SMIN:
4882 case ISD::SMAX:
4883 case ISD::UMIN:
4884 case ISD::UMAX:
4885 return isKnownToBeAPowerOfTwo(Val.getOperand(1), DemandedElts, OrZero,
4886 Depth + 1) &&
4887 isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4888 Depth + 1);
4889
4890 case ISD::SELECT:
4891 case ISD::VSELECT:
4892 return isKnownToBeAPowerOfTwo(Val.getOperand(2), DemandedElts, OrZero,
4893 Depth + 1) &&
4894 isKnownToBeAPowerOfTwo(Val.getOperand(1), DemandedElts, OrZero,
4895 Depth + 1);
4896
4897 case ISD::ZERO_EXTEND:
4898 return isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedElts, OrZero,
4899 Depth + 1);
4900
4901 case ISD::VSCALE:
4902 // vscale(power-of-two) is a power-of-two
4903 return isKnownToBeAPowerOfTwo(Val.getOperand(0), /*OrZero=*/false,
4904 Depth + 1);
4905
4906 case ISD::VECTOR_SHUFFLE: {
4908 // Demanded elements with undef shuffle mask elements are unknown
4909 // - we cannot guarantee they are a power of two, so return false.
4910 APInt DemandedLHS, DemandedRHS;
4912 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4913 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
4914 DemandedLHS, DemandedRHS))
4915 return false;
4916
4917 // All demanded elements from LHS must be known power of two.
4918 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val.getOperand(0), DemandedLHS,
4919 OrZero, Depth + 1))
4920 return false;
4921
4922 // All demanded elements from RHS must be known power of two.
4923 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val.getOperand(1), DemandedRHS,
4924 OrZero, Depth + 1))
4925 return false;
4926
4927 return true;
4928 }
4929 }
4930
4931 // More could be done here, though the above checks are enough
4932 // to handle some common cases.
4933 return false;
4934}
4935
4937 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Val, true))
4938 return C1->getValueAPF().getExactLog2Abs() >= 0;
4939
4940 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4941 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4942
4943 return false;
4944}
4945
4947 APInt DemandedElts = getDemandAllEltsMask(Op);
4948 return ComputeNumSignBits(Op, DemandedElts, Depth);
4949}
4950
4951unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4952 unsigned Depth) const {
4953 EVT VT = Op.getValueType();
4954 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4955 unsigned VTBits = VT.getScalarSizeInBits();
4956 unsigned NumElts = DemandedElts.getBitWidth();
4957 unsigned Tmp, Tmp2;
4958 unsigned FirstAnswer = 1;
4959
4960 assert((!VT.isScalableVector() || NumElts == 1) &&
4961 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4962
4963 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4964 const APInt &Val = C->getAPIntValue();
4965 return Val.getNumSignBits();
4966 }
4967
4968 if (Depth >= MaxRecursionDepth)
4969 return 1; // Limit search depth.
4970
4971 if (!DemandedElts)
4972 return 1; // No demanded elts, better to assume we don't know anything.
4973
4974 unsigned Opcode = Op.getOpcode();
4975 switch (Opcode) {
4976 default: break;
4977 case ISD::AssertSext:
4978 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4979 return VTBits-Tmp+1;
4980 case ISD::AssertZext:
4981 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4982 return VTBits-Tmp;
4983 case ISD::FREEZE:
4984 if (isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,
4986 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4987 break;
4988 case ISD::MERGE_VALUES:
4989 return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,
4990 Depth + 1);
4991 case ISD::SPLAT_VECTOR: {
4992 // Check if the sign bits of source go down as far as the truncated value.
4993 unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();
4994 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4995 if (NumSrcSignBits > (NumSrcBits - VTBits))
4996 return NumSrcSignBits - (NumSrcBits - VTBits);
4997 break;
4998 }
4999 case ISD::BUILD_VECTOR:
5000 assert(!VT.isScalableVector());
5001 Tmp = VTBits;
5002 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
5003 if (!DemandedElts[i])
5004 continue;
5005
5006 SDValue SrcOp = Op.getOperand(i);
5007 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
5008 // for constant nodes to ensure we only look at the sign bits.
5010 APInt T = C->getAPIntValue().trunc(VTBits);
5011 Tmp2 = T.getNumSignBits();
5012 } else if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5013 SrcOp.getOperand(0).getScalarValueSizeInBits() >= VTBits) {
5014 // EXTRACT_VECTOR_ELT can extend the value with high bits undefined. If
5015 // this BUILD_VECTOR truncates those undefined bits we can just look
5016 // through the SrcOp and query the vector directly.
5017 SDValue InVec = SrcOp.getOperand(0);
5018 EVT InVecVT = InVec.getValueType();
5019
5020 APInt DemandedSrcElts;
5021 if (InVecVT.isScalableVector())
5022 // Demand all elements.
5023 DemandedSrcElts = APInt(1, 1);
5024 else {
5025 unsigned NumSrcElts = InVecVT.getVectorNumElements();
5026 auto *ConstEltNo = dyn_cast<ConstantSDNode>(SrcOp.getOperand(1));
5027 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
5028 DemandedSrcElts =
5029 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
5030 else
5031 DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
5032 }
5033
5034 Tmp2 = ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
5035 unsigned ExtraBits = InVec.getScalarValueSizeInBits() - VTBits;
5036 if (ExtraBits)
5037 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5038 } else {
5039 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
5040
5041 if (SrcOp.getValueSizeInBits() != VTBits) {
5042 assert(SrcOp.getValueSizeInBits() > VTBits &&
5043 "Expected BUILD_VECTOR implicit truncation");
5044 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
5045 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5046 }
5047 }
5048 Tmp = std::min(Tmp, Tmp2);
5049 }
5050 return Tmp;
5051
5052 case ISD::VECTOR_COMPRESS: {
5053 SDValue Vec = Op.getOperand(0);
5054 SDValue PassThru = Op.getOperand(2);
5055 Tmp = ComputeNumSignBits(PassThru, DemandedElts, Depth + 1);
5056 if (Tmp == 1)
5057 return 1;
5058 Tmp2 = ComputeNumSignBits(Vec, Depth + 1);
5059 Tmp = std::min(Tmp, Tmp2);
5060 return Tmp;
5061 }
5062
5063 case ISD::VECTOR_SHUFFLE: {
5064 // Collect the minimum number of sign bits that are shared by every vector
5065 // element referenced by the shuffle.
5066 APInt DemandedLHS, DemandedRHS;
5068 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
5069 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
5070 DemandedLHS, DemandedRHS))
5071 return 1;
5072
5073 Tmp = std::numeric_limits<unsigned>::max();
5074 if (!!DemandedLHS)
5075 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
5076 if (!!DemandedRHS) {
5077 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
5078 Tmp = std::min(Tmp, Tmp2);
5079 }
5080 // If we don't know anything, early out and try computeKnownBits fall-back.
5081 if (Tmp == 1)
5082 break;
5083 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5084 return Tmp;
5085 }
5086
5087 case ISD::BITCAST: {
5088 if (VT.isScalableVector())
5089 break;
5090 SDValue N0 = Op.getOperand(0);
5091 EVT SrcVT = N0.getValueType();
5092 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5093
5094 // Ignore bitcasts from unsupported types..
5095 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5096 break;
5097
5098 // Fast handling of 'identity' bitcasts.
5099 if (VTBits == SrcBits)
5100 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
5101
5102 bool IsLE = getDataLayout().isLittleEndian();
5103
5104 // Bitcast 'large element' scalar/vector to 'small element' vector.
5105 if ((SrcBits % VTBits) == 0) {
5106 assert(VT.isVector() && "Expected bitcast to vector");
5107
5108 unsigned Scale = SrcBits / VTBits;
5109 APInt SrcDemandedElts =
5110 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
5111
5112 // Fast case - sign splat can be simply split across the small elements.
5113 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
5114 if (Tmp == SrcBits)
5115 return VTBits;
5116
5117 // Slow case - determine how far the sign extends into each sub-element.
5118 Tmp2 = VTBits;
5119 for (unsigned i = 0; i != NumElts; ++i)
5120 if (DemandedElts[i]) {
5121 unsigned SubOffset = i % Scale;
5122 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5123 SubOffset = SubOffset * VTBits;
5124 if (Tmp <= SubOffset)
5125 return 1;
5126 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
5127 }
5128 return Tmp2;
5129 }
5130 break;
5131 }
5132
5134 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5135 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
5136 return VTBits - Tmp + 1;
5137 case ISD::SIGN_EXTEND:
5138 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
5139 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
5141 // Max of the input and what this extends.
5142 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
5143 Tmp = VTBits-Tmp+1;
5144 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
5145 return std::max(Tmp, Tmp2);
5147 if (VT.isScalableVector())
5148 break;
5149 SDValue Src = Op.getOperand(0);
5150 EVT SrcVT = Src.getValueType();
5151 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
5152 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5153 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
5154 }
5155 case ISD::SRA:
5156 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5157 // SRA X, C -> adds C sign bits.
5158 if (std::optional<unsigned> ShAmt =
5159 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))
5160 Tmp = std::min(Tmp + *ShAmt, VTBits);
5161 return Tmp;
5162 case ISD::SHL:
5163 if (std::optional<ConstantRange> ShAmtRange =
5164 getValidShiftAmountRange(Op, DemandedElts, Depth + 1)) {
5165 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5166 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5167 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5168 // shifted out, then we can compute the number of sign bits for the
5169 // operand being extended. A future improvement could be to pass along the
5170 // "shifted left by" information in the recursive calls to
5171 // ComputeKnownSignBits. Allowing us to handle this more generically.
5172 if (ISD::isExtOpcode(Op.getOperand(0).getOpcode())) {
5173 SDValue Ext = Op.getOperand(0);
5174 EVT ExtVT = Ext.getValueType();
5175 SDValue Extendee = Ext.getOperand(0);
5176 EVT ExtendeeVT = Extendee.getValueType();
5177 unsigned SizeDifference =
5178 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5179 if (SizeDifference <= MinShAmt) {
5180 Tmp = SizeDifference +
5181 ComputeNumSignBits(Extendee, DemandedElts, Depth + 1);
5182 if (MaxShAmt < Tmp)
5183 return Tmp - MaxShAmt;
5184 }
5185 }
5186 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5187 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5188 if (MaxShAmt < Tmp)
5189 return Tmp - MaxShAmt;
5190 }
5191 break;
5192 case ISD::AND:
5193 case ISD::OR:
5194 case ISD::XOR: // NOT is handled here.
5195 // Logical binary ops preserve the number of sign bits at the worst.
5196 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
5197 if (Tmp != 1) {
5198 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
5199 FirstAnswer = std::min(Tmp, Tmp2);
5200 // We computed what we know about the sign bits as our first
5201 // answer. Now proceed to the generic code that uses
5202 // computeKnownBits, and pick whichever answer is better.
5203 }
5204 break;
5205
5206 case ISD::SELECT:
5207 case ISD::VSELECT:
5208 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
5209 if (Tmp == 1) return 1; // Early out.
5210 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
5211 return std::min(Tmp, Tmp2);
5212 case ISD::SELECT_CC:
5213 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
5214 if (Tmp == 1) return 1; // Early out.
5215 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
5216 return std::min(Tmp, Tmp2);
5217
5218 case ISD::SMIN:
5219 case ISD::SMAX: {
5220 // If we have a clamp pattern, we know that the number of sign bits will be
5221 // the minimum of the clamp min/max range.
5222 bool IsMax = (Opcode == ISD::SMAX);
5223 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5224 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
5225 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5226 CstHigh =
5227 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
5228 if (CstLow && CstHigh) {
5229 if (!IsMax)
5230 std::swap(CstLow, CstHigh);
5231 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
5232 Tmp = CstLow->getAPIntValue().getNumSignBits();
5233 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5234 return std::min(Tmp, Tmp2);
5235 }
5236 }
5237
5238 // Fallback - just get the minimum number of sign bits of the operands.
5239 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5240 if (Tmp == 1)
5241 return 1; // Early out.
5242 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5243 return std::min(Tmp, Tmp2);
5244 }
5245 case ISD::UMIN:
5246 case ISD::UMAX:
5247 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5248 if (Tmp == 1)
5249 return 1; // Early out.
5250 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5251 return std::min(Tmp, Tmp2);
5252 case ISD::SSUBO_CARRY:
5253 case ISD::USUBO_CARRY:
5254 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5255 if (Op.getResNo() == 0 && Op.getOperand(0) == Op.getOperand(1))
5256 return VTBits;
5257 [[fallthrough]];
5258 case ISD::SADDO:
5259 case ISD::UADDO:
5260 case ISD::SADDO_CARRY:
5261 case ISD::UADDO_CARRY:
5262 case ISD::SSUBO:
5263 case ISD::USUBO:
5264 case ISD::SMULO:
5265 case ISD::UMULO:
5266 if (Op.getResNo() != 1)
5267 break;
5268 // The boolean result conforms to getBooleanContents. Fall through.
5269 // If setcc returns 0/-1, all bits are sign bits.
5270 // We know that we have an integer-based boolean since these operations
5271 // are only available for integer.
5272 if (TLI->getBooleanContents(VT.isVector(), false) ==
5274 return VTBits;
5275 break;
5276 case ISD::SETCC:
5277 case ISD::SETCCCARRY:
5278 case ISD::STRICT_FSETCC:
5279 case ISD::STRICT_FSETCCS: {
5280 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5281 // If setcc returns 0/-1, all bits are sign bits.
5282 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
5284 return VTBits;
5285 break;
5286 }
5289 // Semantically similar to icmp ult.
5290 if (TLI->getBooleanContents(VT.isVector(), /*isFloat=*/false) ==
5292 return VTBits;
5293 break;
5294 case ISD::ROTL:
5295 case ISD::ROTR: {
5296 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5297 ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
5298 FirstAnswer = SignBitsOps::rot(
5299 Tmp, VTBits, C ? std::optional(C->getAPIntValue()) : std::nullopt,
5300 Opcode == ISD::ROTR);
5301 break;
5302 }
5303 case ISD::ADD:
5304 case ISD::ADDC:
5305 // TODO: Move Operand 1 check before Operand 0 check
5306 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5307 if (Tmp == 1) return 1; // Early out.
5308
5309 // Special case decrementing a value (ADD X, -1):
5310 if (ConstantSDNode *CRHS =
5311 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
5312 if (CRHS->isAllOnes()) {
5314 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
5315
5316 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5317 // sign bits set.
5318 if ((Known.Zero | 1).isAllOnes())
5319 return VTBits;
5320
5321 // If we are subtracting one from a positive number, there is no carry
5322 // out of the result.
5323 if (Known.isNonNegative())
5324 return Tmp;
5325 }
5326
5327 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5328 if (Tmp2 == 1) return 1; // Early out.
5329
5330 // Add can have at most one carry bit. Thus we know that the output
5331 // is, at worst, one more bit than the inputs.
5332 return std::min(Tmp, Tmp2) - 1;
5333 case ISD::SUB:
5334 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5335 if (Tmp2 == 1) return 1; // Early out.
5336
5337 // Handle NEG.
5338 if (ConstantSDNode *CLHS =
5339 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
5340 if (CLHS->isZero()) {
5342 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
5343 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5344 // sign bits set.
5345 if ((Known.Zero | 1).isAllOnes())
5346 return VTBits;
5347
5348 // If the input is known to be positive (the sign bit is known clear),
5349 // the output of the NEG has the same number of sign bits as the input.
5350 if (Known.isNonNegative())
5351 return Tmp2;
5352
5353 // Otherwise, we treat this like a SUB.
5354 }
5355
5356 // Sub can have at most one carry bit. Thus we know that the output
5357 // is, at worst, one more bit than the inputs.
5358 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5359 if (Tmp == 1) return 1; // Early out.
5360 return std::min(Tmp, Tmp2) - 1;
5361 case ISD::MUL: {
5362 // The output of the Mul can be at most twice the valid bits in the inputs.
5363 unsigned SignBitsOp0 =
5364 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5365 if (SignBitsOp0 == 1)
5366 break;
5367 unsigned SignBitsOp1 =
5368 ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5369 if (SignBitsOp1 == 1)
5370 break;
5371 unsigned OutValidBits =
5372 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5373 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5374 }
5375 case ISD::AVGCEILS:
5376 case ISD::AVGFLOORS:
5377 case ISD::AVGSUBFLOORS:
5378 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5379 if (Tmp == 1)
5380 return 1; // Early out.
5381 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
5382 return std::min(Tmp, Tmp2);
5383 case ISD::SREM:
5384 // The sign bit is the LHS's sign bit, except when the result of the
5385 // remainder is zero. The magnitude of the result should be less than or
5386 // equal to the magnitude of the LHS. Therefore, the result should have
5387 // at least as many sign bits as the left hand side.
5388 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5389 case ISD::TRUNCATE: {
5390 // Check if the sign bits of source go down as far as the truncated value.
5391 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
5392 unsigned NumSrcSignBits =
5393 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
5394 if (NumSrcSignBits > (NumSrcBits - VTBits))
5395 return NumSrcSignBits - (NumSrcBits - VTBits);
5396 break;
5397 }
5398 case ISD::EXTRACT_ELEMENT: {
5399 if (VT.isScalableVector())
5400 break;
5401 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
5402 const int BitWidth = Op.getValueSizeInBits();
5403 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
5404
5405 // Get reverse index (starting from 1), Op1 value indexes elements from
5406 // little end. Sign starts at big end.
5407 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
5408
5409 // If the sign portion ends in our element the subtraction gives correct
5410 // result. Otherwise it gives either negative or > bitwidth result
5411 return std::clamp(KnownSign - rIndex * BitWidth, 1, BitWidth);
5412 }
5414 if (VT.isScalableVector())
5415 break;
5416 // If we know the element index, split the demand between the
5417 // source vector and the inserted element, otherwise assume we need
5418 // the original demanded vector elements and the value.
5419 SDValue InVec = Op.getOperand(0);
5420 SDValue InVal = Op.getOperand(1);
5421 SDValue EltNo = Op.getOperand(2);
5422 bool DemandedVal = true;
5423 APInt DemandedVecElts = DemandedElts;
5424 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
5425 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
5426 unsigned EltIdx = CEltNo->getZExtValue();
5427 DemandedVal = !!DemandedElts[EltIdx];
5428 DemandedVecElts.clearBit(EltIdx);
5429 }
5430 Tmp = std::numeric_limits<unsigned>::max();
5431 if (DemandedVal) {
5432 // TODO - handle implicit truncation of inserted elements.
5433 if (InVal.getScalarValueSizeInBits() != VTBits)
5434 break;
5435 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
5436 Tmp = std::min(Tmp, Tmp2);
5437 }
5438 if (!!DemandedVecElts) {
5439 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
5440 Tmp = std::min(Tmp, Tmp2);
5441 }
5442 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5443 return Tmp;
5444 }
5446 SDValue InVec = Op.getOperand(0);
5447 SDValue EltNo = Op.getOperand(1);
5448 EVT VecVT = InVec.getValueType();
5449 // ComputeNumSignBits not yet implemented for scalable vectors.
5450 if (VecVT.isScalableVector())
5451 break;
5452 const unsigned BitWidth = Op.getValueSizeInBits();
5453 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
5454 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5455
5456 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5457 // anything about sign bits. But if the sizes match we can derive knowledge
5458 // about sign bits from the vector operand.
5459 if (BitWidth != EltBitWidth)
5460 break;
5461
5462 // If we know the element index, just demand that vector element, else for
5463 // an unknown element index, ignore DemandedElts and demand them all.
5464 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
5465 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
5466 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
5467 DemandedSrcElts =
5468 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
5469
5470 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
5471 }
5473 // Offset the demanded elts by the subvector index.
5474 SDValue Src = Op.getOperand(0);
5475
5476 APInt DemandedSrcElts;
5477 if (Src.getValueType().isScalableVector())
5478 DemandedSrcElts = APInt(1, 1);
5479 else {
5480 uint64_t Idx = Op.getConstantOperandVal(1);
5481 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5482 DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
5483 }
5484 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
5485 }
5486 case ISD::CONCAT_VECTORS: {
5487 if (VT.isScalableVector())
5488 break;
5489 // Determine the minimum number of sign bits across all demanded
5490 // elts of the input vectors. Early out if the result is already 1.
5491 Tmp = std::numeric_limits<unsigned>::max();
5492 EVT SubVectorVT = Op.getOperand(0).getValueType();
5493 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5494 unsigned NumSubVectors = Op.getNumOperands();
5495 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5496 APInt DemandedSub =
5497 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
5498 if (!DemandedSub)
5499 continue;
5500 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
5501 Tmp = std::min(Tmp, Tmp2);
5502 }
5503 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5504 return Tmp;
5505 }
5506 case ISD::INSERT_SUBVECTOR: {
5507 SDValue Src = Op.getOperand(0);
5508 SDValue Sub = Op.getOperand(1);
5509 if (VT.isScalableVector()) {
5510 Tmp = ComputeNumSignBits(Sub, Depth + 1);
5511 Tmp = std::min(Tmp, ComputeNumSignBits(Src, Depth + 1));
5512 return Tmp;
5513 }
5514 // Demand any elements from the subvector and the remainder from the src its
5515 // inserted into.
5516 uint64_t Idx = Op.getConstantOperandVal(2);
5517 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5518 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
5519 APInt DemandedSrcElts = DemandedElts;
5520 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
5521
5522 Tmp = std::numeric_limits<unsigned>::max();
5523 if (!!DemandedSubElts) {
5524 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
5525 if (Tmp == 1)
5526 return 1; // early-out
5527 }
5528 if (!!DemandedSrcElts) {
5529 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
5530 Tmp = std::min(Tmp, Tmp2);
5531 }
5532 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5533 return Tmp;
5534 }
5535 case ISD::LOAD: {
5536 // If we are looking at the loaded value of the SDNode.
5537 if (Op.getResNo() != 0)
5538 break;
5539
5541 if (const MDNode *Ranges = LD->getRanges()) {
5542 if (DemandedElts != 1)
5543 break;
5544
5546 if (VTBits > CR.getBitWidth()) {
5547 switch (LD->getExtensionType()) {
5548 case ISD::SEXTLOAD:
5549 CR = CR.signExtend(VTBits);
5550 break;
5551 case ISD::ZEXTLOAD:
5552 CR = CR.zeroExtend(VTBits);
5553 break;
5554 default:
5555 break;
5556 }
5557 }
5558
5559 if (VTBits != CR.getBitWidth())
5560 break;
5561 return std::min(CR.getSignedMin().getNumSignBits(),
5563 }
5564
5565 unsigned ExtType = LD->getExtensionType();
5566 switch (ExtType) {
5567 default:
5568 break;
5569 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5570 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5571 return VTBits - Tmp + 1;
5572 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5573 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5574 return VTBits - Tmp;
5575 case ISD::NON_EXTLOAD:
5576 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5577 // We only need to handle vectors - computeKnownBits should handle
5578 // scalar cases.
5579 Type *CstTy = Cst->getType();
5580 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5581 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5582 VTBits == CstTy->getScalarSizeInBits()) {
5583 Tmp = VTBits;
5584 for (unsigned i = 0; i != NumElts; ++i) {
5585 if (!DemandedElts[i])
5586 continue;
5587 if (Constant *Elt = Cst->getAggregateElement(i)) {
5588 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
5589 const APInt &Value = CInt->getValue();
5590 Tmp = std::min(Tmp, Value.getNumSignBits());
5591 continue;
5592 }
5593 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
5594 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5595 Tmp = std::min(Tmp, Value.getNumSignBits());
5596 continue;
5597 }
5598 }
5599 // Unknown type. Conservatively assume no bits match sign bit.
5600 return 1;
5601 }
5602 return Tmp;
5603 }
5604 }
5605 break;
5606 }
5607
5608 break;
5609 }
5612 case ISD::ATOMIC_SWAP:
5624 case ISD::ATOMIC_LOAD: {
5625 auto *AT = cast<AtomicSDNode>(Op);
5626 // If we are looking at the loaded value.
5627 if (Op.getResNo() == 0) {
5628 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5629 if (Tmp == VTBits)
5630 return 1; // early-out
5631
5632 // For atomic_load, prefer to use the extension type.
5633 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5634 switch (AT->getExtensionType()) {
5635 default:
5636 break;
5637 case ISD::SEXTLOAD:
5638 return VTBits - Tmp + 1;
5639 case ISD::ZEXTLOAD:
5640 return VTBits - Tmp;
5641 }
5642 }
5643
5644 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5645 return VTBits - Tmp + 1;
5646 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5647 return VTBits - Tmp;
5648 }
5649 break;
5650 }
5651 }
5652
5653 // Allow the target to implement this method for its nodes.
5654 if (Opcode >= ISD::BUILTIN_OP_END ||
5655 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5656 Opcode == ISD::INTRINSIC_W_CHAIN ||
5657 Opcode == ISD::INTRINSIC_VOID) {
5658 // TODO: This can probably be removed once target code is audited. This
5659 // is here purely to reduce patch size and review complexity.
5660 if (!VT.isScalableVector()) {
5661 unsigned NumBits =
5662 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
5663 if (NumBits > 1)
5664 FirstAnswer = std::max(FirstAnswer, NumBits);
5665 }
5666 }
5667
5668 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5669 // use this information.
5670 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5671 return std::max(FirstAnswer, Known.countMinSignBits());
5672}
5673
5675 unsigned Depth) const {
5676 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5677 return Op.getScalarValueSizeInBits() - SignBits + 1;
5678}
5679
5681 const APInt &DemandedElts,
5682 unsigned Depth) const {
5683 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5684 return Op.getScalarValueSizeInBits() - SignBits + 1;
5685}
5686
5688 UndefPoisonKind Kind,
5689 unsigned Depth) const {
5690 // Early out for FREEZE.
5691 if (Op.getOpcode() == ISD::FREEZE)
5692 return true;
5693
5694 APInt DemandedElts = getDemandAllEltsMask(Op);
5695 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5696}
5697
5699 const APInt &DemandedElts,
5700 UndefPoisonKind Kind,
5701 unsigned Depth) const {
5702 unsigned Opcode = Op.getOpcode();
5703 unsigned NumElts = DemandedElts.getBitWidth();
5704
5705 // Early out for FREEZE.
5706 if (Opcode == ISD::FREEZE)
5707 return true;
5708
5709 if (Depth >= MaxRecursionDepth)
5710 return false; // Limit search depth.
5711
5712 if (isIntOrFPConstant(Op))
5713 return true;
5714
5715 switch (Opcode) {
5716 case ISD::CONDCODE:
5717 case ISD::VALUETYPE:
5718 case ISD::FrameIndex:
5720 case ISD::CopyFromReg:
5721 return true;
5722
5723 case ISD::POISON:
5724 return !includesPoison(Kind);
5725
5726 case ISD::UNDEF:
5727 return !includesUndef(Kind);
5728
5729 case ISD::BITCAST: {
5730 SDValue Src = Op.getOperand(0);
5731 EVT SrcVT = Src.getValueType();
5732 EVT DstVT = Op.getValueType();
5733
5734 if (!SrcVT.isVector() || !DstVT.isVector())
5735 return isGuaranteedNotToBeUndefOrPoison(Src, Kind, Depth + 1);
5736
5737 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5738 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5739 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5740 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5741
5742 if (SrcEltBits == DstEltBits)
5743 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedElts, Kind,
5744 Depth + 1);
5745
5746 if (SrcEltBits < DstEltBits) {
5747 if (DstEltBits % SrcEltBits != 0)
5748 return isGuaranteedNotToBeUndefOrPoison(Src, Kind, Depth + 1);
5749
5750 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5751 "Unexpected vector bitcast");
5752 APInt DemandedSrcElts =
5753 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts.getKnownMinValue());
5754 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, Kind,
5755 Depth + 1);
5756 }
5757
5758 if (SrcEltBits % DstEltBits != 0)
5759 return isGuaranteedNotToBeUndefOrPoison(Src, Kind, Depth + 1);
5760
5761 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5762 "Unexpected vector bitcast");
5763 APInt DemandedSrcElts =
5764 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts.getKnownMinValue());
5765 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, Kind,
5766 Depth + 1);
5767 }
5768
5769 case ISD::BUILD_VECTOR:
5770 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5771 // this shouldn't affect the result.
5772 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5773 if (!DemandedElts[i])
5774 continue;
5775 if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), Kind, Depth + 1))
5776 return false;
5777 }
5778 return true;
5779
5780 case ISD::CONCAT_VECTORS: {
5781 EVT VT = Op.getValueType();
5782 if (!VT.isFixedLengthVector())
5783 break;
5784
5785 EVT SubVT = Op.getOperand(0).getValueType();
5786 unsigned NumSubElts = SubVT.getVectorNumElements();
5787 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5788 APInt DemandedSubElts =
5789 DemandedElts.extractBits(NumSubElts, I * NumSubElts);
5790 if (!!DemandedSubElts &&
5791 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(I), DemandedSubElts,
5792 Kind, Depth + 1))
5793 return false;
5794 }
5795 return true;
5796 }
5797
5799 SDValue Src = Op.getOperand(0);
5800 if (Src.getValueType().isScalableVector())
5801 break;
5802 uint64_t Idx = Op.getConstantOperandVal(1);
5803 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5804 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
5805 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, Kind,
5806 Depth + 1);
5807 }
5808
5809 case ISD::INSERT_SUBVECTOR: {
5810 if (Op.getValueType().isScalableVector())
5811 break;
5812 SDValue Src = Op.getOperand(0);
5813 SDValue Sub = Op.getOperand(1);
5814 uint64_t Idx = Op.getConstantOperandVal(2);
5815 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5816 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
5817 APInt DemandedSrcElts = DemandedElts;
5818 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
5819
5820 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5821 Sub, DemandedSubElts, Kind, Depth + 1))
5822 return false;
5823 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5824 Src, DemandedSrcElts, Kind, Depth + 1))
5825 return false;
5826 return true;
5827 }
5828
5830 SDValue Src = Op.getOperand(0);
5831 auto *IndexC = dyn_cast<ConstantSDNode>(Op.getOperand(1));
5832 EVT SrcVT = Src.getValueType();
5833 if (SrcVT.isFixedLengthVector() && IndexC &&
5834 IndexC->getAPIntValue().ult(SrcVT.getVectorNumElements())) {
5835 APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
5836 IndexC->getZExtValue());
5837 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, Kind,
5838 Depth + 1);
5839 }
5840 break;
5841 }
5842
5844 SDValue InVec = Op.getOperand(0);
5845 SDValue InVal = Op.getOperand(1);
5846 SDValue EltNo = Op.getOperand(2);
5847 EVT VT = InVec.getValueType();
5848 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
5849 if (IndexC && VT.isFixedLengthVector() &&
5850 IndexC->getAPIntValue().ult(VT.getVectorNumElements())) {
5851 if (DemandedElts[IndexC->getZExtValue()] &&
5852 !isGuaranteedNotToBeUndefOrPoison(InVal, Kind, Depth + 1))
5853 return false;
5854 APInt InVecDemandedElts = DemandedElts;
5855 InVecDemandedElts.clearBit(IndexC->getZExtValue());
5856 if (!!InVecDemandedElts &&
5858 peekThroughInsertVectorElt(InVec, InVecDemandedElts),
5859 InVecDemandedElts, Kind, Depth + 1))
5860 return false;
5861 return true;
5862 }
5863 break;
5864 }
5865
5867 // Check upper (known poison) elements.
5868 if (DemandedElts.ugt(1) && includesPoison(Kind))
5869 return false;
5870 // Check element zero.
5871 if (DemandedElts[0] &&
5872 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), Kind, Depth + 1))
5873 return false;
5874 return true;
5875
5876 case ISD::SPLAT_VECTOR:
5877 return isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), Kind, Depth + 1);
5878
5879 case ISD::SELECT: {
5880 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5881 /*ConsiderFlags*/ true, Depth) &&
5882 isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), Kind,
5883 Depth + 1) &&
5884 isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedElts,
5885 Kind, Depth + 1) &&
5886 isGuaranteedNotToBeUndefOrPoison(Op.getOperand(2), DemandedElts,
5887 Kind, Depth + 1);
5888 }
5889
5890 case ISD::VECTOR_SHUFFLE: {
5891 APInt DemandedLHS, DemandedRHS;
5892 auto *SVN = cast<ShuffleVectorSDNode>(Op);
5893 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
5894 DemandedLHS, DemandedRHS,
5895 /*AllowUndefElts=*/false))
5896 return false;
5897 if (!DemandedLHS.isZero() &&
5898 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedLHS, Kind,
5899 Depth + 1))
5900 return false;
5901 if (!DemandedRHS.isZero() &&
5902 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedRHS, Kind,
5903 Depth + 1))
5904 return false;
5905 return true;
5906 }
5907
5908 case ISD::SHL:
5909 case ISD::SRL:
5910 case ISD::SRA:
5911 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5912 // enough to check operand 0 if Op can't create undef/poison.
5913 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5914 /*ConsiderFlags*/ true, Depth) &&
5915 isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,
5916 Kind, Depth + 1);
5917
5918 case ISD::BSWAP:
5919 case ISD::CTPOP:
5920 case ISD::BITREVERSE:
5921 case ISD::AND:
5922 case ISD::OR:
5923 case ISD::XOR:
5924 case ISD::ADD:
5925 case ISD::SUB:
5926 case ISD::MUL:
5927 case ISD::SADDSAT:
5928 case ISD::UADDSAT:
5929 case ISD::SSUBSAT:
5930 case ISD::USUBSAT:
5931 case ISD::SSHLSAT:
5932 case ISD::USHLSAT:
5933 case ISD::SMIN:
5934 case ISD::SMAX:
5935 case ISD::UMIN:
5936 case ISD::UMAX:
5937 case ISD::ZERO_EXTEND:
5938 case ISD::SIGN_EXTEND:
5939 case ISD::ANY_EXTEND:
5940 case ISD::TRUNCATE:
5941 case ISD::VSELECT: {
5942 // If Op can't create undef/poison and none of its operands are undef/poison
5943 // then Op is never undef/poison. A difference from the more common check
5944 // below, outside the switch, is that we handle elementwise operations for
5945 // which the DemandedElts mask is valid for all operands here.
5946 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5947 /*ConsiderFlags*/ true, Depth) &&
5948 all_of(Op->ops(), [&](SDValue V) {
5949 return isGuaranteedNotToBeUndefOrPoison(V, DemandedElts, Kind,
5950 Depth + 1);
5951 });
5952 }
5953
5954 case ISD::LOAD: {
5955 // If we're loading an unindexed constant pool, check if the constant
5956 // contains poison/undef - skip ANYEXT loads.
5957 // TODO: type size mismatch handling.
5958 auto *LD = cast<LoadSDNode>(Op);
5959 if (ISD::isUNINDEXEDLoad(LD) && !ISD::isEXTLoad(LD)) {
5960 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5961 EVT VT = Op.getValueType();
5962 Type *CstTy = Cst->getType();
5963
5964 auto IsSafe = [Kind](const Constant *C) -> bool {
5965 // TODO: expose isGuaranteedNotToBeUndefOrPoison(UndefPoisonKind)
5966 if (includesPoison(Kind) && includesUndef(Kind))
5967 return ::isGuaranteedNotToBeUndefOrPoison(C);
5968 if (includesPoison(Kind))
5969 return ::isGuaranteedNotToBePoison(C);
5970 assert(includesUndef(Kind) && "UndefOnly expected");
5971 return ::isGuaranteedNotToBeUndef(C);
5972 };
5973
5974 // If Cst matches VT vector dimensions, then check per-element.
5975 if (CstTy->isVectorTy() && VT.isFixedLengthVector() &&
5976 VT.getSizeInBits() == CstTy->getPrimitiveSizeInBits() &&
5977 VT.getScalarSizeInBits() == CstTy->getScalarSizeInBits()) {
5978 for (unsigned I = 0; I != NumElts; ++I) {
5979 if (!DemandedElts[I])
5980 continue;
5981 if (Constant *Elt = Cst->getAggregateElement(I)) {
5982 if (IsSafe(Elt))
5983 continue;
5984 }
5985 return false;
5986 }
5987 } else {
5988 return IsSafe(Cst);
5989 }
5990 }
5991 }
5992 break;
5993 }
5994
5995 // TODO: Search for noundef attributes from library functions.
5996
5997 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5998
5999 default:
6000 // Allow the target to implement this method for its nodes.
6001 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6002 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6003 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
6004 Op, DemandedElts, *this, Kind, Depth);
6005 break;
6006 }
6007
6008 // If Op can't create undef/poison and none of its operands are undef/poison
6009 // then Op is never undef/poison.
6010 // NOTE: TargetNodes can handle this in themselves in
6011 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
6012 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
6013 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
6014 all_of(Op->ops(), [&](SDValue V) {
6015 return isGuaranteedNotToBeUndefOrPoison(V, Kind, Depth + 1);
6016 });
6017}
6018
6020 bool ConsiderFlags,
6021 unsigned Depth) const {
6022 APInt DemandedElts = getDemandAllEltsMask(Op);
6023 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
6024}
6025
6027 UndefPoisonKind Kind,
6028 bool ConsiderFlags,
6029 unsigned Depth) const {
6030 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
6031 return true;
6032
6033 unsigned Opcode = Op.getOpcode();
6034 switch (Opcode) {
6035 case ISD::AssertSext:
6036 case ISD::AssertZext:
6037 case ISD::AssertAlign:
6039 // Assertion nodes can create poison if the assertion fails.
6040 return includesPoison(Kind);
6041
6042 case ISD::FREEZE:
6046 case ISD::SADDSAT:
6047 case ISD::UADDSAT:
6048 case ISD::SSUBSAT:
6049 case ISD::USUBSAT:
6050 case ISD::MULHU:
6051 case ISD::MULHS:
6052 case ISD::AVGFLOORS:
6053 case ISD::AVGFLOORU:
6054 case ISD::AVGCEILS:
6055 case ISD::AVGCEILU:
6056 case ISD::AVGSUBFLOORS:
6057 case ISD::AVGSUBFLOORU:
6058 case ISD::ABDU:
6059 case ISD::ABDS:
6060 case ISD::SMIN:
6061 case ISD::SMAX:
6062 case ISD::SCMP:
6063 case ISD::UMIN:
6064 case ISD::UMAX:
6065 case ISD::UCMP:
6066 case ISD::AND:
6067 case ISD::XOR:
6068 case ISD::ROTL:
6069 case ISD::ROTR:
6070 case ISD::FSHL:
6071 case ISD::FSHR:
6072 case ISD::BSWAP:
6073 case ISD::CTTZ:
6074 case ISD::CTLZ:
6075 case ISD::CTLS:
6076 case ISD::CTPOP:
6077 case ISD::BITREVERSE:
6078 case ISD::PARITY:
6079 case ISD::SIGN_EXTEND:
6080 case ISD::TRUNCATE:
6084 case ISD::BITCAST:
6085 case ISD::BUILD_VECTOR:
6086 case ISD::BUILD_PAIR:
6087 case ISD::SPLAT_VECTOR:
6088 case ISD::FABS:
6089 case ISD::FCEIL:
6090 case ISD::FFLOOR:
6091 case ISD::FTRUNC:
6092 case ISD::FRINT:
6093 case ISD::FNEARBYINT:
6094 case ISD::FROUND:
6095 case ISD::FROUNDEVEN:
6096 return false;
6097
6098 case ISD::ABS:
6099 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
6100 // Different to Intrinsic::abs.
6101 return false;
6103 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
6104 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1) <= 1;
6105
6106 case ISD::ADDC:
6107 case ISD::SUBC:
6108 case ISD::ADDE:
6109 case ISD::SUBE:
6110 case ISD::SADDO:
6111 case ISD::SSUBO:
6112 case ISD::SMULO:
6113 case ISD::SADDO_CARRY:
6114 case ISD::SSUBO_CARRY:
6115 case ISD::UADDO:
6116 case ISD::USUBO:
6117 case ISD::UMULO:
6118 case ISD::UADDO_CARRY:
6119 case ISD::USUBO_CARRY:
6120 // No poison on result or overflow flags.
6121 return false;
6122
6123 case ISD::SELECT_CC:
6124 case ISD::SETCC: {
6125 // Integer setcc cannot create undef or poison.
6126 if (Op.getOperand(0).getValueType().isInteger())
6127 return false;
6128
6129 // FP compares are more complicated. They can create poison for nan/infinity
6130 // based on options and flags. The options and flags also cause special
6131 // nonan condition codes to be used. Those condition codes may be preserved
6132 // even if the nonan flag is dropped somewhere.
6133 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6134 ISD::CondCode CCCode = Op.getOperandCondCode(CCOp);
6135 return (unsigned)CCCode & 0x10U;
6136 }
6137
6138 case ISD::OR:
6139 case ISD::ZERO_EXTEND:
6140 case ISD::SELECT:
6141 case ISD::VSELECT:
6142 case ISD::ADD:
6143 case ISD::SUB:
6144 case ISD::MUL:
6145 case ISD::FNEG:
6146 case ISD::FADD:
6147 case ISD::FSUB:
6148 case ISD::FMUL:
6149 case ISD::FDIV:
6150 case ISD::FREM:
6151 case ISD::FCOPYSIGN:
6152 case ISD::FMA:
6153 case ISD::FMAD:
6154 case ISD::FMULADD:
6155 case ISD::FP_EXTEND:
6156 case ISD::FMINNUM:
6157 case ISD::FMAXNUM:
6158 case ISD::FMINNUM_IEEE:
6159 case ISD::FMAXNUM_IEEE:
6160 case ISD::FMINIMUM:
6161 case ISD::FMAXIMUM:
6162 case ISD::FMINIMUMNUM:
6163 case ISD::FMAXIMUMNUM:
6169 // No poison except from flags (which is handled above)
6170 return false;
6171
6172 case ISD::SHL:
6173 case ISD::SRL:
6174 case ISD::SRA:
6175 // If the max shift amount isn't in range, then the shift can
6176 // create poison.
6177 return includesPoison(Kind) &&
6178 !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);
6179
6182 // If the amount is zero then the result will be poison.
6183 // TODO: Add isKnownNeverZero DemandedElts handling.
6184 return includesPoison(Kind) &&
6185 !isKnownNeverZero(Op.getOperand(0), Depth + 1);
6186
6188 // Check if we demand any upper (poison) elements.
6189 return includesPoison(Kind) && DemandedElts.ugt(1);
6190
6193 // Ensure that the element index is in bounds.
6194 if (includesPoison(Kind)) {
6195 EVT VecVT = Op.getOperand(0).getValueType();
6196 SDValue Idx = Op.getOperand(Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6197 KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);
6198 return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
6199 }
6200 return false;
6201 }
6202
6203 case ISD::VECTOR_SHUFFLE: {
6204 // Check for any demanded shuffle element that is undef.
6205 auto *SVN = cast<ShuffleVectorSDNode>(Op);
6206 for (auto [Idx, Elt] : enumerate(SVN->getMask()))
6207 if (Elt < 0 && DemandedElts[Idx])
6208 return true;
6209 return false;
6210 }
6211
6213 return false;
6214
6215 default:
6216 // Allow the target to implement this method for its nodes.
6217 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6218 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6219 return TLI->canCreateUndefOrPoisonForTargetNode(
6220 Op, DemandedElts, *this, Kind, ConsiderFlags, Depth);
6221 break;
6222 }
6223
6224 // Be conservative and return true.
6225 return true;
6226}
6227
6228bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6229 unsigned Opcode = Op.getOpcode();
6230 if (Opcode == ISD::OR)
6231 return Op->getFlags().hasDisjoint() ||
6232 haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));
6233 if (Opcode == ISD::XOR)
6234 return !NoWrap && isMinSignedConstant(Op.getOperand(1));
6235 return false;
6236}
6237
6239 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Op.getOperand(1)) &&
6240 (Op.isAnyAdd() || isADDLike(Op));
6241}
6242
6244 FPClassTest InterestedClasses,
6245 unsigned Depth) const {
6246 APInt DemandedElts = getDemandAllEltsMask(Op);
6247 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6248}
6249
6251 const APInt &DemandedElts,
6252 FPClassTest InterestedClasses,
6253 unsigned Depth) const {
6255
6256 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Op))
6257 return KnownFPClass(CFP->getValueAPF());
6258
6259 if (Depth >= MaxRecursionDepth)
6260 return Known;
6261
6262 if (Op.getOpcode() == ISD::UNDEF)
6263 return Known;
6264
6265 EVT VT = Op.getValueType();
6266 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6267 assert((!VT.isFixedLengthVector() ||
6268 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6269 "Unexpected vector size");
6270
6271 if (!DemandedElts)
6272 return Known;
6273
6274 unsigned Opcode = Op.getOpcode();
6275 switch (Opcode) {
6276 case ISD::POISON: {
6278 break;
6279 }
6280 case ISD::FNEG: {
6281 Known = computeKnownFPClass(Op.getOperand(0), DemandedElts,
6282 InterestedClasses, Depth + 1);
6283 Known.fneg();
6284 break;
6285 }
6286 case ISD::BUILD_VECTOR: {
6287 assert(!VT.isScalableVector());
6288 bool First = true;
6289 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6290 if (!DemandedElts[I])
6291 continue;
6292
6293 if (First) {
6294 Known =
6295 computeKnownFPClass(Op.getOperand(I), InterestedClasses, Depth + 1);
6296 First = false;
6297 } else {
6298 Known |=
6299 computeKnownFPClass(Op.getOperand(I), InterestedClasses, Depth + 1);
6300 }
6301
6302 if (Known.isUnknown())
6303 break;
6304 }
6305 break;
6306 }
6308 SDValue Src = Op.getOperand(0);
6309 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6310 EVT SrcVT = Src.getValueType();
6311 if (SrcVT.isFixedLengthVector() && CIdx) {
6312 if (CIdx->getAPIntValue().ult(SrcVT.getVectorNumElements())) {
6313 APInt DemandedSrcElts = APInt::getOneBitSet(
6314 SrcVT.getVectorNumElements(), CIdx->getZExtValue());
6315 Known = computeKnownFPClass(Src, DemandedSrcElts, InterestedClasses,
6316 Depth + 1);
6317 } else {
6318 // Out of bounds index is poison.
6320 }
6321 } else {
6322 Known = computeKnownFPClass(Src, InterestedClasses, Depth + 1);
6323 }
6324 break;
6325 }
6326 case ISD::SPLAT_VECTOR: {
6327 Known = computeKnownFPClass(Op.getOperand(0), InterestedClasses, Depth + 1);
6328 break;
6329 }
6330 case ISD::BITCAST: {
6331 // FIXME: It should not be necessary to check for an elementwise bitcast.
6332 // If a bitcast is not elementwise between vector / scalar types,
6333 // computeKnownBits already splices the known bits of the source elements
6334 // appropriately so as to line up with the bits of the result's demanded
6335 // elements.
6336 EVT SrcVT = Op.getOperand(0).getValueType();
6337 if (VT.isScalableVector() || SrcVT.isScalableVector())
6338 break;
6339 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6340 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6341 if (VTNumElts != SrcVTNumElts)
6342 break;
6343
6344 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth + 1);
6346 break;
6347 }
6348 case ISD::FABS: {
6349 Known = computeKnownFPClass(Op.getOperand(0), DemandedElts,
6350 InterestedClasses, Depth + 1);
6351 Known.fabs();
6352 break;
6353 }
6354 case ISD::FCOPYSIGN: {
6355 Known = computeKnownFPClass(Op.getOperand(0), DemandedElts,
6356 InterestedClasses, Depth + 1);
6357 KnownFPClass KnownSign = computeKnownFPClass(Op.getOperand(1), DemandedElts,
6358 InterestedClasses, Depth + 1);
6359 Known.copysign(KnownSign);
6360 break;
6361 }
6362 case ISD::AssertNoFPClass: {
6363 Known = computeKnownFPClass(Op.getOperand(0), DemandedElts,
6364 InterestedClasses, Depth + 1);
6365 FPClassTest AssertedClasses =
6366 static_cast<FPClassTest>(Op->getConstantOperandVal(1));
6367 Known.setKnownFPClasses(Known.getKnownFPClasses() & ~AssertedClasses);
6368 break;
6369 }
6371 SDValue Src = Op.getOperand(0);
6372 EVT SrcVT = Src.getValueType();
6373 if (SrcVT.isFixedLengthVector()) {
6374 unsigned Idx = Op.getConstantOperandVal(1);
6375 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6376
6377 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(NumSrcElts).shl(Idx);
6378 Known = computeKnownFPClass(Src, DemandedSrcElts, InterestedClasses,
6379 Depth + 1);
6380 } else {
6381 Known = computeKnownFPClass(Src, InterestedClasses, Depth + 1);
6382 }
6383 break;
6384 }
6385 case ISD::INSERT_SUBVECTOR: {
6386 SDValue BaseVector = Op.getOperand(0);
6387 SDValue SubVector = Op.getOperand(1);
6388 EVT BaseVT = BaseVector.getValueType();
6389 if (BaseVT.isFixedLengthVector()) {
6390 unsigned Idx = Op.getConstantOperandVal(2);
6391 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6392 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6393
6394 APInt DemandedMask =
6395 APInt::getBitsSet(NumBaseElts, Idx, Idx + NumSubElts);
6396 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6397 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
6398
6399 if (!DemandedSrcElts.isZero())
6400 Known = computeKnownFPClass(BaseVector, DemandedSrcElts,
6401 InterestedClasses, Depth + 1);
6402 if (!DemandedSubElts.isZero()) {
6404 SubVector, DemandedSubElts, InterestedClasses, Depth + 1);
6405 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6406 }
6407 } else {
6408 Known = computeKnownFPClass(SubVector, InterestedClasses, Depth + 1);
6409 if (!Known.isUnknown())
6410 Known |= computeKnownFPClass(BaseVector, InterestedClasses, Depth + 1);
6411 }
6412 break;
6413 }
6414 case ISD::SELECT:
6415 case ISD::VSELECT: {
6416 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6417 // IR-level ValueTracking.
6418 KnownFPClass KnownFalseClass = computeKnownFPClass(
6419 Op.getOperand(2), DemandedElts, InterestedClasses, Depth + 1);
6420 if (KnownFalseClass.isUnknown())
6421 break;
6422 KnownFPClass KnownTrueClass = computeKnownFPClass(
6423 Op.getOperand(1), DemandedElts, InterestedClasses, Depth + 1);
6424 Known = KnownTrueClass.intersectWith(KnownFalseClass);
6425 break;
6426 }
6427 default:
6428 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6429 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6430 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, *this,
6431 Depth);
6432 }
6433 break;
6434 }
6435
6436 return Known;
6437}
6438
6440 unsigned Depth) const {
6441 APInt DemandedElts = getDemandAllEltsMask(Op);
6442 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6443}
6444
6446 bool SNaN, unsigned Depth) const {
6447 assert(!DemandedElts.isZero() && "No demanded elements");
6448
6449 // If we're told that NaNs won't happen, assume they won't.
6450 if (Op->getFlags().hasNoNaNs())
6451 return true;
6452
6453 if (Depth >= MaxRecursionDepth)
6454 return false; // Limit search depth.
6455
6456 unsigned Opcode = Op.getOpcode();
6457 switch (Opcode) {
6458 case ISD::FADD:
6459 case ISD::FSUB:
6460 case ISD::FMUL:
6461 case ISD::FDIV:
6462 case ISD::FREM:
6463 case ISD::FSIN:
6464 case ISD::FCOS:
6465 case ISD::FTAN:
6466 case ISD::FASIN:
6467 case ISD::FACOS:
6468 case ISD::FATAN:
6469 case ISD::FATAN2:
6470 case ISD::FSINH:
6471 case ISD::FCOSH:
6472 case ISD::FTANH:
6473 case ISD::FMA:
6474 case ISD::FMULADD:
6475 case ISD::FMAD: {
6476 if (SNaN)
6477 return true;
6478 // TODO: Need isKnownNeverInfinity
6479 return false;
6480 }
6481 case ISD::FCANONICALIZE:
6482 case ISD::FEXP:
6483 case ISD::FEXP2:
6484 case ISD::FEXP10:
6485 case ISD::FTRUNC:
6486 case ISD::FFLOOR:
6487 case ISD::FCEIL:
6488 case ISD::FROUND:
6489 case ISD::FROUNDEVEN:
6490 case ISD::LROUND:
6491 case ISD::LLROUND:
6492 case ISD::FRINT:
6493 case ISD::LRINT:
6494 case ISD::LLRINT:
6495 case ISD::FNEARBYINT:
6496 case ISD::FLDEXP: {
6497 if (SNaN)
6498 return true;
6499 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
6500 }
6501 case ISD::FABS:
6502 case ISD::FNEG:
6503 case ISD::FCOPYSIGN: {
6504 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
6505 }
6506 case ISD::SELECT:
6507 return isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1) &&
6508 isKnownNeverNaN(Op.getOperand(2), DemandedElts, SNaN, Depth + 1);
6509 case ISD::FP_EXTEND:
6510 case ISD::FP_ROUND: {
6511 if (SNaN)
6512 return true;
6513 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
6514 }
6515 case ISD::SINT_TO_FP:
6516 case ISD::UINT_TO_FP:
6517 return true;
6518 case ISD::FSQRT: // Need is known positive
6519 case ISD::FLOG:
6520 case ISD::FLOG2:
6521 case ISD::FLOG10:
6522 case ISD::FPOWI:
6523 case ISD::FPOW: {
6524 if (SNaN)
6525 return true;
6526 // TODO: Refine on operand
6527 return false;
6528 }
6529 case ISD::FMINNUM:
6530 case ISD::FMAXNUM:
6531 case ISD::FMINIMUMNUM:
6532 case ISD::FMAXIMUMNUM: {
6533 // Only one needs to be known not-nan, since it will be returned if the
6534 // other ends up being one.
6535 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) ||
6536 isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
6537 }
6538 case ISD::FMINNUM_IEEE:
6539 case ISD::FMAXNUM_IEEE: {
6540 if (SNaN)
6541 return true;
6542 // This can return a NaN if either operand is an sNaN, or if both operands
6543 // are NaN.
6544 return (isKnownNeverNaN(Op.getOperand(0), DemandedElts, false, Depth + 1) &&
6545 isKnownNeverSNaN(Op.getOperand(1), DemandedElts, Depth + 1)) ||
6546 (isKnownNeverNaN(Op.getOperand(1), DemandedElts, false, Depth + 1) &&
6547 isKnownNeverSNaN(Op.getOperand(0), DemandedElts, Depth + 1));
6548 }
6549 case ISD::FMINIMUM:
6550 case ISD::FMAXIMUM: {
6551 // TODO: Does this quiet or return the origina NaN as-is?
6552 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) &&
6553 isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
6554 }
6556 SDValue Src = Op.getOperand(0);
6557 auto *Idx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6558 EVT SrcVT = Src.getValueType();
6559 if (SrcVT.isFixedLengthVector() && Idx &&
6560 Idx->getAPIntValue().ult(SrcVT.getVectorNumElements())) {
6561 APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
6562 Idx->getZExtValue());
6563 return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);
6564 }
6565 return isKnownNeverNaN(Src, SNaN, Depth + 1);
6566 }
6568 SDValue Src = Op.getOperand(0);
6569 if (Src.getValueType().isFixedLengthVector()) {
6570 unsigned Idx = Op.getConstantOperandVal(1);
6571 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6572 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
6573 return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);
6574 }
6575 return isKnownNeverNaN(Src, SNaN, Depth + 1);
6576 }
6577 case ISD::INSERT_SUBVECTOR: {
6578 SDValue BaseVector = Op.getOperand(0);
6579 SDValue SubVector = Op.getOperand(1);
6580 EVT BaseVectorVT = BaseVector.getValueType();
6581 if (BaseVectorVT.isFixedLengthVector()) {
6582 unsigned Idx = Op.getConstantOperandVal(2);
6583 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6584 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6585
6586 // Clear/Extract the bits at the position where the subvector will be
6587 // inserted.
6588 APInt DemandedMask =
6589 APInt::getBitsSet(NumBaseElts, Idx, Idx + NumSubElts);
6590 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6591 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
6592
6593 bool NeverNaN = true;
6594 if (!DemandedSrcElts.isZero())
6595 NeverNaN &=
6596 isKnownNeverNaN(BaseVector, DemandedSrcElts, SNaN, Depth + 1);
6597 if (NeverNaN && !DemandedSubElts.isZero())
6598 NeverNaN &=
6599 isKnownNeverNaN(SubVector, DemandedSubElts, SNaN, Depth + 1);
6600 return NeverNaN;
6601 }
6602 return isKnownNeverNaN(BaseVector, SNaN, Depth + 1) &&
6603 isKnownNeverNaN(SubVector, SNaN, Depth + 1);
6604 }
6605 case ISD::BUILD_VECTOR: {
6606 unsigned NumElts = Op.getNumOperands();
6607 for (unsigned I = 0; I != NumElts; ++I)
6608 if (DemandedElts[I] &&
6609 !isKnownNeverNaN(Op.getOperand(I), SNaN, Depth + 1))
6610 return false;
6611 return true;
6612 }
6613 case ISD::SPLAT_VECTOR:
6614 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
6615 case ISD::AssertNoFPClass: {
6616 FPClassTest NoFPClass =
6617 static_cast<FPClassTest>(Op.getConstantOperandVal(1));
6618 if ((NoFPClass & fcNan) == fcNan)
6619 return true;
6620 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6621 return true;
6622 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
6623 }
6624 default:
6625 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6626 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6627 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, *this, SNaN,
6628 Depth);
6629 }
6630 break;
6631 }
6632
6633 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6634 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, NanMask, Depth);
6635 return Known.isKnownNever(NanMask);
6636}
6637
6639 APInt DemandedElts = getDemandAllEltsMask(Op);
6640 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6641}
6642
6644 const APInt &DemandedElts,
6645 unsigned Depth) const {
6646 assert(!DemandedElts.isZero() && "No demanded elements");
6647 EVT VT = Op.getValueType();
6649 computeKnownFPClass(Op, DemandedElts, fcZero | fcSubnormal, Depth);
6650 return Known.isKnownNeverLogicalZero(getDenormalMode(VT));
6651}
6652
6654 APInt DemandedElts = getDemandAllEltsMask(Op);
6655 return isKnownNeverZero(Op, DemandedElts, Depth);
6656}
6657
6659 unsigned Depth) const {
6660 if (Depth >= MaxRecursionDepth)
6661 return false; // Limit search depth.
6662
6663 EVT OpVT = Op.getValueType();
6664 unsigned BitWidth = OpVT.getScalarSizeInBits();
6665
6666 assert(!Op.getValueType().isFloatingPoint() &&
6667 "Floating point types unsupported - use isKnownNeverLogicalZero");
6668
6669 // If the value is a constant, we can obviously see if it is a zero or not.
6670 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6671 APInt V = C->getAPIntValue().zextOrTrunc(BitWidth);
6672 return !V.isZero();
6673 };
6674
6675 if (ISD::matchUnaryPredicate(Op, DemandedElts, IsNeverZero,
6676 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
6677 return true;
6678
6679 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6680 // some degree.
6681 switch (Op.getOpcode()) {
6682 default:
6683 break;
6684
6686 SDValue InVec = Op.getOperand(0);
6687 SDValue EltNo = Op.getOperand(1);
6688 EVT VecVT = InVec.getValueType();
6689
6690 // Skip scalable vectors or implicit extensions.
6691 if (VecVT.isScalableVector() ||
6692 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6693 break;
6694
6695 // If we know the element index, just demand that vector element, else for
6696 // an unknown element index, ignore DemandedElts and demand them all.
6697 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6698 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
6699 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
6700 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
6701 DemandedSrcElts =
6702 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
6703
6704 return isKnownNeverZero(InVec, DemandedSrcElts, Depth + 1);
6705 }
6706
6707 case ISD::OR:
6708 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) ||
6709 isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6710
6711 case ISD::VSELECT:
6712 case ISD::SELECT:
6713 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) &&
6714 isKnownNeverZero(Op.getOperand(2), DemandedElts, Depth + 1);
6715
6716 case ISD::SHL: {
6717 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6718 return isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6719 KnownBits ValKnown =
6720 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
6721 // 1 << X is never zero.
6722 if (ValKnown.One[0])
6723 return true;
6724 // If max shift cnt of known ones is non-zero, result is non-zero.
6725 APInt MaxCnt = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1)
6726 .getMaxValue();
6727 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
6728 !ValKnown.One.shl(MaxCnt).isZero())
6729 return true;
6730 break;
6731 }
6732
6733 case ISD::VECTOR_SHUFFLE: {
6734 if (Op.getValueType().isScalableVector())
6735 return false;
6736
6737 unsigned NumElts = DemandedElts.getBitWidth();
6738
6739 // All demanded elements from LHS and RHS must be known non-zero.
6740 // Demanded elements with undef shuffle mask elements are unknown.
6741
6742 APInt DemandedLHS, DemandedRHS;
6743 auto *SVN = cast<ShuffleVectorSDNode>(Op);
6744 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6745 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
6746 DemandedLHS, DemandedRHS))
6747 return false;
6748
6749 return (!DemandedLHS ||
6750 isKnownNeverZero(Op.getOperand(0), DemandedLHS, Depth + 1)) &&
6751 (!DemandedRHS ||
6752 isKnownNeverZero(Op.getOperand(1), DemandedRHS, Depth + 1));
6753 }
6754
6755 case ISD::UADDSAT:
6756 case ISD::UMAX:
6757 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) ||
6758 isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6759
6760 case ISD::UMIN:
6761 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) &&
6762 isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6763
6764 // For smin/smax: If either operand is known negative/positive
6765 // respectively we don't need the other to be known at all.
6766 case ISD::SMAX: {
6767 KnownBits Op1 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
6768 if (Op1.isStrictlyPositive())
6769 return true;
6770
6771 KnownBits Op0 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
6772 if (Op0.isStrictlyPositive())
6773 return true;
6774
6775 if (Op1.isNonZero() && Op0.isNonZero())
6776 return true;
6777
6778 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) &&
6779 isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6780 }
6781 case ISD::SMIN: {
6782 KnownBits Op1 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
6783 if (Op1.isNegative())
6784 return true;
6785
6786 KnownBits Op0 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
6787 if (Op0.isNegative())
6788 return true;
6789
6790 if (Op1.isNonZero() && Op0.isNonZero())
6791 return true;
6792
6793 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) &&
6794 isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6795 }
6796
6797 case ISD::ROTL:
6798 case ISD::ROTR:
6799 case ISD::BITREVERSE:
6800 case ISD::BSWAP:
6801 case ISD::CTPOP:
6802 case ISD::ABS:
6804 return isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6805
6806 case ISD::SRA:
6807 case ISD::SRL: {
6808 if (Op->getFlags().hasExact())
6809 return isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6810 KnownBits ValKnown =
6811 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
6812 if (ValKnown.isNegative())
6813 return true;
6814 // If max shift cnt of known ones is non-zero, result is non-zero.
6815 APInt MaxCnt = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1)
6816 .getMaxValue();
6817 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
6818 !ValKnown.One.lshr(MaxCnt).isZero())
6819 return true;
6820 break;
6821 }
6822 case ISD::UDIV:
6823 case ISD::SDIV:
6824 // div exact can only produce a zero if the dividend is zero.
6825 // TODO: For udiv this is also true if Op1 u<= Op0
6826 if (Op->getFlags().hasExact())
6827 return isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6828 break;
6829
6830 case ISD::ADD:
6831 if (Op->getFlags().hasNoUnsignedWrap())
6832 if (isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1) ||
6833 isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1))
6834 return true;
6835 // TODO: There are a lot more cases we can prove for add.
6836 break;
6837
6838 case ISD::SUB: {
6839 if (isNullConstant(Op.getOperand(0)))
6840 return isKnownNeverZero(Op.getOperand(1), DemandedElts, Depth + 1);
6841
6842 std::optional<bool> ne = KnownBits::ne(
6843 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1),
6844 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1));
6845 return ne && *ne;
6846 }
6847
6848 case ISD::MUL:
6849 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6850 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
6851 isKnownNeverZero(Op.getOperand(0), Depth + 1))
6852 return true;
6853 break;
6854
6855 case ISD::ZERO_EXTEND:
6856 case ISD::SIGN_EXTEND:
6857 return isKnownNeverZero(Op.getOperand(0), DemandedElts, Depth + 1);
6858 case ISD::VSCALE: {
6860 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
6861 ConstantRange CR =
6862 getVScaleRange(&F, Op.getScalarValueSizeInBits()).multiply(Multiplier);
6863 if (!CR.contains(APInt(CR.getBitWidth(), 0)))
6864 return true;
6865 break;
6866 }
6867 }
6868
6869 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6870}
6871
6874 return !C1->isNegative();
6875
6876 switch (Op.getOpcode()) {
6877 case ISD::FABS:
6878 case ISD::FEXP:
6879 case ISD::FEXP2:
6880 case ISD::FEXP10:
6881 return true;
6882 default:
6883 return false;
6884 }
6885
6886 llvm_unreachable("covered opcode switch");
6887}
6888
6889/// Return true if the standard nsz semantics may not apply for this opcode.
6890/// Opcode is assumed to take float inputs.
6891/// FIXME: Target min/max nodes (e.g. X86ISD::FMINC) aren't included.
6892static bool hasSpecialNszSemantics(unsigned Opcode) {
6893 switch (Opcode) {
6894 case ISD::FMINNUM:
6895 case ISD::FMAXNUM:
6896 case ISD::FMINNUM_IEEE:
6897 case ISD::FMAXNUM_IEEE:
6898 case ISD::FMINIMUM:
6899 case ISD::FMAXIMUM:
6900 case ISD::FMINIMUMNUM:
6901 case ISD::FMAXIMUMNUM:
6906 case ISD::PSEUDO_FMIN:
6907 case ISD::PSEUDO_FMAX:
6916 case ISD::VP_REDUCE_FMIN:
6917 case ISD::VP_REDUCE_FMAX:
6918 case ISD::VP_REDUCE_FMINIMUM:
6919 case ISD::VP_REDUCE_FMAXIMUM:
6920 return true;
6921 default:
6922 return false;
6923 }
6924}
6925
6927 assert(Use.getValueType().isFloatingPoint());
6928 const SDNode *User = Use.getUser();
6929 unsigned Opcode = User->getOpcode();
6930 if (User->getFlags().hasNoSignedZeros() && !hasSpecialNszSemantics(Opcode))
6931 return true;
6932
6933 unsigned OperandNo = Use.getOperandNo();
6934 // Check if this use is insensitive to the sign of zero
6935 switch (Opcode) {
6936 case ISD::SETCC:
6937 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6938 case ISD::FABS:
6939 // fabs always produces +0.0.
6940 return true;
6941 case ISD::FCOPYSIGN:
6942 // copysign overwrites the sign bit of the first operand.
6943 return OperandNo == 0;
6944 case ISD::FADD:
6945 case ISD::FSUB: {
6946 // Arithmetic with non-zero constants fixes the uncertainty around the
6947 // sign bit.
6948 SDValue Other = User->getOperand(1 - OperandNo);
6950 }
6951 case ISD::FP_TO_SINT:
6952 case ISD::FP_TO_UINT:
6953 // fp-to-int conversions normalize signed zeros.
6954 return true;
6955 case ISD::FMINNUM:
6956 case ISD::FMINNUM_IEEE:
6957 case ISD::FMINIMUM:
6958 case ISD::FMINIMUMNUM:
6959 case ISD::FMAXNUM:
6960 case ISD::FMAXNUM_IEEE:
6961 case ISD::FMAXIMUM:
6962 case ISD::FMAXIMUMNUM:
6963 case ISD::PSEUDO_FMIN:
6964 case ISD::PSEUDO_FMAX: {
6965 // If other value always beats zero, the zero can be ignored.
6966 bool IsMin = Opcode == ISD::FMINNUM || Opcode == ISD::FMINNUM_IEEE ||
6967 Opcode == ISD::FMINIMUM || Opcode == ISD::FMINIMUMNUM ||
6968 Opcode == ISD::PSEUDO_FMIN;
6969 FPClassTest AlwaysBeatsZero =
6971 SDValue Other = User->getOperand(1 - OperandNo);
6972 KnownFPClass Known = computeKnownFPClass(Other, ~AlwaysBeatsZero);
6973 return Known.isKnownNever(~AlwaysBeatsZero);
6974 }
6975 default:
6976 return false;
6977 }
6978}
6979
6981 assert(Op->getNumValues() == 1);
6982 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6983 // regression. Ideally, this should be implemented as a demanded-bits
6984 // optimization that stems from the users.
6985 if (Op->use_size() > 2)
6986 return false;
6987 return all_of(Op->uses(),
6988 [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6989}
6990
6992 // Check the obvious case.
6993 if (A == B) return true;
6994
6995 // For negative and positive zero.
6998 if (CA->isZero() && CB->isZero()) return true;
6999
7000 // Otherwise they may not be equal.
7001 return false;
7002}
7003
7004// Only bits set in Mask must be negated, other bits may be arbitrary.
7006 if (isBitwiseNot(V, AllowUndefs))
7007 return V.getOperand(0);
7008
7009 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
7010 // bits in the non-extended part.
7011 ConstantSDNode *MaskC = isConstOrConstSplat(Mask);
7012 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
7013 return SDValue();
7014 SDValue ExtArg = V.getOperand(0);
7015 if (ExtArg.getScalarValueSizeInBits() >=
7016 MaskC->getAPIntValue().getActiveBits() &&
7017 isBitwiseNot(ExtArg, AllowUndefs) &&
7018 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&
7019 ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())
7020 return ExtArg.getOperand(0).getOperand(0);
7021 return SDValue();
7022}
7023
7025 // Match masked merge pattern (X & ~M) op (Y & M)
7026 // Including degenerate case (X & ~M) op M
7027 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
7028 SDValue Other) {
7029 if (SDValue NotOperand =
7030 getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {
7031 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
7032 NotOperand->getOpcode() == ISD::TRUNCATE)
7033 NotOperand = NotOperand->getOperand(0);
7034
7035 if (Other == NotOperand)
7036 return true;
7037 if (Other->getOpcode() == ISD::AND)
7038 return NotOperand == Other->getOperand(0) ||
7039 NotOperand == Other->getOperand(1);
7040 }
7041 return false;
7042 };
7043
7044 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
7045 A = A->getOperand(0);
7046
7047 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
7048 B = B->getOperand(0);
7049
7050 if (A->getOpcode() == ISD::AND)
7051 return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||
7052 MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);
7053 return false;
7054}
7055
7056// FIXME: unify with llvm::haveNoCommonBitsSet.
7058 assert(A.getValueType() == B.getValueType() &&
7059 "Values must have the same type");
7062 return true;
7065}
7066
7067static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
7068 SelectionDAG &DAG) {
7069 if (cast<ConstantSDNode>(Step)->isZero())
7070 return DAG.getConstant(0, DL, VT);
7071
7072 return SDValue();
7073}
7074
7077 SelectionDAG &DAG) {
7078 int NumOps = Ops.size();
7079 assert(NumOps != 0 && "Can't build an empty vector!");
7080 assert(!VT.isScalableVector() &&
7081 "BUILD_VECTOR cannot be used with scalable types");
7082 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
7083 "Incorrect element count in BUILD_VECTOR!");
7084
7085 // BUILD_VECTOR of UNDEFs is UNDEF.
7086 bool AllPoison = true;
7087 if (llvm::all_of(Ops, [&AllPoison](SDValue Op) {
7088 AllPoison &= Op.getOpcode() == ISD::POISON;
7089 return Op.isUndef();
7090 }))
7091 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7092
7093 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
7094 SDValue IdentitySrc;
7095 bool IsIdentity = true;
7096 for (int i = 0; i != NumOps; ++i) {
7097 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7098 Ops[i].getOperand(0).getValueType() != VT ||
7099 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
7100 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
7101 Ops[i].getConstantOperandAPInt(1) != i) {
7102 IsIdentity = false;
7103 break;
7104 }
7105 IdentitySrc = Ops[i].getOperand(0);
7106 }
7107 if (IsIdentity)
7108 return IdentitySrc;
7109
7110 return SDValue();
7111}
7112
7113/// Try to simplify vector concatenation to an input value, undef, or build
7114/// vector.
7117 SelectionDAG &DAG) {
7118 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
7120 [Ops](SDValue Op) {
7121 return Ops[0].getValueType() == Op.getValueType();
7122 }) &&
7123 "Concatenation of vectors with inconsistent value types!");
7124 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
7125 VT.getVectorElementCount() &&
7126 "Incorrect element count in vector concatenation!");
7127
7128 if (Ops.size() == 1)
7129 return Ops[0];
7130
7131 // Concat of UNDEFs is UNDEF.
7132 bool AllPoison = true;
7133 if (llvm::all_of(Ops, [&AllPoison](SDValue Op) {
7134 AllPoison &= Op.getOpcode() == ISD::POISON;
7135 return Op.isUndef();
7136 }))
7137 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7138
7139 // Scan the operands and look for extract operations from a single source
7140 // that correspond to insertion at the same location via this concatenation:
7141 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
7142 SDValue IdentitySrc;
7143 bool IsIdentity = true;
7144 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
7145 SDValue Op = Ops[i];
7146 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
7147 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
7148 Op.getOperand(0).getValueType() != VT ||
7149 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
7150 Op.getConstantOperandVal(1) != IdentityIndex) {
7151 IsIdentity = false;
7152 break;
7153 }
7154 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
7155 "Unexpected identity source vector for concat of extracts");
7156 IdentitySrc = Op.getOperand(0);
7157 }
7158 if (IsIdentity) {
7159 assert(IdentitySrc && "Failed to set source vector of extracts");
7160 return IdentitySrc;
7161 }
7162
7163 // The code below this point is only designed to work for fixed width
7164 // vectors, so we bail out for now.
7165 if (VT.isScalableVector())
7166 return SDValue();
7167
7168 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
7169 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
7170 // BUILD_VECTOR.
7171 // FIXME: Add support for SCALAR_TO_VECTOR as well.
7172 EVT SVT = VT.getScalarType();
7174 for (SDValue Op : Ops) {
7175 EVT OpVT = Op.getValueType();
7176 if (Op.getOpcode() == ISD::POISON)
7177 Elts.append(OpVT.getVectorNumElements(), DAG.getPOISON(SVT));
7178 else if (Op.getOpcode() == ISD::UNDEF)
7179 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
7180 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7181 Elts.append(Op->op_begin(), Op->op_end());
7182 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7183 OpVT.getVectorNumElements() == 1 &&
7184 isNullConstant(Op.getOperand(2)))
7185 Elts.push_back(Op.getOperand(1));
7186 else
7187 return SDValue();
7188 }
7189
7190 // BUILD_VECTOR requires all inputs to be of the same type, find the
7191 // maximum type and extend them all.
7192 for (SDValue Op : Elts)
7193 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
7194
7195 if (SVT.bitsGT(VT.getScalarType())) {
7196 for (SDValue &Op : Elts) {
7197 if (Op.getOpcode() == ISD::POISON)
7198 Op = DAG.getPOISON(SVT);
7199 else if (Op.getOpcode() == ISD::UNDEF)
7200 Op = DAG.getUNDEF(SVT);
7201 else
7202 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
7203 ? DAG.getZExtOrTrunc(Op, DL, SVT)
7204 : DAG.getSExtOrTrunc(Op, DL, SVT);
7205 }
7206 }
7207
7208 SDValue V = DAG.getBuildVector(VT, DL, Elts);
7209 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
7210 return V;
7211}
7212
7213/// Gets or creates the specified node.
7214SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7215 SDVTList VTs = getVTList(VT);
7216 SDNodeKey ID(Opcode, VTs, {});
7217 FoldingSetInsertToken InsertToken;
7218 if (SDNode *E = lookupNode(ID, DL, InsertToken))
7219 return SDValue(E, 0);
7220
7221 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7222 CSEMap.insert(N, InsertToken);
7223
7224 InsertNode(N);
7225 SDValue V = SDValue(N, 0);
7226 NewSDValueDbgMsg(V, "Creating new node: ", this);
7227 return V;
7228}
7229
7230SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7231 SDValue N1) {
7232 SDNodeFlags Flags;
7233 if (Inserter)
7234 Flags = Inserter->getFlags();
7235 return getNode(Opcode, DL, VT, N1, Flags);
7236}
7237
7238SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7239 SDValue N1, const SDNodeFlags Flags) {
7240 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7241
7242 // Constant fold unary operations with a vector integer or float operand.
7243 switch (Opcode) {
7244 default:
7245 // FIXME: Entirely reasonable to perform folding of other unary
7246 // operations here as the need arises.
7247 break;
7248 case ISD::FNEG:
7249 case ISD::FABS:
7250 case ISD::FCEIL:
7251 case ISD::FTRUNC:
7252 case ISD::FFLOOR:
7253 case ISD::FP_EXTEND:
7254 case ISD::FP_TO_SINT:
7255 case ISD::FP_TO_UINT:
7256 case ISD::FP_TO_FP16:
7257 case ISD::FP_TO_BF16:
7258 case ISD::TRUNCATE:
7259 case ISD::ANY_EXTEND:
7260 case ISD::ZERO_EXTEND:
7261 case ISD::SIGN_EXTEND:
7262 case ISD::UINT_TO_FP:
7263 case ISD::SINT_TO_FP:
7264 case ISD::FP16_TO_FP:
7265 case ISD::BF16_TO_FP:
7266 case ISD::BITCAST:
7267 case ISD::ABS:
7269 case ISD::BITREVERSE:
7270 case ISD::BSWAP:
7271 case ISD::CTLZ:
7273 case ISD::CTTZ:
7275 case ISD::CTPOP:
7276 case ISD::PARITY:
7277 case ISD::CTLS:
7278 case ISD::VECREDUCE_ADD:
7283 case ISD::VECREDUCE_MUL:
7284 case ISD::VECREDUCE_AND:
7285 case ISD::VECREDUCE_OR:
7286 case ISD::VECREDUCE_XOR:
7287 case ISD::STEP_VECTOR: {
7288 SDValue Ops = {N1};
7289 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops, Flags))
7290 return Fold;
7291 }
7292 }
7293
7294 unsigned OpOpcode = N1.getNode()->getOpcode();
7295 switch (Opcode) {
7296 case ISD::STEP_VECTOR:
7297 assert(VT.isScalableVector() &&
7298 "STEP_VECTOR can only be used with scalable types");
7299 assert(OpOpcode == ISD::TargetConstant &&
7300 VT.getVectorElementType() == N1.getValueType() &&
7301 "Unexpected step operand");
7302 break;
7303 case ISD::FREEZE:
7304 assert(VT == N1.getValueType() && "Unexpected VT!");
7306 return N1;
7307 break;
7308 case ISD::TokenFactor:
7309 case ISD::MERGE_VALUES:
7311 return N1; // Factor, merge or concat of one node? No need.
7312 case ISD::BUILD_VECTOR: {
7313 // Attempt to simplify BUILD_VECTOR.
7314 SDValue Ops[] = {N1};
7315 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7316 return V;
7317 break;
7318 }
7319 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7320 case ISD::FP_EXTEND:
7322 "Invalid FP cast!");
7323 if (N1.getValueType() == VT) return N1; // noop conversion.
7324 assert((!VT.isVector() || VT.getVectorElementCount() ==
7326 "Vector element count mismatch!");
7327 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7328 if (N1.isUndef())
7329 return getUNDEF(VT);
7330 break;
7331 case ISD::FP_TO_SINT:
7332 case ISD::FP_TO_UINT:
7333 if (N1.isUndef())
7334 return getUNDEF(VT);
7335 break;
7336 case ISD::SINT_TO_FP:
7337 case ISD::UINT_TO_FP:
7338 // [us]itofp(undef) = 0, because the result value is bounded.
7339 if (N1.isUndef())
7340 return getConstantFP(0.0, DL, VT);
7341 break;
7342 case ISD::SIGN_EXTEND:
7343 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7344 "Invalid SIGN_EXTEND!");
7345 assert(VT.isVector() == N1.getValueType().isVector() &&
7346 "SIGN_EXTEND result type type should be vector iff the operand "
7347 "type is vector!");
7348 if (N1.getValueType() == VT) return N1; // noop extension
7349 assert((!VT.isVector() || VT.getVectorElementCount() ==
7351 "Vector element count mismatch!");
7352 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7353 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7354 SDNodeFlags Flags;
7355 if (OpOpcode == ISD::ZERO_EXTEND)
7356 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7357 SDValue NewVal = getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
7358 transferDbgValues(N1, NewVal);
7359 return NewVal;
7360 }
7361
7362 if (OpOpcode == ISD::POISON)
7363 return getPOISON(VT);
7364
7365 if (N1.isUndef())
7366 // sext(undef) = 0, because the top bits will all be the same.
7367 return getConstant(0, DL, VT);
7368
7369 // Skip unnecessary sext_inreg pattern:
7370 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7371 if (OpOpcode == ISD::TRUNCATE) {
7372 SDValue OpOp = N1.getOperand(0);
7373 if (OpOp.getValueType() == VT) {
7374 unsigned NumSignExtBits =
7376 if (ComputeNumSignBits(OpOp) > NumSignExtBits) {
7377 transferDbgValues(N1, OpOp);
7378 return OpOp;
7379 }
7380 }
7381 }
7382 break;
7383 case ISD::ZERO_EXTEND:
7384 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7385 "Invalid ZERO_EXTEND!");
7386 assert(VT.isVector() == N1.getValueType().isVector() &&
7387 "ZERO_EXTEND result type type should be vector iff the operand "
7388 "type is vector!");
7389 if (N1.getValueType() == VT) return N1; // noop extension
7390 assert((!VT.isVector() || VT.getVectorElementCount() ==
7392 "Vector element count mismatch!");
7393 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7394 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7395 SDNodeFlags Flags;
7396 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7397 SDValue NewVal =
7398 getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);
7399 transferDbgValues(N1, NewVal);
7400 return NewVal;
7401 }
7402
7403 if (OpOpcode == ISD::POISON)
7404 return getPOISON(VT);
7405
7406 if (N1.isUndef())
7407 // zext(undef) = 0, because the top bits will be zero.
7408 return getConstant(0, DL, VT);
7409
7410 // Skip unnecessary zext_inreg pattern:
7411 // (zext (trunc x)) -> x iff the upper bits are known zero.
7412 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7413 // use to recognise zext_inreg patterns.
7414 if (OpOpcode == ISD::TRUNCATE) {
7415 SDValue OpOp = N1.getOperand(0);
7416 if (OpOp.getValueType() == VT) {
7417 if (OpOp.getOpcode() != ISD::AND) {
7420 if (MaskedValueIsZero(OpOp, HiBits)) {
7421 transferDbgValues(N1, OpOp);
7422 return OpOp;
7423 }
7424 }
7425 }
7426 }
7427 break;
7428 case ISD::ANY_EXTEND:
7429 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7430 "Invalid ANY_EXTEND!");
7431 assert(VT.isVector() == N1.getValueType().isVector() &&
7432 "ANY_EXTEND result type type should be vector iff the operand "
7433 "type is vector!");
7434 if (N1.getValueType() == VT) return N1; // noop extension
7435 assert((!VT.isVector() || VT.getVectorElementCount() ==
7437 "Vector element count mismatch!");
7438 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7439
7440 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7441 OpOpcode == ISD::ANY_EXTEND) {
7442 SDNodeFlags Flags;
7443 if (OpOpcode == ISD::ZERO_EXTEND)
7444 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7445 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7446 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
7447 }
7448 if (N1.isUndef())
7449 return getUNDEF(VT);
7450
7451 // (ext (trunc x)) -> x
7452 if (OpOpcode == ISD::TRUNCATE) {
7453 SDValue OpOp = N1.getOperand(0);
7454 if (OpOp.getValueType() == VT) {
7455 transferDbgValues(N1, OpOp);
7456 return OpOp;
7457 }
7458 }
7459 break;
7460 case ISD::TRUNCATE:
7461 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7462 "Invalid TRUNCATE!");
7463 assert(VT.isVector() == N1.getValueType().isVector() &&
7464 "TRUNCATE result type type should be vector iff the operand "
7465 "type is vector!");
7466 if (N1.getValueType() == VT) return N1; // noop truncate
7467 assert((!VT.isVector() || VT.getVectorElementCount() ==
7469 "Vector element count mismatch!");
7470 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7471 if (OpOpcode == ISD::TRUNCATE)
7472 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
7473 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7474 OpOpcode == ISD::ANY_EXTEND) {
7475 // If the source is smaller than the dest, we still need an extend.
7477 VT.getScalarType())) {
7478 SDNodeFlags Flags;
7479 if (OpOpcode == ISD::ZERO_EXTEND)
7480 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7481 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
7482 }
7483 if (N1.getOperand(0).getValueType().bitsGT(VT))
7484 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
7485 return N1.getOperand(0);
7486 }
7487 if (N1.isUndef())
7488 return getUNDEF(VT);
7489 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7490 return getVScale(DL, VT,
7492 break;
7496 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7497 assert(N1.getValueType().bitsLE(VT) &&
7498 "The input must be the same size or smaller than the result.");
7501 "The destination vector type must have fewer lanes than the input.");
7502 break;
7503 case ISD::ABS:
7504 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7505 if (N1.isUndef())
7506 return getConstant(0, DL, VT);
7507 break;
7509 assert(VT.isInteger() && VT == N1.getValueType() &&
7510 "Invalid ABS_MIN_POISON!");
7511 if (N1.isUndef())
7512 return getConstant(0, DL, VT);
7513 break;
7514 case ISD::BSWAP:
7515 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7516 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7517 "BSWAP types must be a multiple of 16 bits!");
7518 if (N1.isUndef())
7519 return getUNDEF(VT);
7520 // bswap(bswap(X)) -> X.
7521 if (OpOpcode == ISD::BSWAP)
7522 return N1.getOperand(0);
7523 break;
7524 case ISD::BITREVERSE:
7525 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7526 if (N1.isUndef())
7527 return getUNDEF(VT);
7528 break;
7529 case ISD::BITCAST:
7531 "Cannot BITCAST between types of different sizes!");
7532 if (VT == N1.getValueType()) return N1; // noop conversion.
7533 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7534 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));
7535 if (N1.isUndef())
7536 return getUNDEF(VT);
7537 break;
7539 assert(VT.isVector() && !N1.getValueType().isVector() &&
7540 (VT.getVectorElementType() == N1.getValueType() ||
7542 N1.getValueType().isInteger() &&
7544 "Illegal SCALAR_TO_VECTOR node!");
7545 if (N1.isUndef())
7546 return getUNDEF(VT);
7547 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7548 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7550 N1.getConstantOperandVal(1) == 0 &&
7551 N1.getOperand(0).getValueType() == VT)
7552 return N1.getOperand(0);
7553 break;
7554 case ISD::FNEG:
7555 // Negation of an unknown bag of bits is still completely undefined.
7556 if (N1.isUndef())
7557 return getUNDEF(VT);
7558
7559 if (OpOpcode == ISD::FNEG) // --X -> X
7560 return N1.getOperand(0);
7561 break;
7562 case ISD::FABS:
7563 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7564 return getNode(ISD::FABS, DL, VT, N1.getOperand(0));
7565 break;
7566 case ISD::VSCALE:
7567 assert(VT == N1.getValueType() && "Unexpected VT!");
7568 break;
7569 case ISD::CTPOP:
7570 if (N1.getValueType().getScalarType() == MVT::i1)
7571 return N1;
7572 break;
7573 case ISD::CTLZ:
7574 case ISD::CTTZ:
7575 if (N1.getValueType().getScalarType() == MVT::i1)
7576 return getNOT(DL, N1, N1.getValueType());
7577 break;
7578 case ISD::CTLS:
7579 if (N1.getValueType().getScalarType() == MVT::i1)
7580 return getConstant(0, DL, VT);
7581 break;
7582 case ISD::VECREDUCE_ADD:
7583 if (N1.getValueType().getScalarType() == MVT::i1)
7584 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);
7585 break;
7588 if (N1.getValueType().getScalarType() == MVT::i1)
7589 return getNode(ISD::VECREDUCE_OR, DL, VT, N1);
7590 break;
7593 if (N1.getValueType().getScalarType() == MVT::i1)
7594 return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
7595 break;
7597 // With a single lane, the result is true iff that lane is false.
7599 return getLogicalNOT(DL, N1, VT);
7600 break;
7601 case ISD::VECTOR_REPEAT:
7603 "VECTOR_REPEAT requires a fixed-length vector operand");
7604 assert(VT.isScalableVector() &&
7605 "VECTOR_REPEAT requires a scalable vector result");
7608 "VECTOR_REPEAT operand and result element counts must match");
7609 if (VT.getVectorMinNumElements() == 1)
7610 return getSplatVector(
7611 VT, DL, getExtractVectorElt(DL, VT.getVectorElementType(), N1, 0));
7612 break;
7613 case ISD::SPLAT_VECTOR:
7614 assert(VT.isVector() && "Wrong return type!");
7615 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7616 // that for now.
7618 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7620 N1.getValueType().isInteger() &&
7622 "Wrong operand type!");
7623 break;
7624 }
7625
7626 SDNode *N;
7627 SDVTList VTs = getVTList(VT);
7628 SDValue Ops[] = {N1};
7629 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7630 SDNodeKey ID(Opcode, VTs, Ops);
7631 FoldingSetInsertToken InsertToken;
7632 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
7633 E->intersectFlagsWith(Flags);
7634 return SDValue(E, 0);
7635 }
7636
7637 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7638 N->setFlags(Flags);
7639 createOperands(N, Ops);
7640 CSEMap.insert(N, InsertToken);
7641 } else {
7642 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7643 createOperands(N, Ops);
7644 }
7645
7646 InsertNode(N);
7647 SDValue V = SDValue(N, 0);
7648 NewSDValueDbgMsg(V, "Creating new node: ", this);
7649 return V;
7650}
7651
7652static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth) {
7653 switch (Opcode) {
7654 default:
7655 llvm_unreachable("Unexpected integer identity opcode");
7656 case ISD::ADD:
7657 case ISD::OR:
7658 case ISD::XOR:
7659 case ISD::UMAX:
7660 return APInt::getZero(BitWidth);
7661 case ISD::MUL:
7662 return APInt(BitWidth, 1);
7663 case ISD::AND:
7664 case ISD::UMIN:
7666 case ISD::SMAX:
7668 case ISD::SMIN:
7670 }
7671}
7672
7673static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7674 const APInt &C2) {
7675 switch (Opcode) {
7676 case ISD::ADD: return C1 + C2;
7677 case ISD::SUB: return C1 - C2;
7678 case ISD::MUL: return C1 * C2;
7679 case ISD::AND: return C1 & C2;
7680 case ISD::OR: return C1 | C2;
7681 case ISD::XOR: return C1 ^ C2;
7682 case ISD::SHL: return C1 << C2;
7683 case ISD::SRL: return C1.lshr(C2);
7684 case ISD::SRA: return C1.ashr(C2);
7685 case ISD::ROTL: return C1.rotl(C2);
7686 case ISD::ROTR: return C1.rotr(C2);
7687 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
7688 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
7689 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
7690 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
7691 case ISD::SADDSAT: return C1.sadd_sat(C2);
7692 case ISD::UADDSAT: return C1.uadd_sat(C2);
7693 case ISD::SSUBSAT: return C1.ssub_sat(C2);
7694 case ISD::USUBSAT: return C1.usub_sat(C2);
7695 case ISD::SSHLSAT: return C1.sshl_sat(C2);
7696 case ISD::USHLSAT: return C1.ushl_sat(C2);
7697 case ISD::UDIV:
7698 if (!C2.getBoolValue())
7699 break;
7700 return C1.udiv(C2);
7701 case ISD::UREM:
7702 if (!C2.getBoolValue())
7703 break;
7704 return C1.urem(C2);
7705 case ISD::SDIV:
7706 if (!C2.getBoolValue())
7707 break;
7708 return C1.sdiv(C2);
7709 case ISD::SREM:
7710 if (!C2.getBoolValue())
7711 break;
7712 return C1.srem(C2);
7713 case ISD::AVGFLOORS:
7714 return APIntOps::avgFloorS(C1, C2);
7715 case ISD::AVGFLOORU:
7716 return APIntOps::avgFloorU(C1, C2);
7717 case ISD::AVGCEILS:
7718 return APIntOps::avgCeilS(C1, C2);
7719 case ISD::AVGCEILU:
7720 return APIntOps::avgCeilU(C1, C2);
7721 case ISD::AVGSUBFLOORS:
7722 return APIntOps::avgSubFloorS(C1, C2);
7723 case ISD::AVGSUBFLOORU:
7724 return APIntOps::avgSubFloorU(C1, C2);
7725 case ISD::ABDS:
7726 return APIntOps::abds(C1, C2);
7727 case ISD::ABDU:
7728 return APIntOps::abdu(C1, C2);
7729 case ISD::MULHS:
7730 return APIntOps::mulhs(C1, C2);
7731 case ISD::MULHU:
7732 return APIntOps::mulhu(C1, C2);
7733 case ISD::CLMUL:
7734 return APIntOps::clmul(C1, C2);
7735 case ISD::CLMULR:
7736 return APIntOps::clmulr(C1, C2);
7737 case ISD::CLMULH:
7738 return APIntOps::clmulh(C1, C2);
7739 case ISD::PEXT:
7740 return APIntOps::pext(C1, C2);
7741 case ISD::PDEP:
7742 return APIntOps::pdep(C1, C2);
7743 }
7744 return std::nullopt;
7745}
7746// Handle constant folding with UNDEF.
7747// TODO: Handle more cases.
7748static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7749 bool IsUndef1, const APInt &C2,
7750 bool IsUndef2) {
7751 if (!(IsUndef1 || IsUndef2))
7752 return FoldValue(Opcode, C1, C2);
7753
7754 // Fold and(x, undef) -> 0
7755 // Fold mul(x, undef) -> 0
7756 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7757 return APInt::getZero(C1.getBitWidth());
7758
7759 return std::nullopt;
7760}
7761
7763 const GlobalAddressSDNode *GA,
7764 const SDNode *N2) {
7765 if (GA->getOpcode() != ISD::GlobalAddress)
7766 return SDValue();
7767 if (!TLI->isOffsetFoldingLegal(GA))
7768 return SDValue();
7769 auto *C2 = dyn_cast<ConstantSDNode>(N2);
7770 if (!C2)
7771 return SDValue();
7772 int64_t Offset = C2->getSExtValue();
7773 switch (Opcode) {
7774 case ISD::ADD:
7775 case ISD::PTRADD:
7776 break;
7777 case ISD::SUB: Offset = -uint64_t(Offset); break;
7778 default: return SDValue();
7779 }
7780 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
7781 GA->getOffset() + uint64_t(Offset));
7782}
7783
7785 switch (Opcode) {
7786 case ISD::SDIV:
7787 case ISD::UDIV:
7788 case ISD::SREM:
7789 case ISD::UREM: {
7790 // If a divisor is zero/undef or any element of a divisor vector is
7791 // zero/undef, the whole op is undef.
7792 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7793 SDValue Divisor = Ops[1];
7794 if (Divisor.isUndef() || isNullConstant(Divisor))
7795 return true;
7796
7797 return ISD::isBuildVectorOfConstantSDNodes(Divisor) &&
7798 llvm::any_of(Divisor->op_values(), [](SDValue V) {
7799 return V.isUndef() || isNullConstant(V);
7800 });
7801 // TODO: Handle signed overflow.
7802 }
7803 // TODO: Handle oversized shifts.
7804 default:
7805 return false;
7806 }
7807}
7808
7811 SDNodeFlags Flags) {
7812 // If the opcode is a target-specific ISD node, there's nothing we can
7813 // do here and the operand rules may not line up with the below, so
7814 // bail early.
7815 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7816 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7817 // foldCONCAT_VECTORS in getNode before this is called.
7818 // VECTOR_SHUFFLE_VAR is not elementwise.
7819 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS ||
7820 Opcode == ISD::VECTOR_SHUFFLE_VAR)
7821 return SDValue();
7822
7823 unsigned NumOps = Ops.size();
7824 if (NumOps == 0)
7825 return SDValue();
7826
7827 if (isUndef(Opcode, Ops))
7828 return getUNDEF(VT);
7829
7830 // Handle unary special cases.
7831 if (NumOps == 1) {
7832 SDValue N1 = Ops[0];
7833
7834 // Constant fold unary operations with an integer constant operand. Even
7835 // opaque constant will be folded, because the folding of unary operations
7836 // doesn't create new constants with different values. Nevertheless, the
7837 // opaque flag is preserved during folding to prevent future folding with
7838 // other constants.
7839 if (auto *C = dyn_cast<ConstantSDNode>(N1)) {
7840 const APInt &Val = C->getAPIntValue();
7841 switch (Opcode) {
7842 case ISD::SIGN_EXTEND:
7843 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
7844 C->isTargetOpcode(), C->isOpaque());
7845 case ISD::TRUNCATE:
7846 if (C->isOpaque())
7847 break;
7848 [[fallthrough]];
7849 case ISD::ZERO_EXTEND:
7850 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
7851 C->isTargetOpcode(), C->isOpaque());
7852 case ISD::ANY_EXTEND:
7853 // Some targets like RISCV prefer to sign extend some types.
7854 if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))
7855 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
7856 C->isTargetOpcode(), C->isOpaque());
7857 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
7858 C->isTargetOpcode(), C->isOpaque());
7859 case ISD::ABS:
7860 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
7861 C->isOpaque());
7863 if (Val.isMinSignedValue())
7864 return getPOISON(VT);
7865 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
7866 C->isOpaque());
7867 case ISD::BITREVERSE:
7868 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
7869 C->isOpaque());
7870 case ISD::BSWAP:
7871 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
7872 C->isOpaque());
7873 case ISD::CTPOP:
7874 return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),
7875 C->isOpaque());
7876 case ISD::PARITY:
7877 return getConstant(Val.popcount() & 1, DL, VT, C->isTargetOpcode(),
7878 C->isOpaque());
7879 case ISD::CTLZ:
7881 return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),
7882 C->isOpaque());
7883 case ISD::CTTZ:
7885 return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
7886 C->isOpaque());
7887 case ISD::CTLS:
7888 // CTLS returns the number of extra sign bits so subtract one.
7889 return getConstant(Val.getNumSignBits() - 1, DL, VT,
7890 C->isTargetOpcode(), C->isOpaque());
7891 case ISD::UINT_TO_FP:
7892 case ISD::SINT_TO_FP: {
7894 (void)FPV.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,
7896 return getConstantFP(FPV, DL, VT);
7897 }
7898 case ISD::FP16_TO_FP:
7899 case ISD::BF16_TO_FP: {
7900 bool Ignored;
7901 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7902 : APFloat::BFloat(),
7903 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
7904
7905 // This can return overflow, underflow, or inexact; we don't care.
7906 // FIXME need to be more flexible about rounding mode.
7908 &Ignored);
7909 return getConstantFP(FPV, DL, VT);
7910 }
7911 case ISD::STEP_VECTOR:
7912 if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))
7913 return V;
7914 break;
7915 case ISD::BITCAST:
7916 if (C->isOpaque())
7917 break;
7918 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
7919 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7920 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
7921 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7922 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
7923 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7924 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
7925 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
7926 break;
7927 }
7928 }
7929
7930 // Constant fold unary operations with a floating point constant operand.
7931 if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {
7932 APFloat V = C->getValueAPF(); // make copy
7933 switch (Opcode) {
7934 case ISD::FNEG:
7935 V.changeSign();
7936 return getConstantFP(V, DL, VT);
7937 case ISD::FABS:
7938 V.clearSign();
7939 return getConstantFP(V, DL, VT);
7940 case ISD::FCEIL: {
7941 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
7943 return getConstantFP(V, DL, VT);
7944 return SDValue();
7945 }
7946 case ISD::FTRUNC: {
7947 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
7949 return getConstantFP(V, DL, VT);
7950 return SDValue();
7951 }
7952 case ISD::FFLOOR: {
7953 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
7955 return getConstantFP(V, DL, VT);
7956 return SDValue();
7957 }
7958 case ISD::FP_EXTEND: {
7959 bool ignored;
7960 // This can return overflow, underflow, or inexact; we don't care.
7961 // FIXME need to be more flexible about rounding mode.
7962 (void)V.convert(VT.getFltSemantics(), APFloat::rmNearestTiesToEven,
7963 &ignored);
7964 return getConstantFP(V, DL, VT);
7965 }
7966 case ISD::FP_TO_SINT:
7967 case ISD::FP_TO_UINT: {
7968 bool ignored;
7969 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7970 // FIXME need to be more flexible about rounding mode.
7972 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
7973 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7974 break;
7975 return getConstant(IntVal, DL, VT);
7976 }
7977 case ISD::FP_TO_FP16:
7978 case ISD::FP_TO_BF16: {
7979 bool Ignored;
7980 // This can return overflow, underflow, or inexact; we don't care.
7981 // FIXME need to be more flexible about rounding mode.
7982 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7983 : APFloat::BFloat(),
7985 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
7986 }
7987 case ISD::BITCAST:
7988 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
7989 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7990 VT);
7991 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
7992 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7993 VT);
7994 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
7995 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7996 VT);
7997 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
7998 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
7999 break;
8000 }
8001 }
8002
8003 // Early-out if we failed to constant fold a bitcast.
8004 if (Opcode == ISD::BITCAST)
8005 return SDValue();
8006
8007 // Constant fold integer vector reductions with constant BUILD_VECTORs.
8008 if ((Opcode == ISD::VECREDUCE_ADD || Opcode == ISD::VECREDUCE_SMAX ||
8009 Opcode == ISD::VECREDUCE_SMIN || Opcode == ISD::VECREDUCE_UMAX ||
8010 Opcode == ISD::VECREDUCE_UMIN || Opcode == ISD::VECREDUCE_MUL ||
8011 Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_XOR ||
8012 Opcode == ISD::VECREDUCE_AND) &&
8014 unsigned EltBits = N1.getValueType().getScalarSizeInBits();
8015 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
8016 APInt Acc = getIntegerIdentity(BaseOpcode, EltBits);
8017 for (SDValue Elt : N1->op_values()) {
8018 if (Elt.getOpcode() == ISD::POISON)
8019 return getPOISON(VT);
8020 if (Elt.isUndef() || cast<ConstantSDNode>(Elt)->isOpaque())
8021 return SDValue();
8022 APInt Value = cast<ConstantSDNode>(Elt)->getAPIntValue().trunc(EltBits);
8023 std::optional<APInt> Folded = FoldValue(BaseOpcode, Acc, Value);
8024 assert(Folded &&
8025 "Expected vector reduction base opcode to be foldable");
8026 Acc = *Folded;
8027 }
8028 EVT EltVT = N1.getValueType().getScalarType();
8029 return getAnyExtOrTrunc(getConstant(Acc, DL, EltVT), DL, VT);
8030 }
8031 }
8032
8033 // Handle binops special cases.
8034 if (NumOps == 2) {
8035 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
8036 return CFP;
8037
8038 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
8039 if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
8040 if (C1->isOpaque() || C2->isOpaque())
8041 return SDValue();
8042
8043 std::optional<APInt> FoldAttempt =
8044 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
8045 if (!FoldAttempt)
8046 return SDValue();
8047
8048 SDValue Folded = getConstant(*FoldAttempt, DL, VT);
8049 assert((!Folded || !VT.isVector()) &&
8050 "Can't fold vectors ops with scalar operands");
8051 return Folded;
8052 }
8053 }
8054
8055 // fold (add Sym, c) -> Sym+c
8057 return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());
8058 if (TLI->isCommutativeBinOp(Opcode))
8060 return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());
8061
8062 // fold (sext_in_reg c1) -> c2
8063 if (Opcode == ISD::SIGN_EXTEND_INREG) {
8064 EVT EVT = cast<VTSDNode>(Ops[1])->getVT();
8065
8066 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
8067 unsigned FromBits = EVT.getScalarSizeInBits();
8068 Val <<= Val.getBitWidth() - FromBits;
8069 Val.ashrInPlace(Val.getBitWidth() - FromBits);
8070 return getConstant(Val, DL, ConstantVT);
8071 };
8072
8073 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
8074 const APInt &Val = C1->getAPIntValue();
8075 return SignExtendInReg(Val, VT);
8076 }
8077
8079 SmallVector<SDValue, 8> ScalarOps;
8080 llvm::EVT OpVT = Ops[0].getOperand(0).getValueType();
8081 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
8082 SDValue Op = Ops[0].getOperand(I);
8083 if (Op.isUndef()) {
8084 ScalarOps.push_back(getUNDEF(OpVT));
8085 continue;
8086 }
8087 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
8088 ScalarOps.push_back(SignExtendInReg(Val, OpVT));
8089 }
8090 return getBuildVector(VT, DL, ScalarOps);
8091 }
8092
8093 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
8094 isa<ConstantSDNode>(Ops[0].getOperand(0)))
8095 return getNode(ISD::SPLAT_VECTOR, DL, VT,
8096 SignExtendInReg(Ops[0].getConstantOperandAPInt(0),
8097 Ops[0].getOperand(0).getValueType()));
8098 }
8099 }
8100
8101 // Handle fshl/fshr special cases.
8102 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
8103 auto *C1 = dyn_cast<ConstantSDNode>(Ops[0]);
8104 auto *C2 = dyn_cast<ConstantSDNode>(Ops[1]);
8105 auto *C3 = dyn_cast<ConstantSDNode>(Ops[2]);
8106
8107 if (C1 && C2 && C3) {
8108 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
8109 return SDValue();
8110 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
8111 &V3 = C3->getAPIntValue();
8112
8113 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(V1, V2, V3)
8114 : APIntOps::fshr(V1, V2, V3);
8115 return getConstant(FoldedVal, DL, VT);
8116 }
8117 }
8118
8119 // Handle fma/fmad special cases.
8120 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8121 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8122 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8123 Ops[2].getValueType() == VT && "FMA types must match!");
8127 if (C1 && C2 && C3) {
8128 APFloat V1 = C1->getValueAPF();
8129 const APFloat &V2 = C2->getValueAPF();
8130 const APFloat &V3 = C3->getValueAPF();
8131 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8132 V1.multiply(V2, APFloat::rmNearestTiesToEven);
8134 } else
8135 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
8136 return getConstantFP(V1, DL, VT);
8137 }
8138 }
8139
8140 // This is for vector folding only from here on.
8141 if (!VT.isVector())
8142 return SDValue();
8143
8144 // Constant fold integer partial reductions with constant BUILD_VECTOR
8145 // operands. The reduction order is deliberately unspecified. Use the same
8146 // subvector layout as TargetLowering::expandPartialReduceMLA(), where input
8147 // lane I contributes to accumulator lane I % NumAccElts.
8148 if (Opcode == ISD::PARTIAL_REDUCE_SMLA ||
8149 Opcode == ISD::PARTIAL_REDUCE_UMLA ||
8150 Opcode == ISD::PARTIAL_REDUCE_SUMLA) {
8151 // These nodes have no scalar form, so unsupported cases must not fall
8152 // through to generic per-lane vector folding.
8153 if (!llvm::all_of(Ops, [](SDValue Op) {
8155 }))
8156 return SDValue();
8157
8158 unsigned AccEltBits = VT.getScalarSizeInBits();
8159 unsigned InputEltBits = Ops[1].getScalarValueSizeInBits();
8160 unsigned NumAccElts = VT.getVectorNumElements();
8161 unsigned NumInputElts = Ops[1].getValueType().getVectorNumElements();
8162 SmallVector<APInt, 8> Results(NumAccElts, APInt::getZero(AccEltBits));
8163 BitVector PoisonElts(NumAccElts);
8164
8165 for (unsigned I = 0; I != NumAccElts; ++I) {
8166 SDValue Elt = Ops[0].getOperand(I);
8167 if (Elt.getOpcode() == ISD::POISON) {
8168 PoisonElts.set(I);
8169 continue;
8170 }
8171 auto *C = dyn_cast<ConstantSDNode>(Elt);
8172 if (!C || C->isOpaque())
8173 return SDValue();
8174 Results[I] = C->getAPIntValue().trunc(AccEltBits);
8175 }
8176
8177 bool IsLHSSigned = Opcode != ISD::PARTIAL_REDUCE_UMLA;
8178 bool IsRHSSigned = Opcode == ISD::PARTIAL_REDUCE_SMLA;
8179 for (unsigned I = 0; I != NumInputElts; ++I) {
8180 const unsigned AccIdx = I % NumAccElts;
8181 SDValue LHSElt = Ops[1].getOperand(I);
8182 SDValue RHSElt = Ops[2].getOperand(I);
8183 if (LHSElt.getOpcode() == ISD::POISON ||
8184 RHSElt.getOpcode() == ISD::POISON) {
8185 PoisonElts.set(AccIdx);
8186 continue;
8187 }
8188
8189 auto *LHS = dyn_cast<ConstantSDNode>(LHSElt);
8190 auto *RHS = dyn_cast<ConstantSDNode>(RHSElt);
8191 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
8192 return SDValue();
8193
8194 APInt LHSVal = LHS->getAPIntValue().trunc(InputEltBits);
8195 APInt RHSVal = RHS->getAPIntValue().trunc(InputEltBits);
8196 LHSVal = IsLHSSigned ? LHSVal.sext(AccEltBits) : LHSVal.zext(AccEltBits);
8197 RHSVal = IsRHSSigned ? RHSVal.sext(AccEltBits) : RHSVal.zext(AccEltBits);
8198 Results[AccIdx] += LHSVal * RHSVal;
8199 }
8200
8201 // After type legalization the vector element type may not be a legal
8202 // scalar type (e.g. i16 on AArch64). Create the folded constants in the
8203 // promoted legal scalar type instead, matching the generic per-lane path
8204 // below. Bail out if legalization would narrow the type, since the lane
8205 // value would not fit.
8206 EVT AccEltVT = VT.getVectorElementType();
8207 EVT LegalSVT = AccEltVT;
8208 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8209 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
8210 if (LegalSVT.bitsLT(AccEltVT))
8211 return SDValue();
8212 }
8213
8214 SmallVector<SDValue, 8> ResultOps;
8215 for (unsigned I = 0; I != NumAccElts; ++I)
8216 ResultOps.push_back(
8217 PoisonElts[I] ? getPOISON(LegalSVT)
8218 : getConstant(Results[I].sext(LegalSVT.getSizeInBits()),
8219 DL, LegalSVT));
8220 return getBuildVector(VT, DL, ResultOps);
8221 }
8222
8223 ElementCount NumElts = VT.getVectorElementCount();
8224
8225 // See if we can fold through any bitcasted integer ops.
8226 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
8227 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8228 (Ops[0].getOpcode() == ISD::BITCAST ||
8229 Ops[1].getOpcode() == ISD::BITCAST)) {
8232 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
8233 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
8234 if (BV1 && BV2 && N1.getValueType().isInteger() &&
8235 N2.getValueType().isInteger()) {
8236 bool IsLE = getDataLayout().isLittleEndian();
8237 unsigned EltBits = VT.getScalarSizeInBits();
8238 SmallVector<APInt> RawBits1, RawBits2;
8239 BitVector UndefElts1, UndefElts2;
8240 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
8241 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
8242 SmallVector<APInt> RawBits;
8243 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
8244 std::optional<APInt> Fold = FoldValueWithUndef(
8245 Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);
8246 if (!Fold)
8247 break;
8248 RawBits.push_back(*Fold);
8249 }
8250 if (RawBits.size() == NumElts.getFixedValue()) {
8251 // We have constant folded, but we might need to cast this again back
8252 // to the original (possibly legalized) type.
8253 EVT BVVT, BVEltVT;
8254 if (N1.getValueType() == VT) {
8255 BVVT = N1.getValueType();
8256 BVEltVT = BV1->getOperand(0).getValueType();
8257 } else {
8258 BVVT = N2.getValueType();
8259 BVEltVT = BV2->getOperand(0).getValueType();
8260 }
8261 unsigned BVEltBits = BVEltVT.getSizeInBits();
8262 SmallVector<APInt> DstBits;
8263 BitVector DstUndefs;
8265 DstBits, RawBits, DstUndefs,
8266 BitVector(RawBits.size(), false));
8267 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
8268 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
8269 if (DstUndefs[I])
8270 continue;
8271 Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);
8272 }
8273 return getBitcast(VT, getBuildVector(BVVT, DL, Ops));
8274 }
8275 }
8276 }
8277 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
8278 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(N1) &&
8279 isa<ConstantSDNode>(N2)) {
8280 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, N1.getValueType(),
8281 {N1, N2}, Flags))
8282 return getBitcast(VT, Res);
8283 }
8284 }
8285
8286 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
8287 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
8288 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
8289 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
8290 APInt RHSVal;
8291 if (ISD::isConstantSplatVector(Ops[1], RHSVal)) {
8292 APInt NewStep = Opcode == ISD::MUL
8293 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
8294 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
8295 return getStepVector(DL, VT, NewStep);
8296 }
8297 }
8298
8299 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
8300 return !Op.getValueType().isVector() ||
8301 Op.getValueType().getVectorElementCount() == NumElts;
8302 };
8303
8304 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
8305 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
8306 Op.getOpcode() == ISD::BUILD_VECTOR ||
8307 Op.getOpcode() == ISD::SPLAT_VECTOR;
8308 };
8309
8310 // All operands must be vector types with the same number of elements as
8311 // the result type and must be either UNDEF or a build/splat vector
8312 // or UNDEF scalars.
8313 if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
8314 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
8315 return SDValue();
8316
8317 // If we are comparing vectors, then the result needs to be a i1 boolean that
8318 // is then extended back to the legal result type depending on how booleans
8319 // are represented.
8320 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
8321 ISD::NodeType ExtendCode =
8322 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8323 ? TargetLowering::getExtendForContent(TLI->getBooleanContents(VT))
8325
8326 // Find legal integer scalar type for constant promotion and
8327 // ensure that its scalar size is at least as large as source.
8328 EVT LegalSVT = VT.getScalarType();
8329 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8330 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
8331 if (LegalSVT.bitsLT(VT.getScalarType()))
8332 return SDValue();
8333 }
8334
8335 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8336 // only have one operand to check. For fixed-length vector types we may have
8337 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8338 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8339
8340 // Constant fold each scalar lane separately.
8341 SmallVector<SDValue, 4> ScalarResults;
8342 for (unsigned I = 0; I != NumVectorElts; I++) {
8343 SmallVector<SDValue, 4> ScalarOps;
8344 for (SDValue Op : Ops) {
8345 EVT InSVT = Op.getValueType().getScalarType();
8346 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8347 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8348 if (Op.isUndef())
8349 ScalarOps.push_back(getUNDEF(InSVT));
8350 else
8351 ScalarOps.push_back(Op);
8352 continue;
8353 }
8354
8355 SDValue ScalarOp =
8356 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8357 EVT ScalarVT = ScalarOp.getValueType();
8358
8359 // Build vector (integer) scalar operands may need implicit
8360 // truncation - do this before constant folding.
8361 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
8362 // Don't create illegally-typed nodes unless they're constants or undef
8363 // - if we fail to constant fold we can't guarantee the (dead) nodes
8364 // we're creating will be cleaned up before being visited for
8365 // legalization.
8366 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8367 !isa<ConstantSDNode>(ScalarOp) &&
8368 TLI->getTypeAction(*getContext(), InSVT) !=
8370 return SDValue();
8371 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
8372 }
8373
8374 ScalarOps.push_back(ScalarOp);
8375 }
8376
8377 // Constant fold the scalar operands.
8378 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
8379
8380 // Scalar folding only succeeded if the result is a constant or UNDEF.
8381 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8382 ScalarResult.getOpcode() != ISD::ConstantFP)
8383 return SDValue();
8384
8385 // Legalize the (integer) scalar constant if necessary. We only do
8386 // this once we know the folding succeeded, since otherwise we would
8387 // get a node with illegal type which has a user.
8388 if (LegalSVT != SVT)
8389 ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);
8390
8391 ScalarResults.push_back(ScalarResult);
8392 }
8393
8394 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
8395 : getBuildVector(VT, DL, ScalarResults);
8396 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
8397 return V;
8398}
8399
8401 SDVTList VTList,
8403 SDNodeFlags Flags) {
8404 if (VTList.NumVTs == 1)
8405 return FoldConstantArithmetic(Opcode, DL, VTList.VTs[0], Ops, Flags);
8406
8407 switch (Opcode) {
8408 case ISD::SADDO:
8409 case ISD::UADDO:
8410 case ISD::SSUBO:
8411 case ISD::USUBO:
8412 case ISD::SMULO:
8413 case ISD::UMULO: {
8414 // TODO: Support non-splat vector constants.
8417 if (!C1 || !C2 || C1->isOpaque() || C2->isOpaque())
8418 return SDValue();
8419
8420 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue();
8421 bool Overflow;
8422 APInt Result;
8423 switch (Opcode) {
8424 default:
8425 llvm_unreachable("Unexpected overflow opcode");
8426 case ISD::SADDO:
8427 Result = V1.sadd_ov(V2, Overflow);
8428 break;
8429 case ISD::UADDO:
8430 Result = V1.uadd_ov(V2, Overflow);
8431 break;
8432 case ISD::SSUBO:
8433 Result = V1.ssub_ov(V2, Overflow);
8434 break;
8435 case ISD::USUBO:
8436 Result = V1.usub_ov(V2, Overflow);
8437 break;
8438 case ISD::SMULO:
8439 Result = V1.smul_ov(V2, Overflow);
8440 break;
8441 case ISD::UMULO:
8442 Result = V1.umul_ov(V2, Overflow);
8443 break;
8444 }
8445
8446 EVT VT = VTList.VTs[0];
8447 return getNode(ISD::MERGE_VALUES, DL, VTList,
8448 {getConstant(Result, DL, VT),
8449 getBoolConstant(Overflow, DL, VTList.VTs[1], VT)},
8450 Flags);
8451 }
8452 case ISD::SMUL_LOHI:
8453 case ISD::UMUL_LOHI:
8454 // TODO: Support non-splat vector constants.
8457 if (!C1 || !C2 || C1->isOpaque() || C2->isOpaque())
8458 return SDValue();
8459
8460 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue();
8461 APInt Lo = V1 * V2;
8462 APInt Hi = (Opcode == ISD::SMUL_LOHI) ? APIntOps::mulhs(V1, V2)
8463 : APIntOps::mulhu(V1, V2);
8464 return getNode(ISD::MERGE_VALUES, DL, VTList,
8465 {getConstant(Lo, DL, VTList.VTs[0]),
8466 getConstant(Hi, DL, VTList.VTs[1])},
8467 Flags);
8468 }
8469 return SDValue();
8470}
8471
8474 // TODO: Add support for unary/ternary fp opcodes.
8475 if (Ops.size() != 2)
8476 return SDValue();
8477
8478 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8479 // should. That will require dealing with a potentially non-default
8480 // rounding mode, checking the "opStatus" return value from the APFloat
8481 // math calculations, and possibly other variations.
8482 SDValue N1 = Ops[0];
8483 SDValue N2 = Ops[1];
8484 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
8485 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
8486 if (N1CFP && N2CFP) {
8487 APFloat C1 = N1CFP->getValueAPF(); // make copy
8488 const APFloat &C2 = N2CFP->getValueAPF();
8489 switch (Opcode) {
8490 case ISD::FADD:
8492 return getConstantFP(C1, DL, VT);
8493 case ISD::FSUB:
8494 C1.subtract(C2, APFloat::rmNearestTiesToEven);
8495 return getConstantFP(C1, DL, VT);
8496 case ISD::FMUL:
8497 C1.multiply(C2, APFloat::rmNearestTiesToEven);
8498 return getConstantFP(C1, DL, VT);
8499 case ISD::FDIV:
8500 C1.divide(C2, APFloat::rmNearestTiesToEven);
8501 return getConstantFP(C1, DL, VT);
8502 case ISD::FREM:
8503 C1.mod(C2);
8504 return getConstantFP(C1, DL, VT);
8505 case ISD::FCOPYSIGN:
8506 C1.copySign(C2);
8507 return getConstantFP(C1, DL, VT);
8508 case ISD::FMINNUM:
8509 return getConstantFP(minnum(C1, C2), DL, VT);
8510 case ISD::FMAXNUM:
8511 return getConstantFP(maxnum(C1, C2), DL, VT);
8512 case ISD::FMINIMUM:
8513 return getConstantFP(minimum(C1, C2), DL, VT);
8514 case ISD::FMAXIMUM:
8515 return getConstantFP(maximum(C1, C2), DL, VT);
8516 case ISD::FMINIMUMNUM:
8517 return getConstantFP(minimumnum(C1, C2), DL, VT);
8518 case ISD::FMAXIMUMNUM:
8519 return getConstantFP(maximumnum(C1, C2), DL, VT);
8520 default: break;
8521 }
8522 }
8523 if (N1CFP && Opcode == ISD::FP_ROUND) {
8524 APFloat C1 = N1CFP->getValueAPF(); // make copy
8525 bool Unused;
8526 // This can return overflow, underflow, or inexact; we don't care.
8527 // FIXME need to be more flexible about rounding mode.
8529 &Unused);
8530 return getConstantFP(C1, DL, VT);
8531 }
8532
8533 switch (Opcode) {
8534 case ISD::FSUB:
8535 // -0.0 - undef --> undef (consistent with "fneg undef")
8536 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
8537 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8538 return getUNDEF(VT);
8539 [[fallthrough]];
8540
8541 case ISD::FADD:
8542 case ISD::FMUL:
8543 case ISD::FDIV:
8544 case ISD::FREM:
8545 // If both operands are undef, the result is undef. If 1 operand is undef,
8546 // the result is NaN. This should match the behavior of the IR optimizer.
8547 if (N1.isUndef() && N2.isUndef())
8548 return getUNDEF(VT);
8549 if (N1.isUndef() || N2.isUndef())
8551 }
8552 return SDValue();
8553}
8554
8556 const SDLoc &DL, EVT DstEltVT) {
8557 EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
8558
8559 // If this is already the right type, we're done.
8560 if (SrcEltVT == DstEltVT)
8561 return SDValue(BV, 0);
8562
8563 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8564 unsigned DstBitSize = DstEltVT.getSizeInBits();
8565
8566 // If this is a conversion of N elements of one type to N elements of another
8567 // type, convert each element. This handles FP<->INT cases.
8568 if (SrcBitSize == DstBitSize) {
8570 for (SDValue Op : BV->op_values()) {
8571 // If the vector element type is not legal, the BUILD_VECTOR operands
8572 // are promoted and implicitly truncated. Make that explicit here.
8573 if (Op.getValueType() != SrcEltVT)
8574 Op = getNode(ISD::TRUNCATE, DL, SrcEltVT, Op);
8575 Ops.push_back(getBitcast(DstEltVT, Op));
8576 }
8577 EVT VT = EVT::getVectorVT(*getContext(), DstEltVT,
8579 return getBuildVector(VT, DL, Ops);
8580 }
8581
8582 // Otherwise, we're growing or shrinking the elements. To avoid having to
8583 // handle annoying details of growing/shrinking FP values, we convert them to
8584 // int first.
8585 if (SrcEltVT.isFloatingPoint()) {
8586 // Convert the input float vector to a int vector where the elements are the
8587 // same sizes.
8588 EVT IntEltVT = EVT::getIntegerVT(*getContext(), SrcEltVT.getSizeInBits());
8589 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))
8591 DstEltVT);
8592 return SDValue();
8593 }
8594
8595 // Now we know the input is an integer vector. If the output is a FP type,
8596 // convert to integer first, then to FP of the right size.
8597 if (DstEltVT.isFloatingPoint()) {
8598 EVT IntEltVT = EVT::getIntegerVT(*getContext(), DstEltVT.getSizeInBits());
8599 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))
8601 DstEltVT);
8602 return SDValue();
8603 }
8604
8605 // Okay, we know the src/dst types are both integers of differing types.
8606 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8607
8608 // Extract the constant raw bit data.
8609 BitVector UndefElements;
8610 SmallVector<APInt> RawBits;
8611 bool IsLE = getDataLayout().isLittleEndian();
8612 if (!BV->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
8613 return SDValue();
8614
8616 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8617 if (UndefElements[I])
8618 Ops.push_back(getUNDEF(DstEltVT));
8619 else
8620 Ops.push_back(getConstant(RawBits[I], DL, DstEltVT));
8621 }
8622
8623 EVT VT = EVT::getVectorVT(*getContext(), DstEltVT, Ops.size());
8624 return getBuildVector(VT, DL, Ops);
8625}
8626
8628 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8629
8630 // There's no need to assert on a byte-aligned pointer. All pointers are at
8631 // least byte aligned.
8632 if (A == Align(1))
8633 return Val;
8634
8635 SDVTList VTs = getVTList(Val.getValueType());
8636 SDValue Ops[] = {Val};
8637 SDNodeKey ID(ISD::AssertAlign, VTs, Ops);
8638 ID.AddInteger(A.value());
8639
8640 FoldingSetInsertToken InsertToken;
8641 if (SDNode *E = lookupNode(ID, DL, InsertToken))
8642 return SDValue(E, 0);
8643
8644 auto *N =
8645 newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, A);
8646 createOperands(N, {Val});
8647
8648 CSEMap.insert(N, InsertToken);
8649 InsertNode(N);
8650
8651 SDValue V(N, 0);
8652 NewSDValueDbgMsg(V, "Creating new node: ", this);
8653 return V;
8654}
8655
8656SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8657 SDValue N1, SDValue N2) {
8658 SDNodeFlags Flags;
8659 if (Inserter)
8660 Flags = Inserter->getFlags();
8661 return getNode(Opcode, DL, VT, N1, N2, Flags);
8662}
8663
8665 SDValue &N2) const {
8666 if (!TLI->isCommutativeBinOp(Opcode))
8667 return;
8668
8669 // Canonicalize:
8670 // binop(const, nonconst) -> binop(nonconst, const)
8673 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N1);
8674 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N2);
8675 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8676 std::swap(N1, N2);
8677
8678 // Canonicalize:
8679 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8680 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8682 std::swap(N1, N2);
8683}
8684
8685SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8686 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8688 N2.getOpcode() != ISD::DELETED_NODE &&
8689 "Operand is DELETED_NODE!");
8690
8691 canonicalizeCommutativeBinop(Opcode, N1, N2);
8692
8693 auto *N1C = dyn_cast<ConstantSDNode>(N1);
8694 auto *N2C = dyn_cast<ConstantSDNode>(N2);
8695
8696 // Don't allow undefs in vector splats - we might be returning N2 when folding
8697 // to zero etc.
8698 ConstantSDNode *N2CV =
8699 isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8700
8701 switch (Opcode) {
8702 default: break;
8703 case ISD::TokenFactor:
8704 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8705 N2.getValueType() == MVT::Other && "Invalid token factor!");
8706 // Fold trivial token factors.
8707 if (N1.getOpcode() == ISD::EntryToken) return N2;
8708 if (N2.getOpcode() == ISD::EntryToken) return N1;
8709 if (N1 == N2) return N1;
8710 break;
8711 case ISD::BUILD_VECTOR: {
8712 // Attempt to simplify BUILD_VECTOR.
8713 SDValue Ops[] = {N1, N2};
8714 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
8715 return V;
8716 break;
8717 }
8718 case ISD::CONCAT_VECTORS: {
8719 SDValue Ops[] = {N1, N2};
8720 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
8721 return V;
8722 break;
8723 }
8724 case ISD::AND:
8725 assert(VT.isInteger() && "This operator does not apply to FP types!");
8726 assert(N1.getValueType() == N2.getValueType() &&
8727 N1.getValueType() == VT && "Binary operator types must match!");
8728 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8729 // worth handling here.
8730 if (N2CV && N2CV->isZero())
8731 return N2;
8732 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8733 return N1;
8734 break;
8735 case ISD::OR:
8736 case ISD::XOR:
8737 case ISD::ADD:
8738 case ISD::PTRADD:
8739 case ISD::SUB:
8740 assert(VT.isInteger() && "This operator does not apply to FP types!");
8741 assert(N1.getValueType() == N2.getValueType() &&
8742 N1.getValueType() == VT && "Binary operator types must match!");
8743 // The equal operand types requirement is unnecessarily strong for PTRADD.
8744 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8745 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8746 // logic everywhere where PTRADDs may be folded or combined to properly
8747 // support them. If/when we introduce pointer types to the SDAG, we will
8748 // need to relax this constraint.
8749
8750 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8751 // it's worth handling here.
8752 if (N2CV && N2CV->isZero())
8753 return N1;
8754 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8755 VT.getScalarType() == MVT::i1)
8756 return getNode(ISD::XOR, DL, VT, N1, N2);
8757 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8758 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8759 N2.getOpcode() == ISD::VSCALE) {
8760 const APInt &C1 = N1->getConstantOperandAPInt(0);
8761 const APInt &C2 = N2->getConstantOperandAPInt(0);
8762 return getVScale(DL, VT, C1 + C2);
8763 }
8764 break;
8765 case ISD::MUL:
8766 assert(VT.isInteger() && "This operator does not apply to FP types!");
8767 assert(N1.getValueType() == N2.getValueType() &&
8768 N1.getValueType() == VT && "Binary operator types must match!");
8769 if (VT.getScalarType() == MVT::i1)
8770 return getNode(ISD::AND, DL, VT, N1, N2);
8771 if (N2CV && N2CV->isZero())
8772 return N2;
8773 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8774 const APInt &MulImm = N1->getConstantOperandAPInt(0);
8775 const APInt &N2CImm = N2C->getAPIntValue();
8776 return getVScale(DL, VT, MulImm * N2CImm);
8777 }
8778 break;
8779 case ISD::UDIV:
8780 case ISD::UREM:
8781 case ISD::MULHU:
8782 case ISD::MULHS:
8783 case ISD::SDIV:
8784 case ISD::SREM:
8785 case ISD::SADDSAT:
8786 case ISD::SSUBSAT:
8787 case ISD::UADDSAT:
8788 case ISD::USUBSAT:
8789 assert(VT.isInteger() && "This operator does not apply to FP types!");
8790 assert(N1.getValueType() == N2.getValueType() &&
8791 N1.getValueType() == VT && "Binary operator types must match!");
8792 if (VT.getScalarType() == MVT::i1) {
8793 // fold (add_sat x, y) -> (or x, y) for bool types.
8794 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8795 return getNode(ISD::OR, DL, VT, N1, N2);
8796 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8797 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8798 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
8799 // mulhs/mulhu on i1 are always 0: the high bit of the 2-bit product of
8800 // two booleans is never set. The i1 constant 1 is also -1, so folding
8801 // (mulhs x, 1) to (sra x, 0) in DAGCombiner would be wrong here.
8802 if (Opcode == ISD::MULHS || Opcode == ISD::MULHU)
8803 return getConstant(0, DL, VT);
8804 }
8805 break;
8806 case ISD::SCMP:
8807 case ISD::UCMP:
8808 assert(N1.getValueType() == N2.getValueType() &&
8809 "Types of operands of UCMP/SCMP must match");
8810 assert(N1.getValueType().isVector() == VT.isVector() &&
8811 "Operands and return type of must both be scalars or vectors");
8812 if (VT.isVector())
8815 "Result and operands must have the same number of elements");
8816 break;
8817 case ISD::AVGFLOORS:
8818 case ISD::AVGFLOORU:
8819 case ISD::AVGCEILS:
8820 case ISD::AVGCEILU:
8821 case ISD::AVGSUBFLOORS:
8822 case ISD::AVGSUBFLOORU:
8823 assert(VT.isInteger() && "This operator does not apply to FP types!");
8824 assert(N1.getValueType() == N2.getValueType() &&
8825 N1.getValueType() == VT && "Binary operator types must match!");
8826 break;
8827 case ISD::ABDS:
8828 case ISD::ABDU:
8829 assert(VT.isInteger() && "This operator does not apply to FP types!");
8830 assert(N1.getValueType() == N2.getValueType() &&
8831 N1.getValueType() == VT && "Binary operator types must match!");
8832 if (VT.getScalarType() == MVT::i1)
8833 return getNode(ISD::XOR, DL, VT, N1, N2);
8834 break;
8835 case ISD::SMIN:
8836 case ISD::UMAX:
8837 assert(VT.isInteger() && "This operator does not apply to FP types!");
8838 assert(N1.getValueType() == N2.getValueType() &&
8839 N1.getValueType() == VT && "Binary operator types must match!");
8840 if (VT.getScalarType() == MVT::i1)
8841 return getNode(ISD::OR, DL, VT, N1, N2);
8842 break;
8843 case ISD::SMAX:
8844 case ISD::UMIN:
8845 assert(VT.isInteger() && "This operator does not apply to FP types!");
8846 assert(N1.getValueType() == N2.getValueType() &&
8847 N1.getValueType() == VT && "Binary operator types must match!");
8848 if (VT.getScalarType() == MVT::i1)
8849 return getNode(ISD::AND, DL, VT, N1, N2);
8850 break;
8851 case ISD::FADD:
8852 case ISD::FSUB:
8853 case ISD::FMUL:
8854 case ISD::FDIV:
8855 case ISD::FREM:
8856 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8857 assert(N1.getValueType() == N2.getValueType() &&
8858 N1.getValueType() == VT && "Binary operator types must match!");
8859 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
8860 return V;
8861 break;
8862 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8863 assert(N1.getValueType() == VT &&
8866 "Invalid FCOPYSIGN!");
8867 break;
8868 case ISD::SHL:
8869 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8870 const APInt &MulImm = N1->getConstantOperandAPInt(0);
8871 const APInt &ShiftImm = N2C->getAPIntValue();
8872 return getVScale(DL, VT, MulImm << ShiftImm);
8873 }
8874 [[fallthrough]];
8875 case ISD::SRA:
8876 case ISD::SRL:
8877 if (SDValue V = simplifyShift(N1, N2))
8878 return V;
8879 [[fallthrough]];
8880 case ISD::ROTL:
8881 case ISD::ROTR:
8882 case ISD::SSHLSAT:
8883 case ISD::USHLSAT:
8884 assert(VT == N1.getValueType() &&
8885 "Shift operators return type must be the same as their first arg");
8886 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8887 "Shifts only work on integers");
8888 assert((!VT.isVector() || VT == N2.getValueType()) &&
8889 "Vector shift amounts must be in the same as their first arg");
8890 // Verify that the shift amount VT is big enough to hold valid shift
8891 // amounts. This catches things like trying to shift an i1024 value by an
8892 // i8, which is easy to fall into in generic code that uses
8893 // TLI.getShiftAmount().
8896 "Invalid use of small shift amount with oversized value!");
8897
8898 // Always fold shifts of i1 values so the code generator doesn't need to
8899 // handle them. Since we know the size of the shift has to be less than the
8900 // size of the value, the shift/rotate count is guaranteed to be zero.
8901 if (VT == MVT::i1)
8902 return N1;
8903 if (N2CV && N2CV->isZero())
8904 return N1;
8905 break;
8906 case ISD::FP_ROUND:
8908 VT.bitsLE(N1.getValueType()) && N2C &&
8909 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8910 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8911 if (N1.getValueType() == VT) return N1; // noop conversion.
8912 break;
8913 case ISD::IS_FPCLASS: {
8915 "IS_FPCLASS is used for a non-floating type");
8916 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8917 // is.fpclass(poison, mask) -> poison
8918 if (N1.getOpcode() == ISD::POISON)
8919 return getPOISON(VT);
8920 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8921 // If all tests are made, it doesn't matter what the value is.
8922 if ((Mask & fcAllFlags) == fcAllFlags)
8923 return getBoolConstant(true, DL, VT, N1.getValueType());
8924 if ((Mask & fcAllFlags) == 0)
8925 return getBoolConstant(false, DL, VT, N1.getValueType());
8926 break;
8927 }
8928 case ISD::AssertNoFPClass: {
8930 "AssertNoFPClass is used for a non-floating type");
8931 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8932 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8933 assert(llvm::to_underlying(NoFPClass) <=
8935 "FPClassTest value too large");
8936 (void)NoFPClass;
8937 break;
8938 }
8939 case ISD::AssertSext:
8940 case ISD::AssertZext: {
8941 EVT EVT = cast<VTSDNode>(N2)->getVT();
8942 assert(VT == N1.getValueType() && "Not an inreg extend!");
8943 assert(VT.isInteger() && EVT.isInteger() &&
8944 "Cannot *_EXTEND_INREG FP types");
8945 assert(!EVT.isVector() &&
8946 "AssertSExt/AssertZExt type should be the vector element type "
8947 "rather than the vector type!");
8948 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8949 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8950 break;
8951 }
8953 EVT EVT = cast<VTSDNode>(N2)->getVT();
8954 assert(VT == N1.getValueType() && "Not an inreg extend!");
8955 assert(VT.isInteger() && EVT.isInteger() &&
8956 "Cannot *_EXTEND_INREG FP types");
8957 assert(EVT.isVector() == VT.isVector() &&
8958 "SIGN_EXTEND_INREG type should be vector iff the operand "
8959 "type is vector!");
8960 assert((!EVT.isVector() ||
8962 "Vector element counts must match in SIGN_EXTEND_INREG");
8963 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8964 if (EVT == VT) return N1; // Not actually extending
8965 break;
8966 }
8968 case ISD::FP_TO_UINT_SAT: {
8969 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8970 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8971 assert(N1.getValueType().isVector() == VT.isVector() &&
8972 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8973 "vector!");
8974 assert((!VT.isVector() || VT.getVectorElementCount() ==
8976 "Vector element counts must match in FP_TO_*INT_SAT");
8977 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8978 "Type to saturate to must be a scalar.");
8979 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8980 "Not extending!");
8981 break;
8982 }
8985 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8986 element type of the vector.");
8987
8988 // Extract from an undefined value or using an undefined index is undefined.
8989 if (N1.isUndef() || N2.isUndef())
8990 return getUNDEF(VT);
8991
8992 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8993 // vectors. For scalable vectors we will provide appropriate support for
8994 // dealing with arbitrary indices.
8995 if (N2C && N1.getValueType().isFixedLengthVector() &&
8996 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
8997 return getPOISON(VT);
8998
8999 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
9000 // expanding copies of large vectors from registers. This only works for
9001 // fixed length vectors, since we need to know the exact number of
9002 // elements.
9003 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
9005 unsigned Factor = N1.getOperand(0).getValueType().getVectorNumElements();
9006 return getExtractVectorElt(DL, VT,
9007 N1.getOperand(N2C->getZExtValue() / Factor),
9008 N2C->getZExtValue() % Factor);
9009 }
9010
9011 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
9012 // lowering is expanding large vector constants.
9013 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
9014 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
9017 "BUILD_VECTOR used for scalable vectors");
9018 unsigned Index =
9019 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
9020 SDValue Elt = N1.getOperand(Index);
9021
9022 if (VT != Elt.getValueType())
9023 // If the vector element type is not legal, the BUILD_VECTOR operands
9024 // are promoted and implicitly truncated, and the result implicitly
9025 // extended. Make that explicit here.
9026 Elt = getAnyExtOrTrunc(Elt, DL, VT);
9027
9028 return Elt;
9029 }
9030
9031 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
9032 // operations are lowered to scalars.
9033 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
9034 // If the indices are the same, return the inserted element else
9035 // if the indices are known different, extract the element from
9036 // the original vector.
9037 SDValue N1Op2 = N1.getOperand(2);
9039
9040 if (N1Op2C && N2C) {
9041 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
9042 if (VT == N1.getOperand(1).getValueType())
9043 return N1.getOperand(1);
9044 if (VT.isFloatingPoint()) {
9046 return getFPExtendOrRound(N1.getOperand(1), DL, VT);
9047 }
9048 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
9049 }
9050 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
9051 }
9052 }
9053
9054 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
9055 // when vector types are scalarized and v1iX is legal.
9056 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
9057 // Here we are completely ignoring the extract element index (N2),
9058 // which is fine for fixed width vectors, since any index other than 0
9059 // is undefined anyway. However, this cannot be ignored for scalable
9060 // vectors - in theory we could support this, but we don't want to do this
9061 // without a profitability check.
9062 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9064 N1.getValueType().getVectorNumElements() == 1) {
9065 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
9066 N1.getOperand(1));
9067 }
9068 break;
9070 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
9071 assert(!N1.getValueType().isVector() && !VT.isVector() &&
9072 (N1.getValueType().isInteger() == VT.isInteger()) &&
9073 N1.getValueType() != VT &&
9074 "Wrong types for EXTRACT_ELEMENT!");
9075
9076 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
9077 // 64-bit integers into 32-bit parts. Instead of building the extract of
9078 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
9079 if (N1.getOpcode() == ISD::BUILD_PAIR)
9080 return N1.getOperand(N2C->getZExtValue());
9081
9082 // EXTRACT_ELEMENT of a constant int is also very common.
9083 if (N1C) {
9084 unsigned ElementSize = VT.getSizeInBits();
9085 unsigned Shift = ElementSize * N2C->getZExtValue();
9086 const APInt &Val = N1C->getAPIntValue();
9087 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
9088 }
9089 break;
9091 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9092 assert(VT == N1.getValueType() &&
9093 "VECTOR_SHUFFLE_VAR result and source types don't match.");
9094 assert(VT.isVector() && MaskVT.isVector() &&
9095 "VECTOR_SHUFFLE_VAR operands must be vectors.");
9097 "VECTOR_SHUFFLE_VAR mask must be a vector of integers.");
9099 "VECTOR_SHUFFLE_VAR mask has the wrong number of elements.");
9100 // The only in-range index is 0; any other index gives poison.
9102 return N1;
9103 break;
9104 }
9106 EVT N1VT = N1.getValueType();
9107 assert(VT.isVector() && N1VT.isVector() &&
9108 "Extract subvector VTs must be vectors!");
9110 "Extract subvector VTs must have the same element type!");
9111 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
9112 "Cannot extract a scalable vector from a fixed length vector!");
9113 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
9115 "Extract subvector must be from larger vector to smaller vector!");
9116 assert(N2C && "Extract subvector index must be a constant");
9117 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
9118 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
9119 N1VT.getVectorMinNumElements()) &&
9120 "Extract subvector overflow!");
9121 assert(N2C->getAPIntValue().getBitWidth() ==
9122 TLI->getVectorIdxWidth(getDataLayout()) &&
9123 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
9124 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
9125 "Extract index is not a multiple of the output vector length");
9126
9127 // Trivial extraction.
9128 if (VT == N1VT)
9129 return N1;
9130
9131 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
9132 if (N1.isUndef())
9133 return getUNDEF(VT);
9134
9135 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
9136 // the concat have the same type as the extract.
9137 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
9138 VT == N1.getOperand(0).getValueType()) {
9139 unsigned Factor = VT.getVectorMinNumElements();
9140 return N1.getOperand(N2C->getZExtValue() / Factor);
9141 }
9142
9143 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
9144 // during shuffle legalization.
9145 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
9146 VT == N1.getOperand(1).getValueType())
9147 return N1.getOperand(1);
9148 break;
9149 }
9150 }
9151
9152 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
9153 switch (Opcode) {
9154 case ISD::XOR:
9155 case ISD::ADD:
9156 case ISD::PTRADD:
9157 case ISD::SUB:
9159 case ISD::UDIV:
9160 case ISD::SDIV:
9161 case ISD::UREM:
9162 case ISD::SREM:
9163 case ISD::MUL:
9164 case ISD::AND:
9165 case ISD::SSUBSAT:
9166 case ISD::USUBSAT:
9167 case ISD::UMIN:
9168 case ISD::OR:
9169 case ISD::SADDSAT:
9170 case ISD::UADDSAT:
9171 case ISD::UMAX:
9172 case ISD::SMAX:
9173 case ISD::SMIN:
9174 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
9175 return N2.getOpcode() == ISD::POISON ? N2 : N1;
9176 }
9177 }
9178
9179 // Canonicalize an UNDEF to the RHS, even over a constant.
9180 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
9181 if (TLI->isCommutativeBinOp(Opcode)) {
9182 std::swap(N1, N2);
9183 } else {
9184 switch (Opcode) {
9185 case ISD::PTRADD:
9186 case ISD::SUB:
9187 // fold op(undef, non_undef_arg2) -> undef.
9188 return N1;
9190 case ISD::UDIV:
9191 case ISD::SDIV:
9192 case ISD::UREM:
9193 case ISD::SREM:
9194 case ISD::SSUBSAT:
9195 case ISD::USUBSAT:
9196 // fold op(undef, non_undef_arg2) -> 0.
9197 return getConstant(0, DL, VT);
9198 }
9199 }
9200 }
9201
9202 // Fold a bunch of operators when the RHS is undef.
9203 if (N2.getOpcode() == ISD::UNDEF) {
9204 switch (Opcode) {
9205 case ISD::XOR:
9206 if (N1.getOpcode() == ISD::UNDEF)
9207 // Handle undef ^ undef -> 0 special case. This is a common
9208 // idiom (misuse).
9209 return getConstant(0, DL, VT);
9210 [[fallthrough]];
9211 case ISD::ADD:
9212 case ISD::PTRADD:
9213 case ISD::SUB:
9214 // fold op(arg1, undef) -> undef.
9215 return N2;
9216 case ISD::UDIV:
9217 case ISD::SDIV:
9218 case ISD::UREM:
9219 case ISD::SREM:
9220 // fold op(arg1, undef) -> poison.
9221 return getPOISON(VT);
9222 case ISD::MUL:
9223 case ISD::AND:
9224 case ISD::SSUBSAT:
9225 case ISD::USUBSAT:
9226 case ISD::UMIN:
9227 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
9228 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(0, DL, VT);
9229 case ISD::OR:
9230 case ISD::SADDSAT:
9231 case ISD::UADDSAT:
9232 case ISD::UMAX:
9233 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
9234 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
9235 case ISD::SMAX:
9236 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
9237 return N1.getOpcode() == ISD::UNDEF
9238 ? N2
9239 : getConstant(
9241 VT);
9242 case ISD::SMIN:
9243 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
9244 return N1.getOpcode() == ISD::UNDEF
9245 ? N2
9246 : getConstant(
9248 VT);
9249 }
9250 }
9251
9252 // Perform trivial constant folding.
9253 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}, Flags))
9254 return SV;
9255
9256 // Memoize this node if possible.
9257 SDNode *N;
9258 SDVTList VTs = getVTList(VT);
9259 SDValue Ops[] = {N1, N2};
9260 if (VT != MVT::Glue) {
9261 SDNodeKey ID(Opcode, VTs, Ops);
9262 FoldingSetInsertToken InsertToken;
9263 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
9264 E->intersectFlagsWith(Flags);
9265 return SDValue(E, 0);
9266 }
9267
9268 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9269 N->setFlags(Flags);
9270 createOperands(N, Ops);
9271 CSEMap.insert(N, InsertToken);
9272 } else {
9273 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9274 createOperands(N, Ops);
9275 }
9276
9277 InsertNode(N);
9278 SDValue V = SDValue(N, 0);
9279 NewSDValueDbgMsg(V, "Creating new node: ", this);
9280 return V;
9281}
9282
9283SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9284 SDValue N1, SDValue N2, SDValue N3) {
9285 SDNodeFlags Flags;
9286 if (Inserter)
9287 Flags = Inserter->getFlags();
9288 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
9289}
9290
9291SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9292 SDValue N1, SDValue N2, SDValue N3,
9293 const SDNodeFlags Flags) {
9295 N2.getOpcode() != ISD::DELETED_NODE &&
9296 N3.getOpcode() != ISD::DELETED_NODE &&
9297 "Operand is DELETED_NODE!");
9298 // Perform various simplifications.
9299 switch (Opcode) {
9300 case ISD::BUILD_VECTOR: {
9301 // Attempt to simplify BUILD_VECTOR.
9302 SDValue Ops[] = {N1, N2, N3};
9303 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
9304 return V;
9305 break;
9306 }
9307 case ISD::CONCAT_VECTORS: {
9308 SDValue Ops[] = {N1, N2, N3};
9309 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
9310 return V;
9311 break;
9312 }
9313 case ISD::SETCC: {
9314 assert(VT.isInteger() && "SETCC result type must be an integer!");
9315 assert(N1.getValueType() == N2.getValueType() &&
9316 "SETCC operands must have the same type!");
9317 assert(VT.isVector() == N1.getValueType().isVector() &&
9318 "SETCC type should be vector iff the operand type is vector!");
9319 assert((!VT.isVector() || VT.getVectorElementCount() ==
9321 "SETCC vector element counts must match!");
9322 // Use FoldSetCC to simplify SETCC's.
9323 if (SDValue V =
9324 FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL, Flags))
9325 return V;
9326 break;
9327 }
9328 case ISD::SELECT:
9329 case ISD::VSELECT:
9330 if (SDValue V = simplifySelect(N1, N2, N3))
9331 return V;
9332 break;
9334 llvm_unreachable("should use getVectorShuffle constructor!");
9336 if (isNullConstant(N3))
9337 return N1;
9338 break;
9340 if (isNullConstant(N3))
9341 return N2;
9342 break;
9344 assert(VT.isVector() && VT == N1.getValueType() &&
9345 "INSERT_VECTOR_ELT vector type mismatch");
9347 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9348 assert((!VT.isFloatingPoint() ||
9349 VT.getVectorElementType() == N2.getValueType()) &&
9350 "INSERT_VECTOR_ELT fp scalar type mismatch");
9351 assert((!VT.isInteger() ||
9353 "INSERT_VECTOR_ELT int scalar size mismatch");
9354
9355 auto *N3C = dyn_cast<ConstantSDNode>(N3);
9356 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
9357 // for scalable vectors where we will generate appropriate code to
9358 // deal with out-of-bounds cases correctly.
9359 if (N3C && VT.isFixedLengthVector() &&
9360 N3C->getZExtValue() >= VT.getVectorNumElements())
9361 return getUNDEF(VT);
9362
9363 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
9364 if (N3.isUndef())
9365 return getUNDEF(VT);
9366
9367 // If inserting poison, just use the input vector.
9368 if (N2.getOpcode() == ISD::POISON)
9369 return N1;
9370
9371 // Inserting undef into undef/poison is still undef.
9372 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9373 return getUNDEF(VT);
9374
9375 // If the inserted element is an UNDEF, just use the input vector.
9376 // But not if skipping the insert could make the result more poisonous.
9377 if (N2.isUndef()) {
9378 if (N3C && VT.isFixedLengthVector()) {
9379 APInt EltMask =
9380 APInt::getOneBitSet(VT.getVectorNumElements(), N3C->getZExtValue());
9381 if (isGuaranteedNotToBePoison(N1, EltMask))
9382 return N1;
9383 } else if (isGuaranteedNotToBePoison(N1))
9384 return N1;
9385 }
9386 break;
9387 }
9388 case ISD::INSERT_SUBVECTOR: {
9389 // If inserting poison, just use the input vector,
9390 if (N2.getOpcode() == ISD::POISON)
9391 return N1;
9392
9393 // Inserting undef into undef/poison is still undef.
9394 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9395 return getUNDEF(VT);
9396
9397 EVT N2VT = N2.getValueType();
9398 assert(VT == N1.getValueType() &&
9399 "Dest and insert subvector source types must match!");
9400 assert(VT.isVector() && N2VT.isVector() &&
9401 "Insert subvector VTs must be vectors!");
9403 "Insert subvector VTs must have the same element type!");
9404 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
9405 "Cannot insert a scalable vector into a fixed length vector!");
9406 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9408 "Insert subvector must be from smaller vector to larger vector!");
9410 "Insert subvector index must be constant");
9411 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9412 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
9414 "Insert subvector overflow!");
9416 TLI->getVectorIdxWidth(getDataLayout()) &&
9417 "Constant index for INSERT_SUBVECTOR has an invalid size");
9418
9419 // Trivial insertion.
9420 if (VT == N2VT)
9421 return N2;
9422
9423 // If this is an insert of an extracted vector into an undef/poison vector,
9424 // we can just use the input to the extract. But not if skipping the
9425 // extract+insert could make the result more poisonous.
9426 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9427 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) {
9428 if (N1.getOpcode() == ISD::POISON)
9429 return N2.getOperand(0);
9430 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9431 unsigned LoBit = N3->getAsZExtVal();
9432 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9433 APInt EltMask =
9434 APInt::getBitsSet(VT.getVectorNumElements(), LoBit, HiBit);
9435 if (isGuaranteedNotToBePoison(N2.getOperand(0), ~EltMask))
9436 return N2.getOperand(0);
9437 } else if (isGuaranteedNotToBePoison(N2.getOperand(0)))
9438 return N2.getOperand(0);
9439 }
9440
9441 // If the inserted subvector is UNDEF, just use the input vector.
9442 // But not if skipping the insert could make the result more poisonous.
9443 if (N2.isUndef()) {
9444 if (VT.isFixedLengthVector()) {
9445 unsigned LoBit = N3->getAsZExtVal();
9446 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9447 APInt EltMask =
9448 APInt::getBitsSet(VT.getVectorNumElements(), LoBit, HiBit);
9449 if (isGuaranteedNotToBePoison(N1, EltMask))
9450 return N1;
9451 } else if (isGuaranteedNotToBePoison(N1))
9452 return N1;
9453 }
9454 break;
9455 }
9456 case ISD::BITCAST:
9457 // Fold bit_convert nodes from a type to themselves.
9458 if (N1.getValueType() == VT)
9459 return N1;
9460 break;
9461 case ISD::VECTOR_COMPRESS: {
9462 [[maybe_unused]] EVT VecVT = N1.getValueType();
9463 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9464 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9465 assert(VT == VecVT && "Vector and result type don't match.");
9466 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9467 "All inputs must be vectors.");
9468 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9470 "Vector and mask must have same number of elements.");
9471
9472 if (N1.isUndef() || N2.isUndef())
9473 return N3;
9474
9475 break;
9476 }
9481 [[maybe_unused]] EVT AccVT = N1.getValueType();
9482 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9483 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9484 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9485 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9486 "node to have the same type!");
9487 assert(VT.isVector() && VT == AccVT &&
9488 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9489 "the same type as its result!");
9491 AccVT.getVectorElementCount()) &&
9492 "Expected the element count of the second and third operands of the "
9493 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9494 "element count of the first operand and the result!");
9496 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9497 "node to have an element type which is the same as or smaller than "
9498 "the element type of the first operand and result!");
9499 break;
9500 }
9501 }
9502
9503 // Perform trivial constant folding for arithmetic operators.
9504 switch (Opcode) {
9508 case ISD::FMA:
9509 case ISD::FMAD:
9510 case ISD::SETCC:
9511 case ISD::FSHL:
9512 case ISD::FSHR:
9513 if (SDValue SV =
9514 FoldConstantArithmetic(Opcode, DL, VT, {N1, N2, N3}, Flags))
9515 return SV;
9516 break;
9517 }
9518
9519 // Memoize node if it doesn't produce a glue result.
9520 SDNode *N;
9521 SDVTList VTs = getVTList(VT);
9522 SDValue Ops[] = {N1, N2, N3};
9523 if (VT != MVT::Glue) {
9524 SDNodeKey ID(Opcode, VTs, Ops);
9525 FoldingSetInsertToken InsertToken;
9526 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
9527 E->intersectFlagsWith(Flags);
9528 return SDValue(E, 0);
9529 }
9530
9531 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9532 N->setFlags(Flags);
9533 createOperands(N, Ops);
9534 CSEMap.insert(N, InsertToken);
9535 } else {
9536 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9537 createOperands(N, Ops);
9538 }
9539
9540 InsertNode(N);
9541 SDValue V = SDValue(N, 0);
9542 NewSDValueDbgMsg(V, "Creating new node: ", this);
9543 return V;
9544}
9545
9546SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9547 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9548 const SDNodeFlags Flags) {
9549 SDValue Ops[] = { N1, N2, N3, N4 };
9550 return getNode(Opcode, DL, VT, Ops, Flags);
9551}
9552
9553SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9554 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9555 SDNodeFlags Flags;
9556 if (Inserter)
9557 Flags = Inserter->getFlags();
9558 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9559}
9560
9561SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9562 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9563 SDValue N5, const SDNodeFlags Flags) {
9564 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9565 return getNode(Opcode, DL, VT, Ops, Flags);
9566}
9567
9568SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9569 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9570 SDValue N5) {
9571 SDNodeFlags Flags;
9572 if (Inserter)
9573 Flags = Inserter->getFlags();
9574 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9575}
9576
9577/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9578/// the incoming stack arguments to be loaded from the stack.
9580 SmallVector<SDValue, 8> ArgChains;
9581
9582 // Include the original chain at the beginning of the list. When this is
9583 // used by target LowerCall hooks, this helps legalize find the
9584 // CALLSEQ_BEGIN node.
9585 ArgChains.push_back(Chain);
9586
9587 // Add a chain value for each stack argument.
9588 for (SDNode *U : getEntryNode().getNode()->users())
9589 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))
9590 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
9591 if (FI->getIndex() < 0)
9592 ArgChains.push_back(SDValue(L, 1));
9593
9594 // Build a tokenfactor for all the chains.
9595 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
9596}
9597
9598/// getMemsetValue - Vectorized representation of the memset value
9599/// operand.
9601 const SDLoc &dl) {
9602 assert(!Value.isUndef());
9603
9604 unsigned NumBits = VT.getScalarSizeInBits();
9606 assert(C->getAPIntValue().getBitWidth() == 8);
9607 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
9608 if (VT.isInteger()) {
9609 bool IsOpaque = VT.getSizeInBits() > 64 ||
9610 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
9611 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
9612 }
9613 return DAG.getConstantFP(APFloat(VT.getFltSemantics(), Val), dl, VT);
9614 }
9615
9616 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9617 EVT IntVT = VT.getScalarType();
9618 if (!IntVT.isInteger())
9619 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
9620
9621 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
9622 if (NumBits > 8) {
9623 // Use a multiplication with 0x010101... to extend the input to the
9624 // required length.
9625 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
9626 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
9627 DAG.getConstant(Magic, dl, IntVT));
9628 }
9629
9630 if (VT != Value.getValueType() && !VT.isInteger())
9631 Value = DAG.getBitcast(VT.getScalarType(), Value);
9632 if (VT != Value.getValueType())
9633 Value = DAG.getSplatBuildVector(VT, dl, Value);
9634
9635 return Value;
9636}
9637
9638/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9639/// used when a memcpy is turned into a memset when the source is a constant
9640/// string ptr.
9642 const TargetLowering &TLI,
9643 const ConstantDataArraySlice &Slice) {
9644 // Handle vector with all elements zero.
9645 if (Slice.Array == nullptr) {
9646 if (VT.isInteger())
9647 return DAG.getConstant(0, dl, VT);
9648 return DAG.getNode(ISD::BITCAST, dl, VT,
9649 DAG.getConstant(0, dl, VT.changeTypeToInteger()));
9650 }
9651
9652 assert(!VT.isVector() && "Can't handle vector type here!");
9653 unsigned NumVTBits = VT.getSizeInBits();
9654 unsigned NumVTBytes = NumVTBits / 8;
9655 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
9656
9657 APInt Val(NumVTBits, 0);
9658 if (DAG.getDataLayout().isLittleEndian()) {
9659 for (unsigned i = 0; i != NumBytes; ++i)
9660 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9661 } else {
9662 for (unsigned i = 0; i != NumBytes; ++i)
9663 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9664 }
9665
9666 // If the "cost" of materializing the integer immediate is less than the cost
9667 // of a load, then it is cost effective to turn the load into the immediate.
9668 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
9669 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
9670 return DAG.getConstant(Val, dl, VT);
9671 return SDValue();
9672}
9673
9675 const SDLoc &DL,
9676 const SDNodeFlags Flags) {
9677 SDValue Index = getTypeSize(DL, Base.getValueType(), Offset);
9678 return getMemBasePlusOffset(Base, Index, DL, Flags);
9679}
9680
9682 const SDLoc &DL,
9683 const SDNodeFlags Flags) {
9684 assert(Offset.getValueType().isInteger());
9685 EVT BasePtrVT = Ptr.getValueType();
9686 if (TLI->shouldPreservePtrArith(this->getMachineFunction().getFunction(),
9687 BasePtrVT))
9688 return getNode(ISD::PTRADD, DL, BasePtrVT, Ptr, Offset, Flags);
9689 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9690 SDNodeFlags AddFlags = Flags;
9691 AddFlags.setInBounds(false);
9692 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, AddFlags);
9693}
9694
9695/// Returns true if memcpy source is constant data.
9697 uint64_t SrcDelta = 0;
9698 GlobalAddressSDNode *G = nullptr;
9699 if (Src.getOpcode() == ISD::GlobalAddress)
9701 else if (Src->isAnyAdd() &&
9702 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
9703 Src.getOperand(1).getOpcode() == ISD::Constant) {
9704 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
9705 SrcDelta = Src.getConstantOperandVal(1);
9706 }
9707 if (!G)
9708 return false;
9709
9710 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
9711 SrcDelta + G->getOffset());
9712}
9713
9715 SelectionDAG &DAG) {
9716 // On Darwin, -Os means optimize for size without hurting performance, so
9717 // only really optimize for size when -Oz (MinSize) is used.
9719 return MF.getFunction().hasMinSize();
9720 return DAG.shouldOptForSize();
9721}
9722
9724 SmallVector<SDValue, 32> &OutChains, unsigned From,
9725 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9726 SmallVector<SDValue, 16> &OutStoreChains) {
9727 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9728 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9729 SmallVector<SDValue, 16> GluedLoadChains;
9730 for (unsigned i = From; i < To; ++i) {
9731 OutChains.push_back(OutLoadChains[i]);
9732 GluedLoadChains.push_back(OutLoadChains[i]);
9733 }
9734
9735 // Chain for all loads.
9736 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
9737 GluedLoadChains);
9738
9739 for (unsigned i = From; i < To; ++i) {
9740 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
9741 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
9742 ST->getBasePtr(), ST->getMemoryVT(),
9743 ST->getMemOperand());
9744 OutChains.push_back(NewStore);
9745 }
9746}
9747
9748static SDValue
9750 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9751 Align SrcAlign, bool isVol, bool AlwaysInline,
9752 MachinePointerInfo DstPtrInfo,
9753 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9754 BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
9755 const MDNode *SrcMemCacheHint) {
9756 // Turn a memcpy of undef to nop.
9757 // FIXME: We need to honor volatile even is Src is undef.
9758 if (Src.isUndef())
9759 return Chain;
9760
9761 // Expand memcpy to a series of load and store ops if the size operand falls
9762 // below a certain threshold.
9763 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9764 // rather than maybe a humongous number of loads and stores.
9766 const DataLayout &DL = DAG.getDataLayout();
9767 LLVMContext &C = *DAG.getContext();
9768 std::vector<EVT> MemOps;
9769 bool DstAlignCanChange = false;
9771 MachineFrameInfo &MFI = MF.getFrameInfo();
9772 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9774 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
9775 DstAlignCanChange = true;
9776 SrcAlign = std::max(SrcAlign, DAG.InferPtrAlign(Src).valueOrOne());
9778 // If marked as volatile, perform a copy even when marked as constant.
9779 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9780 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9781 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9782 const MemOp Op = isZeroConstant
9783 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9784 /*IsZeroMemset*/ true, isVol)
9785 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9786 SrcAlign, isVol, CopyFromConstant);
9787 if (!TLI.findOptimalMemOpLowering(
9788 C, MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
9789 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes(), nullptr))
9790 return SDValue();
9791
9792 if (DstAlignCanChange) {
9793 Type *Ty = MemOps[0].getTypeForEVT(C);
9794 Align NewDstAlign = DL.getABITypeAlign(Ty);
9795
9796 // Don't promote to an alignment that would require dynamic stack
9797 // realignment which may conflict with optimizations such as tail call
9798 // optimization.
9800 if (!TRI->hasStackRealignment(MF))
9801 if (MaybeAlign StackAlign = DL.getStackAlignment())
9802 NewDstAlign = std::min(NewDstAlign, *StackAlign);
9803
9804 if (NewDstAlign > DstAlign) {
9805 // Give the stack frame object a larger alignment if needed.
9806 if (MFI.getObjectAlign(FI->getIndex()) < NewDstAlign)
9807 MFI.setObjectAlignment(FI->getIndex(), NewDstAlign);
9808 DstAlign = NewDstAlign;
9809 }
9810 }
9811
9812 // Prepare AAInfo for loads/stores after lowering this memcpy.
9813 AAMDNodes NewAAInfo = AAInfo;
9814 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9815
9816 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
9817 bool isConstant =
9818 BatchAA && SrcVal &&
9819 BatchAA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
9820
9821 MachineMemOperand::Flags MMOFlags =
9823 SmallVector<SDValue, 16> OutLoadChains;
9824 SmallVector<SDValue, 16> OutStoreChains;
9825 SmallVector<SDValue, 32> OutChains;
9826 unsigned NumMemOps = MemOps.size();
9827 uint64_t SrcOff = 0, DstOff = 0;
9828 for (unsigned i = 0; i != NumMemOps; ++i) {
9829 EVT VT = MemOps[i];
9830 unsigned VTSize = VT.getSizeInBits() / 8;
9832
9833 if (VTSize > Size) {
9834 // Issuing an unaligned load / store pair that overlaps with the previous
9835 // pair. Adjust the offset accordingly.
9836 assert(i == NumMemOps-1 && i != 0);
9837 SrcOff -= VTSize - Size;
9838 DstOff -= VTSize - Size;
9839 }
9840
9841 if (CopyFromConstant &&
9842 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9843 // It's unlikely a store of a vector immediate can be done in a single
9844 // instruction. It would require a load from a constantpool first.
9845 // We only handle zero vectors here.
9846 // FIXME: Handle other cases where store of vector immediate is done in
9847 // a single instruction.
9848 ConstantDataArraySlice SubSlice;
9849 if (SrcOff < Slice.Length) {
9850 SubSlice = Slice;
9851 SubSlice.move(SrcOff);
9852 } else {
9853 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9854 SubSlice.Array = nullptr;
9855 SubSlice.Offset = 0;
9856 SubSlice.Length = VTSize;
9857 }
9858 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
9859 if (Value.getNode()) {
9860 Store = DAG.getStore(
9861 Chain, dl, Value,
9862 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
9863 DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags,
9864 MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9865 OutChains.push_back(Store);
9866 }
9867 }
9868
9869 if (!Store.getNode()) {
9870 // The type might not be legal for the target. This should only happen
9871 // if the type is smaller than a legal type, as on PPC, so the right
9872 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9873 // to Load/Store if NVT==VT.
9874 // FIXME does the case above also need this?
9875 EVT NVT = TLI.getTypeToTransformTo(C, VT);
9876 assert(NVT.bitsGE(VT));
9877
9878 bool isDereferenceable =
9879 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
9880 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9881 if (isDereferenceable)
9883 if (isConstant)
9884 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9885
9886 Value = DAG.getExtLoad(
9887 ISD::EXTLOAD, dl, NVT, Chain,
9888 DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),
9889 SrcPtrInfo.getWithOffset(SrcOff), VT,
9890 commonAlignment(SrcAlign, SrcOff), SrcMMOFlags,
9891 MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
9892 OutLoadChains.push_back(Value.getValue(1));
9893
9894 Store = DAG.getTruncStore(
9895 Chain, dl, Value,
9896 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
9897 DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags,
9898 MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9899 OutStoreChains.push_back(Store);
9900 }
9901 SrcOff += VTSize;
9902 DstOff += VTSize;
9903 Size -= VTSize;
9904 }
9905
9906 unsigned MaxLdStGlue = DAG.getSelectionDAGOptions().ldstmemcpy_glue_max;
9907 unsigned GluedLdStLimit =
9908 MaxLdStGlue == 0 ? TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9909 unsigned NumLdStInMemcpy = OutStoreChains.size();
9910
9911 if (NumLdStInMemcpy) {
9912 // It may be that memcpy might be converted to memset if it's memcpy
9913 // of constants. In such a case, we won't have loads and stores, but
9914 // just stores. In the absence of loads, there is nothing to gang up.
9915 if ((GluedLdStLimit <= 1) ||
9916 !DAG.getSelectionDAGOptions().enable_memcpy_dag_opt) {
9917 // If target does not care, just leave as it.
9918 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9919 OutChains.push_back(OutLoadChains[i]);
9920 OutChains.push_back(OutStoreChains[i]);
9921 }
9922 } else {
9923 // Ld/St less than/equal limit set by target.
9924 if (NumLdStInMemcpy <= GluedLdStLimit) {
9925 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
9926 NumLdStInMemcpy, OutLoadChains,
9927 OutStoreChains);
9928 } else {
9929 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9930 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9931 unsigned GlueIter = 0;
9932
9933 // Residual ld/st.
9934 if (RemainingLdStInMemcpy) {
9936 DAG, dl, OutChains, NumLdStInMemcpy - RemainingLdStInMemcpy,
9937 NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9938 }
9939
9940 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9941 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9942 GlueIter - GluedLdStLimit;
9943 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9944 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
9945 OutLoadChains, OutStoreChains);
9946 GlueIter += GluedLdStLimit;
9947 }
9948 }
9949 }
9950 }
9951 return DAG.getTokenFactor(dl, OutChains);
9952}
9953
9955 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9956 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9957 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9958 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9959 // Turn a memmove of undef to nop.
9960 // FIXME: We need to honor volatile even is Src is undef.
9961 if (Src.isUndef())
9962 return Chain;
9963
9964 // Expand memmove to a series of load and store ops if the size operand falls
9965 // below a certain threshold.
9967 const DataLayout &DL = DAG.getDataLayout();
9968 LLVMContext &C = *DAG.getContext();
9969 std::vector<EVT> MemOps;
9970 bool DstAlignCanChange = false;
9972 MachineFrameInfo &MFI = MF.getFrameInfo();
9973 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9975 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
9976 DstAlignCanChange = true;
9977 SrcAlign = std::max(SrcAlign, DAG.InferPtrAlign(Src).valueOrOne());
9978 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9979 if (!TLI.findOptimalMemOpLowering(
9980 C, MemOps, Limit,
9981 MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, isVol),
9982 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
9983 MF.getFunction().getAttributes(), nullptr))
9984 return SDValue();
9985
9986 if (DstAlignCanChange) {
9987 Type *Ty = MemOps[0].getTypeForEVT(C);
9988 Align NewDstAlign = DL.getABITypeAlign(Ty);
9989
9990 // Don't promote to an alignment that would require dynamic stack
9991 // realignment which may conflict with optimizations such as tail call
9992 // optimization.
9994 if (!TRI->hasStackRealignment(MF))
9995 if (MaybeAlign StackAlign = DL.getStackAlignment())
9996 NewDstAlign = std::min(NewDstAlign, *StackAlign);
9997
9998 if (NewDstAlign > DstAlign) {
9999 // Give the stack frame object a larger alignment if needed.
10000 if (MFI.getObjectAlign(FI->getIndex()) < NewDstAlign)
10001 MFI.setObjectAlignment(FI->getIndex(), NewDstAlign);
10002 DstAlign = NewDstAlign;
10003 }
10004 }
10005
10006 // Prepare AAInfo for loads/stores after lowering this memmove.
10007 AAMDNodes NewAAInfo = AAInfo;
10008 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
10009
10010 MachineMemOperand::Flags MMOFlags =
10012 uint64_t SrcOff = 0;
10013 SmallVector<SDValue, 8> LoadValues;
10014 SmallVector<SDValue, 8> LoadChains;
10015 SmallVector<SDValue, 8> OutChains;
10016 unsigned NumMemOps = MemOps.size();
10017 for (unsigned i = 0; i < NumMemOps; i++) {
10018 EVT VT = MemOps[i];
10019 unsigned VTSize = VT.getSizeInBits() / 8;
10020 SDValue Value;
10021 bool IsOverlapping = false;
10022
10023 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
10024 // Issuing an unaligned load / store pair that overlaps with the previous
10025 // pair. Adjust the offset accordingly.
10026 SrcOff = Size - VTSize;
10027 IsOverlapping = true;
10028 }
10029
10030 // Calculate the actual alignment at the current offset. The alignment at
10031 // SrcOff may be lower than the base alignment, especially when using
10032 // overlapping loads.
10033 Align SrcAlignAtOffset = commonAlignment(SrcAlign, SrcOff);
10034 if (IsOverlapping) {
10035 // Verify that the target allows misaligned memory accesses at the
10036 // adjusted offset when using overlapping loads.
10037 unsigned Fast;
10038 if (!TLI.allowsMisalignedMemoryAccesses(VT, SrcPtrInfo.getAddrSpace(),
10039 SrcAlignAtOffset, MMOFlags,
10040 &Fast) ||
10041 !Fast) {
10042 // This should have been caught by findOptimalMemOpLowering, but verify
10043 // here for safety.
10044 return SDValue();
10045 }
10046 }
10047
10048 bool isDereferenceable =
10049 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
10050 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
10051 if (isDereferenceable)
10053 Value =
10054 DAG.getLoad(VT, dl, Chain,
10055 DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),
10056 SrcPtrInfo.getWithOffset(SrcOff), SrcAlignAtOffset,
10057 SrcMMOFlags, NewAAInfo);
10058 LoadValues.push_back(Value);
10059 LoadChains.push_back(Value.getValue(1));
10060 SrcOff += VTSize;
10061 }
10062 Chain = DAG.getTokenFactor(dl, LoadChains);
10063 OutChains.clear();
10064 uint64_t DstOff = 0;
10065 for (unsigned i = 0; i < NumMemOps; i++) {
10066 EVT VT = MemOps[i];
10067 unsigned VTSize = VT.getSizeInBits() / 8;
10068 SDValue Store;
10069 bool IsOverlapping = false;
10070
10071 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
10072 // Issuing an unaligned load / store pair that overlaps with the previous
10073 // pair. Adjust the offset accordingly.
10074 DstOff = Size - VTSize;
10075 IsOverlapping = true;
10076 }
10077
10078 // Calculate the actual alignment at the current offset. The alignment at
10079 // DstOff may be lower than the base alignment, especially when using
10080 // overlapping stores.
10081 Align DstAlignAtOffset = commonAlignment(DstAlign, DstOff);
10082 if (IsOverlapping) {
10083 // Verify that the target allows misaligned memory accesses at the
10084 // adjusted offset when using overlapping stores.
10085 unsigned Fast;
10086 if (!TLI.allowsMisalignedMemoryAccesses(VT, DstPtrInfo.getAddrSpace(),
10087 DstAlignAtOffset, MMOFlags,
10088 &Fast) ||
10089 !Fast) {
10090 // This should have been caught by findOptimalMemOpLowering, but verify
10091 // here for safety.
10092 return SDValue();
10093 }
10094 }
10095 Store = DAG.getStore(
10096 Chain, dl, LoadValues[i],
10097 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
10098 DstPtrInfo.getWithOffset(DstOff), DstAlignAtOffset, MMOFlags,
10099 NewAAInfo);
10100 OutChains.push_back(Store);
10101 DstOff += VTSize;
10102 }
10103
10104 return DAG.getTokenFactor(dl, OutChains);
10105}
10106
10107/// Lower the call to 'memset' intrinsic function into a series of store
10108/// operations.
10109///
10110/// \param DAG Selection DAG where lowered code is placed.
10111/// \param dl Link to corresponding IR location.
10112/// \param Chain Control flow dependency.
10113/// \param Dst Pointer to destination memory location.
10114/// \param Src Value of byte to write into the memory.
10115/// \param Size Number of bytes to write.
10116/// \param Alignment Alignment of the destination in bytes.
10117/// \param isVol True if destination is volatile.
10118/// \param AlwaysInline Makes sure no function call is generated.
10119/// \param DstPtrInfo IR information on the memory pointer.
10120/// \returns New head in the control flow, if lowering was successful, empty
10121/// SDValue otherwise.
10122///
10123/// The function tries to replace 'llvm.memset' intrinsic with several store
10124/// operations and value calculation code. This is usually profitable for small
10125/// memory size or when the semantic requires inlining.
10127 SDValue Chain, SDValue Dst, SDValue Src,
10128 uint64_t Size, Align Alignment, bool isVol,
10129 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
10130 const AAMDNodes &AAInfo) {
10131 // Turn a memset of undef to nop.
10132 // FIXME: We need to honor volatile even is Src is undef.
10133 if (Src.isUndef())
10134 return Chain;
10135
10136 // Expand memset to a series of load/store ops if the size operand
10137 // falls below a certain threshold.
10139 std::vector<EVT> MemOps;
10140 bool DstAlignCanChange = false;
10141 LLVMContext &C = *DAG.getContext();
10143 MachineFrameInfo &MFI = MF.getFrameInfo();
10144 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
10146 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
10147 DstAlignCanChange = true;
10148 bool IsZeroVal = isNullConstant(Src);
10149 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
10150
10151 EVT LargestVT;
10152 if (!TLI.findOptimalMemOpLowering(
10153 C, MemOps, Limit,
10154 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
10155 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes(),
10156 &LargestVT))
10157 return SDValue();
10158
10159 if (DstAlignCanChange) {
10160 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
10161 const DataLayout &DL = DAG.getDataLayout();
10162 Align NewAlign = DL.getABITypeAlign(Ty);
10163
10164 // Don't promote to an alignment that would require dynamic stack
10165 // realignment which may conflict with optimizations such as tail call
10166 // optimization.
10168 if (!TRI->hasStackRealignment(MF))
10169 if (MaybeAlign StackAlign = DL.getStackAlignment())
10170 NewAlign = std::min(NewAlign, *StackAlign);
10171
10172 if (NewAlign > Alignment) {
10173 // Give the stack frame object a larger alignment if needed.
10174 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
10175 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
10176 Alignment = NewAlign;
10177 }
10178 }
10179
10180 SmallVector<SDValue, 8> OutChains;
10181 uint64_t DstOff = 0;
10182 unsigned NumMemOps = MemOps.size();
10183
10184 // Find the largest store and generate the bit pattern for it.
10185 // If target didn't set LargestVT, compute it from MemOps.
10186 if (!LargestVT.isSimple()) {
10187 LargestVT = MemOps[0];
10188 for (unsigned i = 1; i < NumMemOps; i++)
10189 if (MemOps[i].bitsGT(LargestVT))
10190 LargestVT = MemOps[i];
10191 }
10192 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
10193
10194 // Prepare AAInfo for loads/stores after lowering this memset.
10195 AAMDNodes NewAAInfo = AAInfo;
10196 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
10197
10198 for (unsigned i = 0; i < NumMemOps; i++) {
10199 EVT VT = MemOps[i];
10200 unsigned VTSize = VT.getSizeInBits() / 8;
10201 // The target should specify store types that exactly cover the memset size
10202 // (with the last store potentially being oversized for overlapping stores).
10203 assert(Size > 0 && "Target specified more stores than needed in "
10204 "findOptimalMemOpLowering");
10205 if (VTSize > Size) {
10206 // Issuing an unaligned load / store pair that overlaps with the previous
10207 // pair. Adjust the offset accordingly.
10208 assert(i == NumMemOps-1 && i != 0);
10209 DstOff -= VTSize - Size;
10210 }
10211
10212 // If this store is smaller than the largest store see whether we can get
10213 // the smaller value for free with a truncate or extract vector element and
10214 // then store.
10215 SDValue Value = MemSetValue;
10216 if (VT.bitsLT(LargestVT)) {
10217 unsigned Index;
10218 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
10219 EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);
10220 if (!LargestVT.isVector() && !VT.isVector() &&
10221 TLI.isTruncateFree(LargestVT, VT))
10222 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
10223 else if (LargestVT.isVector() && !VT.isVector() &&
10224 TLI.shallExtractConstSplatVectorElementToStore(
10225 LargestVT.getTypeForEVT(*DAG.getContext()),
10226 VT.getSizeInBits(), Index) &&
10227 TLI.isTypeLegal(SVT) &&
10228 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
10229 // Target which can combine store(extractelement VectorTy, Idx) can get
10230 // the smaller value for free.
10231 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);
10232 Value = DAG.getExtractVectorElt(dl, VT, TailValue, Index);
10233 } else
10234 Value = getMemsetValue(Src, VT, DAG, dl);
10235 }
10236 assert(Value.getValueType() == VT && "Value with wrong type.");
10237 SDValue Store = DAG.getStore(
10238 Chain, dl, Value,
10239 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
10240 DstPtrInfo.getWithOffset(DstOff), Alignment,
10242 NewAAInfo);
10243 OutChains.push_back(Store);
10244 DstOff += VT.getSizeInBits() / 8;
10245 // For oversized overlapping stores, only subtract the remaining bytes.
10246 // For normal stores, subtract the full store size.
10247 if (VTSize > Size) {
10248 Size = 0;
10249 } else {
10250 Size -= VTSize;
10251 }
10252 }
10253
10254 // After processing all stores, Size should be exactly 0. Any remaining bytes
10255 // indicate a bug in the target's findOptimalMemOpLowering implementation.
10256 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
10257 "stores that exactly cover the memset size");
10258
10259 return DAG.getTokenFactor(dl, OutChains);
10260}
10261
10263 const DataLayout &DL, unsigned AS) {
10264 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
10265 // pointer operands can be losslessly bitcasted to pointers of address space 0
10266 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(DL, AS, 0)) {
10267 report_fatal_error("cannot lower memory intrinsic in address space " +
10268 Twine(AS));
10269 }
10270}
10271
10272/// The length of a memory intrinsic (e.g. number of bytes to copy) is unsigned
10273/// and may have any integer type. We zero-extend or truncate it to the pointer
10274/// type of the narrower address space being accessed.
10276 SDValue Size, unsigned DstAS,
10277 unsigned SrcAS) {
10279 MVT DstVT = TLI.getPointerTy(DAG.getDataLayout(), DstAS);
10280 MVT SrcVT = TLI.getPointerTy(DAG.getDataLayout(), SrcAS);
10281 return DAG.getZExtOrTrunc(Size, dl, DstVT.bitsLT(SrcVT) ? DstVT : SrcVT);
10282}
10283
10285 const SelectionDAG *SelDAG,
10286 bool AllowReturnsFirstArg) {
10287 if (!CI || !CI->isTailCall())
10288 return false;
10289 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
10290 // helper symbol we lower to.
10291 return isInTailCallPosition(*CI, SelDAG->getTarget(),
10292 AllowReturnsFirstArg &&
10294}
10295
10296static std::pair<SDValue, SDValue>
10299 const CallInst *CI, RTLIB::Libcall Call,
10300 SelectionDAG *DAG, const TargetLowering *TLI) {
10301 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
10302
10303 if (LCImpl == RTLIB::Unsupported)
10304 return {};
10305
10307 bool IsTailCall =
10308 isInTailCallPositionWrapper(CI, DAG, /*AllowReturnsFirstArg=*/true) &&
10309 // Lowering doesn't support tail calling inside a function with
10310 // a swifterror argument yet.
10311 !DAG->hasSwiftErrorArg();
10312 SDValue Callee =
10313 DAG->getExternalSymbol(LCImpl, TLI->getPointerTy(DAG->getDataLayout()));
10314
10315 CLI.setDebugLoc(dl)
10316 .setChain(Chain)
10318 CI->getType(), Callee, std::move(Args))
10319 .setTailCall(IsTailCall);
10320
10321 return TLI->LowerCallTo(CLI);
10322}
10323
10324std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
10325 const SDLoc &dl, SDValue S1,
10326 SDValue S2,
10327 const CallInst *CI) {
10329 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10330 return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI,
10331 RTLIB::STRCMP, this, TLI);
10332}
10333
10334std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
10335 const SDLoc &dl, SDValue S1,
10336 SDValue S2,
10337 const CallInst *CI) {
10339 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10340 return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI,
10341 RTLIB::STRSTR, this, TLI);
10342}
10343
10344std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
10345 const SDLoc &dl,
10346 SDValue Dst, SDValue Src,
10348 const CallInst *CI) {
10350
10352 {Dst, PT},
10353 {Src, PT},
10356 return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI,
10357 RTLIB::MEMCCPY, this, TLI);
10358}
10359
10360std::pair<SDValue, SDValue>
10362 SDValue Mem1, SDValue Size, const CallInst *CI) {
10365 {Mem0, PT},
10366 {Mem1, PT},
10368 return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI,
10369 RTLIB::MEMCMP, this, TLI);
10370}
10371
10372std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
10373 const SDLoc &dl,
10374 SDValue Dst, SDValue Src,
10375 const CallInst *CI) {
10377 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
10378 return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI,
10379 RTLIB::STRCPY, this, TLI);
10380}
10381
10382std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
10383 const SDLoc &dl,
10384 SDValue Src,
10385 const CallInst *CI) {
10386 // Emit a library call.
10389 return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI,
10390 RTLIB::STRLEN, this, TLI);
10391}
10392
10394 return TLI->supportSwiftError() &&
10395 MF->getFunction().getAttributes().hasAttrSomewhere(
10396 Attribute::SwiftError);
10397}
10398
10400 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
10401 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
10402 const CallInst *CI, std::optional<bool> OverrideTailCall,
10403 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
10404 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
10405 Size = getMemIntrinsicSize(*this, dl, Size, DstPtrInfo.getAddrSpace(),
10406 SrcPtrInfo.getAddrSpace());
10407 // Check to see if we should lower the memcpy to loads and stores first.
10408 // For cases within the target-specified limits, this is the best choice.
10409 const MDNode *DstMemCacheHint =
10410 CI ? getMemCacheHint(*CI, /*OperandNo=*/0) : nullptr;
10411 const MDNode *SrcMemCacheHint =
10412 CI ? getMemCacheHint(*CI, /*OperandNo=*/1) : nullptr;
10413
10415 if (ConstantSize) {
10416 // Memcpy with size zero? Just return the original chain.
10417 if (ConstantSize->isZero())
10418 return Chain;
10419
10421 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
10422 SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10423 DstMemCacheHint, SrcMemCacheHint);
10424 if (Result.getNode())
10425 return Result;
10426 }
10427
10428 // Then check to see if we should lower the memcpy with target-specific
10429 // code. If the target chooses to do this, this is the next best.
10430 if (TSI) {
10431 bool IsTailCall = OverrideTailCall.value_or(isInTailCallPositionWrapper(
10432 CI, this, /*AllowReturnsFirstArg=*/false)) &&
10434 [[maybe_unused]] SDValue OldRoot = getRoot();
10435 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10436 *this, dl, Chain, Dst, Src, Size, DstAlign, SrcAlign, isVol,
10437 AlwaysInline, IsTailCall, DstPtrInfo, SrcPtrInfo);
10438 assert((Result.getNode() || !IsTailCall || getRoot() != OldRoot) &&
10439 "target declined without clearing IsTailCall");
10440 if (Result.getNode() || IsTailCall)
10441 return Result;
10442 }
10443
10444 // If we really need inline code and the target declined to provide it,
10445 // use a (potentially long) sequence of loads and stores.
10446 if (AlwaysInline) {
10447 assert(ConstantSize && "AlwaysInline requires a constant size!");
10449 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
10450 SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10451 DstMemCacheHint, SrcMemCacheHint);
10452 }
10453
10455 DstPtrInfo.getAddrSpace());
10457 SrcPtrInfo.getAddrSpace());
10458
10459 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10460 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10461 // respect volatile, so they may do things like read or write memory
10462 // beyond the given memory regions. But fixing this isn't easy, and most
10463 // people don't care.
10464
10465 // Emit a library call.
10468 Args.emplace_back(Dst, PtrTy);
10469 Args.emplace_back(Src, PtrTy);
10470 Args.emplace_back(Size, getDataLayout().getIntPtrType(*getContext()));
10471 // FIXME: pass in SDLoc
10473 bool IsTailCall = false;
10474 RTLIB::LibcallImpl MemCpyImpl = Libcalls->getMemcpyImpl();
10475
10476 if (OverrideTailCall.has_value()) {
10477 IsTailCall = *OverrideTailCall;
10478 } else {
10479 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10480 IsTailCall = isInTailCallPositionWrapper(CI, this, LowersToMemcpy);
10481 }
10482 // Lowering doesn't support tail calling inside a function with a
10483 // swifterror argument yet.
10484 IsTailCall &= !hasSwiftErrorArg();
10485
10486 CLI.setDebugLoc(dl)
10487 .setChain(Chain)
10488 .setLibCallee(
10489 Libcalls->getLibcallImplCallingConv(MemCpyImpl),
10490 Dst.getValueType().getTypeForEVT(*getContext()),
10491 getExternalSymbol(MemCpyImpl, TLI->getPointerTy(getDataLayout())),
10492 std::move(Args))
10494 .setTailCall(IsTailCall);
10495
10496 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10497 return CallResult.second;
10498}
10499
10502 const Module &M = *DAG->getMachineFunction().getFunction().getParent();
10503 Type *SizeTTy = DAG->getLibInfo().getSizeTType(M);
10505 DAG->getZExtOrTrunc(Size, dl, MVT::getVT(SizeTTy)), SizeTTy);
10506}
10507
10509 SDValue Dst, SDValue Src, SDValue Size,
10510 Type *SizeTy, unsigned ElemSz,
10511 bool isTailCall,
10512 MachinePointerInfo DstPtrInfo,
10513 MachinePointerInfo SrcPtrInfo) {
10514 // Lowering doesn't support tail calling inside a function with a
10515 // swifterror argument yet.
10516 isTailCall &= !hasSwiftErrorArg();
10517
10518 // Emit a library call.
10521 Args.emplace_back(Dst, ArgTy);
10522 Args.emplace_back(Src, ArgTy);
10523 Args.emplace_back(getAtomicMemOpSizeArg(this, dl, Size));
10524
10525 RTLIB::Libcall LibraryCall =
10527 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(LibraryCall);
10528 if (LibcallImpl == RTLIB::Unsupported)
10529 report_fatal_error("Unsupported element size");
10530
10532 CLI.setDebugLoc(dl)
10533 .setChain(Chain)
10534 .setLibCallee(
10535 Libcalls->getLibcallImplCallingConv(LibcallImpl),
10537 getExternalSymbol(LibcallImpl, TLI->getPointerTy(getDataLayout())),
10538 std::move(Args))
10540 .setTailCall(isTailCall);
10541
10542 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10543 return CallResult.second;
10544}
10545
10547 SDValue Src, SDValue Size, Align DstAlign,
10548 Align SrcAlign, bool isVol, const CallInst *CI,
10549 std::optional<bool> OverrideTailCall,
10550 MachinePointerInfo DstPtrInfo,
10551 MachinePointerInfo SrcPtrInfo,
10552 const AAMDNodes &AAInfo,
10553 BatchAAResults *BatchAA) {
10554 Size = getMemIntrinsicSize(*this, dl, Size, DstPtrInfo.getAddrSpace(),
10555 SrcPtrInfo.getAddrSpace());
10556 // Check to see if we should lower the memmove to loads and stores first.
10557 // For cases within the target-specified limits, this is the best choice.
10559 if (ConstantSize) {
10560 // Memmove with size zero? Just return the original chain.
10561 if (ConstantSize->isZero())
10562 return Chain;
10563
10565 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
10566 SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
10567 if (Result.getNode())
10568 return Result;
10569 }
10570
10571 // Then check to see if we should lower the memmove with target-specific
10572 // code. If the target chooses to do this, this is the next best.
10573 if (TSI) {
10574 bool IsTailCall = OverrideTailCall.value_or(isInTailCallPositionWrapper(
10575 CI, this, /*AllowReturnsFirstArg=*/false)) &&
10577 [[maybe_unused]] SDValue OldRoot = getRoot();
10578 SDValue Result = TSI->EmitTargetCodeForMemmove(
10579 *this, dl, Chain, Dst, Src, Size, DstAlign, SrcAlign, isVol, IsTailCall,
10580 DstPtrInfo, SrcPtrInfo);
10581 assert((Result.getNode() || !IsTailCall || getRoot() != OldRoot) &&
10582 "target declined without clearing IsTailCall");
10583 if (Result.getNode() || IsTailCall)
10584 return Result;
10585 }
10586
10588 DstPtrInfo.getAddrSpace());
10590 SrcPtrInfo.getAddrSpace());
10591
10592 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10593 // not be safe. See memcpy above for more details.
10594
10595 // Emit a library call.
10598 Args.emplace_back(Dst, PtrTy);
10599 Args.emplace_back(Src, PtrTy);
10600 Args.emplace_back(Size, getDataLayout().getIntPtrType(*getContext()));
10601 // FIXME: pass in SDLoc
10603
10604 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(RTLIB::MEMMOVE);
10605
10606 bool IsTailCall = false;
10607 if (OverrideTailCall.has_value()) {
10608 IsTailCall = *OverrideTailCall;
10609 } else {
10610 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10611 IsTailCall = isInTailCallPositionWrapper(CI, this, LowersToMemmove);
10612 }
10613 // Lowering doesn't support tail calling inside a function with a
10614 // swifterror argument yet.
10615 IsTailCall &= !hasSwiftErrorArg();
10616
10617 CLI.setDebugLoc(dl)
10618 .setChain(Chain)
10619 .setLibCallee(
10620 Libcalls->getLibcallImplCallingConv(MemmoveImpl),
10621 Dst.getValueType().getTypeForEVT(*getContext()),
10622 getExternalSymbol(MemmoveImpl, TLI->getPointerTy(getDataLayout())),
10623 std::move(Args))
10625 .setTailCall(IsTailCall);
10626
10627 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10628 return CallResult.second;
10629}
10630
10632 SDValue Dst, SDValue Src, SDValue Size,
10633 Type *SizeTy, unsigned ElemSz,
10634 bool isTailCall,
10635 MachinePointerInfo DstPtrInfo,
10636 MachinePointerInfo SrcPtrInfo) {
10637 // Lowering doesn't support tail calling inside a function with a
10638 // swifterror argument yet.
10639 isTailCall &= !hasSwiftErrorArg();
10640
10641 // Emit a library call.
10644 Args.emplace_back(Dst, IntPtrTy);
10645 Args.emplace_back(Src, IntPtrTy);
10646 Args.emplace_back(getAtomicMemOpSizeArg(this, dl, Size));
10647
10648 RTLIB::Libcall LibraryCall =
10650 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(LibraryCall);
10651 if (LibcallImpl == RTLIB::Unsupported)
10652 report_fatal_error("Unsupported element size");
10653
10655 CLI.setDebugLoc(dl)
10656 .setChain(Chain)
10657 .setLibCallee(
10658 Libcalls->getLibcallImplCallingConv(LibcallImpl),
10660 getExternalSymbol(LibcallImpl, TLI->getPointerTy(getDataLayout())),
10661 std::move(Args))
10663 .setTailCall(isTailCall);
10664
10665 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10666 return CallResult.second;
10667}
10668
10670 SDValue Src, SDValue Size, Align Alignment,
10671 bool isVol, bool AlwaysInline,
10672 const CallInst *CI,
10673 MachinePointerInfo DstPtrInfo,
10674 const AAMDNodes &AAInfo) {
10675 Size = getMemIntrinsicSize(*this, dl, Size, DstPtrInfo.getAddrSpace(),
10676 DstPtrInfo.getAddrSpace());
10677 // Check to see if we should lower the memset to stores first.
10678 // For cases within the target-specified limits, this is the best choice.
10680 if (ConstantSize) {
10681 // Memset with size zero? Just return the original chain.
10682 if (ConstantSize->isZero())
10683 return Chain;
10684
10685 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
10686 ConstantSize->getZExtValue(), Alignment,
10687 isVol, false, DstPtrInfo, AAInfo);
10688
10689 if (Result.getNode())
10690 return Result;
10691 }
10692
10693 // Then check to see if we should lower the memset with target-specific
10694 // code. If the target chooses to do this, this is the next best.
10695 if (TSI) {
10696 bool IsTailCall =
10697 isInTailCallPositionWrapper(CI, this, /*AllowReturnsFirstArg=*/false) &&
10699 [[maybe_unused]] SDValue OldRoot = getRoot();
10700 SDValue Result = TSI->EmitTargetCodeForMemset(
10701 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
10702 IsTailCall, DstPtrInfo);
10703 assert((Result.getNode() || !IsTailCall || getRoot() != OldRoot) &&
10704 "target declined without clearing IsTailCall");
10705 if (Result.getNode() || IsTailCall)
10706 return Result;
10707 }
10708
10709 // If we really need inline code and the target declined to provide it,
10710 // use a (potentially long) sequence of loads and stores.
10711 if (AlwaysInline) {
10712 assert(ConstantSize && "AlwaysInline requires a constant size!");
10713 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
10714 ConstantSize->getZExtValue(), Alignment,
10715 isVol, true, DstPtrInfo, AAInfo);
10716 assert(Result &&
10717 "getMemsetStores must return a valid sequence when AlwaysInline");
10718 return Result;
10719 }
10720
10722 DstPtrInfo.getAddrSpace());
10723
10724 // Emit a library call.
10725 auto &Ctx = *getContext();
10726 const auto& DL = getDataLayout();
10727
10729 // FIXME: pass in SDLoc
10730 CLI.setDebugLoc(dl).setChain(Chain);
10731
10732 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(RTLIB::BZERO);
10733 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(Src);
10734
10735 // If zeroing out and bzero is present, use it.
10736 if (UseBZero) {
10738 Args.emplace_back(Dst, PointerType::getUnqual(Ctx));
10739 Args.emplace_back(Size, DL.getIntPtrType(Ctx));
10740 CLI.setLibCallee(
10741 Libcalls->getLibcallImplCallingConv(BzeroImpl), Type::getVoidTy(Ctx),
10742 getExternalSymbol(BzeroImpl, TLI->getPointerTy(DL)), std::move(Args));
10743 } else {
10744 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(RTLIB::MEMSET);
10745
10746 MVT IntVT = MVT::getIntegerVT(LibInfo->getIntSize());
10748 Args.emplace_back(Dst, PointerType::getUnqual(Ctx));
10749 Args.emplace_back(getAnyExtOrTrunc(Src, dl, IntVT),
10750 Type::getIntNTy(Ctx, LibInfo->getIntSize()),
10751 LibInfo->getExtAttrForI32Param());
10752 Args.emplace_back(Size, DL.getIntPtrType(Ctx));
10753
10754 CLI.setLibCallee(Libcalls->getLibcallImplCallingConv(MemsetImpl),
10755 Dst.getValueType().getTypeForEVT(Ctx),
10756 getExternalSymbol(MemsetImpl, TLI->getPointerTy(DL)),
10757 std::move(Args));
10758 }
10759
10760 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(RTLIB::MEMSET);
10761 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10762
10763 // If we're going to use bzero, make sure not to tail call unless the
10764 // subsequent return doesn't need a value, as bzero doesn't return the first
10765 // arg unlike memset.
10766 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI) && !UseBZero;
10767 bool IsTailCall = CI && CI->isTailCall() &&
10769 ReturnsFirstArg && LowersToMemset) &&
10770 // Lowering doesn't support tail calling inside a function
10771 // with a swifterror argument yet.
10773 CLI.setDiscardResult().setTailCall(IsTailCall);
10774
10775 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10776 return CallResult.second;
10777}
10778
10781 Type *SizeTy, unsigned ElemSz,
10782 bool isTailCall,
10783 MachinePointerInfo DstPtrInfo) {
10784 // Lowering doesn't support tail calling inside a function with a
10785 // swifterror argument yet.
10786 isTailCall &= !hasSwiftErrorArg();
10787
10788 // Emit a library call.
10789 MVT IntVT = MVT::getIntegerVT(LibInfo->getIntSize());
10791 Args.emplace_back(Dst, getDataLayout().getIntPtrType(*getContext()));
10792 Args.emplace_back(getAnyExtOrTrunc(Value, dl, IntVT),
10793 Type::getIntNTy(*getContext(), LibInfo->getIntSize()),
10794 LibInfo->getExtAttrForI32Param());
10795 Args.emplace_back(getAtomicMemOpSizeArg(this, dl, Size));
10796
10797 RTLIB::Libcall LibraryCall =
10799 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(LibraryCall);
10800 if (LibcallImpl == RTLIB::Unsupported)
10801 report_fatal_error("Unsupported element size");
10802
10804 CLI.setDebugLoc(dl)
10805 .setChain(Chain)
10806 .setLibCallee(
10807 Libcalls->getLibcallImplCallingConv(LibcallImpl),
10809 getExternalSymbol(LibcallImpl, TLI->getPointerTy(getDataLayout())),
10810 std::move(Args))
10812 .setTailCall(isTailCall);
10813
10814 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10815 return CallResult.second;
10816}
10817
10818SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10820 MachineMemOperand *MMO,
10821 ISD::LoadExtType ExtType) {
10822 SDNodeKey ID(Opcode, VTList, Ops);
10823 ID.AddInteger(MemVT.getRawBits());
10824 ID.AddInteger(getSyntheticNodeSubclassData<AtomicSDNode>(
10825 dl.getIROrder(), Opcode, VTList, MemVT, MMO, ExtType));
10826 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10827 ID.AddInteger(MMO->getFlags());
10828 FoldingSetInsertToken InsertToken;
10829 if (auto *E = cast_or_null<AtomicSDNode>(lookupNode(ID, dl, InsertToken))) {
10830 E->refineAlignment(MMO);
10831 E->refineMMOMetadata(MMO);
10832 return SDValue(E, 0);
10833 }
10834
10835 auto *N = newSDNode<AtomicSDNode>(dl.getIROrder(), dl.getDebugLoc(), Opcode,
10836 VTList, MemVT, MMO, ExtType);
10837 createOperands(N, Ops);
10838
10839 CSEMap.insert(N, InsertToken);
10840 InsertNode(N);
10841 SDValue V(N, 0);
10842 NewSDValueDbgMsg(V, "Creating new node: ", this);
10843 return V;
10844}
10845
10847 EVT MemVT, SDVTList VTs, SDValue Chain,
10848 SDValue Ptr, SDValue Cmp, SDValue Swp,
10849 MachineMemOperand *MMO) {
10850 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10852 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10853
10854 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10855 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
10856}
10857
10858SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10859 SDValue Chain, SDValue Ptr, SDValue Val,
10860 MachineMemOperand *MMO) {
10861 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10862 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10863 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10864 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10865 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10866 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10867 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10868 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10869 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10870 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10871 Opcode == ISD::ATOMIC_LOAD_FMINIMUMNUM ||
10872 Opcode == ISD::ATOMIC_LOAD_FMAXIMUMNUM ||
10873 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10874 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10875 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10876 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10877 Opcode == ISD::ATOMIC_STORE) &&
10878 "Invalid Atomic Op");
10879
10880 EVT VT = Val.getValueType();
10881
10882 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
10883 getVTList(VT, MVT::Other);
10884 SDValue Ops[] = {Chain, Ptr, Val};
10885 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
10886}
10887
10889 EVT MemVT, EVT VT, SDValue Chain,
10890 SDValue Ptr, MachineMemOperand *MMO) {
10891 SDVTList VTs = getVTList(VT, MVT::Other);
10892 SDValue Ops[] = {Chain, Ptr};
10893 return getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, VTs, Ops, MMO, ExtType);
10894}
10895
10896/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10898 if (Ops.size() == 1)
10899 return Ops[0];
10900
10902 VTs.reserve(Ops.size());
10903 for (const SDValue &Op : Ops)
10904 VTs.push_back(Op.getValueType());
10905 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
10906}
10907
10909 SDValue Chain, const SDLoc &dl) {
10910 SmallVector<SDValue, 4> RetValues;
10911 RetValues.reserve(ResultTypes.size());
10912 for (EVT VT : ResultTypes)
10913 RetValues.push_back(VT == MVT::Other ? Chain : getPOISON(VT));
10914 return getMergeValues(RetValues, dl);
10915}
10916
10918 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10919 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10921 const AAMDNodes &AAInfo) {
10922 if (Size.hasValue() && !Size.getValue())
10924
10926 MachineMemOperand *MMO =
10927 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
10928
10929 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10930}
10931
10933 SDVTList VTList,
10934 ArrayRef<SDValue> Ops, EVT MemVT,
10935 MachineMemOperand *MMO) {
10936 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, ArrayRef(MMO));
10937}
10938
10940 SDVTList VTList,
10941 ArrayRef<SDValue> Ops, EVT MemVT,
10943 assert(!MMOs.empty() && "Must have at least one MMO");
10944 assert(
10945 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10946 Opcode == ISD::PREFETCH ||
10947 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10948 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10949 "Opcode is not a memory-accessing opcode!");
10950
10952 if (MMOs.size() == 1) {
10953 MemRefs = MMOs[0];
10954 } else {
10955 // Allocate: [size_t count][MMO*][MMO*]...
10956 size_t AllocSize =
10957 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10958 void *Buffer = Allocator.Allocate(AllocSize, alignof(size_t));
10959 size_t *CountPtr = static_cast<size_t *>(Buffer);
10960 *CountPtr = MMOs.size();
10961 MachineMemOperand **Array =
10962 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10963 llvm::copy(MMOs, Array);
10964 MemRefs = Array;
10965 }
10966
10967 // Memoize the node unless it returns a glue result.
10969 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10970 SDNodeKey ID(Opcode, VTList, Ops);
10971 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10972 Opcode, dl.getIROrder(), VTList, MemVT, MemRefs));
10973 ID.AddInteger(MemVT.getRawBits());
10974 for (const MachineMemOperand *MMO : MMOs) {
10975 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10976 ID.AddInteger(MMO->getFlags());
10977 }
10978 FoldingSetInsertToken InsertToken;
10979 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
10980 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMOs);
10981 return SDValue(E, 0);
10982 }
10983
10984 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
10985 VTList, MemVT, MemRefs);
10986 createOperands(N, Ops);
10987 CSEMap.insert(N, InsertToken);
10988 } else {
10989 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
10990 VTList, MemVT, MemRefs);
10991 createOperands(N, Ops);
10992 }
10993 InsertNode(N);
10994 SDValue V(N, 0);
10995 NewSDValueDbgMsg(V, "Creating new node: ", this);
10996 return V;
10997}
10998
11000 SDValue Chain, int FrameIndex) {
11001 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
11002 const auto VTs = getVTList(MVT::Other);
11003 SDValue Ops[2] = {
11004 Chain,
11005 getFrameIndex(FrameIndex,
11006 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
11007 true)};
11008
11009 SDNodeKey ID(Opcode, VTs, Ops);
11010 FoldingSetInsertToken InsertToken;
11011 if (SDNode *E = lookupNode(ID, dl, InsertToken))
11012 return SDValue(E, 0);
11013
11014 LifetimeSDNode *N =
11015 newSDNode<LifetimeSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs);
11016 createOperands(N, Ops);
11017 CSEMap.insert(N, InsertToken);
11018 InsertNode(N);
11019 SDValue V(N, 0);
11020 NewSDValueDbgMsg(V, "Creating new node: ", this);
11021 return V;
11022}
11023
11025 uint64_t Guid, uint64_t Index,
11026 uint32_t Attr) {
11027 const unsigned Opcode = ISD::PSEUDO_PROBE;
11028 const auto VTs = getVTList(MVT::Other);
11029 SDValue Ops[] = {Chain};
11030 SDNodeKey ID(Opcode, VTs, Ops);
11031 ID.AddInteger(Guid);
11032 ID.AddInteger(Index);
11033 ID.AddInteger(Attr);
11034 FoldingSetInsertToken InsertToken;
11035 if (SDNode *E = lookupNode(ID, Dl, InsertToken))
11036 return SDValue(E, 0);
11037
11038 auto *N = newSDNode<PseudoProbeSDNode>(
11039 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
11040 createOperands(N, Ops);
11041 CSEMap.insert(N, InsertToken);
11042 InsertNode(N);
11043 SDValue V(N, 0);
11044 NewSDValueDbgMsg(V, "Creating new node: ", this);
11045 return V;
11046}
11047
11048/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
11049/// MachinePointerInfo record from it. This is particularly useful because the
11050/// code generator has many cases where it doesn't bother passing in a
11051/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
11053 SelectionDAG &DAG, SDValue Ptr,
11054 int64_t Offset = 0) {
11055 // If this is FI+Offset, we can model it.
11056 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
11058 FI->getIndex(), Offset);
11059
11060 // If this is (FI+Offset1)+Offset2, we can model it.
11061 if (Ptr.getOpcode() != ISD::ADD ||
11064 return Info;
11065
11066 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
11068 DAG.getMachineFunction(), FI,
11069 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
11070}
11071
11072/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
11073/// MachinePointerInfo record from it. This is particularly useful because the
11074/// code generator has many cases where it doesn't bother passing in a
11075/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
11077 SelectionDAG &DAG, SDValue Ptr,
11078 SDValue OffsetOp) {
11079 // If the 'Offset' value isn't a constant, we can't handle this.
11081 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
11082 if (OffsetOp.isUndef())
11083 return InferPointerInfo(Info, DAG, Ptr);
11084 return Info;
11085}
11086
11088 EVT VT, const SDLoc &dl, SDValue Chain,
11089 SDValue Ptr, SDValue Offset,
11090 MachinePointerInfo PtrInfo, EVT MemVT,
11091 Align Alignment,
11092 MachineMemOperand::Flags MMOFlags,
11093 const MMOMetadata &Metadata) {
11094 assert(Chain.getValueType() == MVT::Other &&
11095 "Invalid chain type");
11096
11097 MMOFlags |= MachineMemOperand::MOLoad;
11098 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11099 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11100 // clients.
11101 if (PtrInfo.V.isNull())
11102 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
11103
11104 TypeSize Size = MemVT.getStoreSize();
11106 MachineMemOperand *MMO =
11107 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, Metadata);
11108 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
11109}
11110
11112 EVT VT, const SDLoc &dl, SDValue Chain,
11113 SDValue Ptr, SDValue Offset, EVT MemVT,
11114 MachineMemOperand *MMO) {
11115 if (VT == MemVT) {
11116 ExtType = ISD::NON_EXTLOAD;
11117 } else if (ExtType == ISD::NON_EXTLOAD) {
11118 assert(VT == MemVT && "Non-extending load from different memory type!");
11119 } else {
11120 // Extending load.
11121 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
11122 "Should only be an extending load, not truncating!");
11123 assert(VT.isInteger() == MemVT.isInteger() &&
11124 "Cannot convert from FP to Int or Int -> FP!");
11125 assert(VT.isVector() == MemVT.isVector() &&
11126 "Cannot use an ext load to convert to or from a vector!");
11127 assert((!VT.isVector() ||
11129 "Cannot use an ext load to change the number of vector elements!");
11130 }
11131
11132 assert((!MMO->getRanges() ||
11134 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
11135 MemVT.isInteger())) &&
11136 "Range metadata and load type must match!");
11137
11138 bool Indexed = AM != ISD::UNINDEXED;
11139 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11140 "Unindexed load with an offset!");
11141
11142 SDVTList VTs = Indexed ?
11143 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
11144 SDValue Ops[] = { Chain, Ptr, Offset };
11145 SDNodeKey ID(ISD::LOAD, VTs, Ops);
11146 ID.AddInteger(MemVT.getRawBits());
11147 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
11148 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
11149 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11150 ID.AddInteger(MMO->getFlags());
11151 FoldingSetInsertToken InsertToken;
11152 if (auto *E = cast_or_null<LoadSDNode>(lookupNode(ID, dl, InsertToken))) {
11153 E->refineAlignment(MMO);
11154 E->refineMMOMetadata(MMO);
11155 return SDValue(E, 0);
11156 }
11157 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
11158 ExtType, MemVT, MMO);
11159 createOperands(N, Ops);
11160
11161 CSEMap.insert(N, InsertToken);
11162 InsertNode(N);
11163 SDValue V(N, 0);
11164 NewSDValueDbgMsg(V, "Creating new node: ", this);
11165 return V;
11166}
11167
11169 SDValue Ptr, MachinePointerInfo PtrInfo,
11170 MaybeAlign Alignment,
11171 MachineMemOperand::Flags MMOFlags,
11172 const MMOMetadata &Metadata) {
11174 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
11175 PtrInfo, VT, Alignment, MMOFlags, Metadata);
11176}
11177
11179 SDValue Ptr, MachineMemOperand *MMO) {
11181 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
11182 VT, MMO);
11183}
11184
11186 EVT VT, SDValue Chain, SDValue Ptr,
11187 MachinePointerInfo PtrInfo, EVT MemVT,
11188 MaybeAlign Alignment,
11189 MachineMemOperand::Flags MMOFlags,
11190 const MMOMetadata &Metadata) {
11192 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
11193 MemVT, Alignment, MMOFlags, Metadata);
11194}
11195
11197 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
11198 MachineMemOperand *MMO) {
11200 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
11201 MemVT, MMO);
11202}
11203
11207 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
11208 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11209 "Load is already a indexed load!");
11210 // Don't propagate the invariant or dereferenceable flags.
11211 auto MMOFlags =
11212 LD->getMemOperand()->getFlags() &
11214 return getLoad(
11215 AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, LD->getChain(),
11216 Base, Offset, LD->getPointerInfo(), LD->getMemoryVT(), LD->getAlign(),
11217 MMOFlags,
11218 MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
11219}
11220
11222 SDValue Ptr, MachinePointerInfo PtrInfo,
11223 Align Alignment,
11224 MachineMemOperand::Flags MMOFlags,
11225 const MMOMetadata &Metadata) {
11226 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11227
11228 MMOFlags |= MachineMemOperand::MOStore;
11229 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11230 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11231
11232 if (PtrInfo.V.isNull())
11233 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
11234
11237 MachineMemOperand *MMO =
11238 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, Metadata);
11239 return getStore(Chain, dl, Val, Ptr, MMO);
11240}
11241
11243 SDValue Ptr, MachineMemOperand *MMO) {
11245 return getStore(Chain, dl, Val, Ptr, Undef, Val.getValueType(), MMO,
11247}
11248
11250 SDValue Ptr, SDValue Offset, EVT SVT,
11252 bool IsTruncating) {
11253 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11254 EVT VT = Val.getValueType();
11255 if (VT == SVT) {
11256 IsTruncating = false;
11257 } else if (!IsTruncating) {
11258 assert(VT == SVT && "No-truncating store from different memory type!");
11259 } else {
11261 "Should only be a truncating store, not extending!");
11262 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11263 assert(VT.isVector() == SVT.isVector() &&
11264 "Cannot use trunc store to convert to or from a vector!");
11265 assert((!VT.isVector() ||
11267 "Cannot use trunc store to change the number of vector elements!");
11268 }
11269
11270 bool Indexed = AM != ISD::UNINDEXED;
11271 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11272 "Unindexed store with an offset!");
11273 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
11274 : getVTList(MVT::Other);
11275 SDValue Ops[] = {Chain, Val, Ptr, Offset};
11276 SDNodeKey ID(ISD::STORE, VTs, Ops);
11277 ID.AddInteger(SVT.getRawBits());
11278 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
11279 dl.getIROrder(), VTs, AM, IsTruncating, SVT, MMO));
11280 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11281 ID.AddInteger(MMO->getFlags());
11282 FoldingSetInsertToken InsertToken;
11283 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11284 cast<StoreSDNode>(E)->refineAlignment(MMO);
11285 cast<StoreSDNode>(E)->refineMMOMetadata(MMO);
11286 return SDValue(E, 0);
11287 }
11288 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
11289 IsTruncating, SVT, MMO);
11290 createOperands(N, Ops);
11291
11292 CSEMap.insert(N, InsertToken);
11293 InsertNode(N);
11294 SDValue V(N, 0);
11295 NewSDValueDbgMsg(V, "Creating new node: ", this);
11296 return V;
11297}
11298
11300 SDValue Ptr, SDValue Offset,
11301 MachinePointerInfo PtrInfo, EVT SVT,
11302 Align Alignment,
11303 MachineMemOperand::Flags MMOFlags,
11304 const MMOMetadata &Metadata) {
11305 assert(Chain.getValueType() == MVT::Other &&
11306 "Invalid chain type");
11307
11308 MMOFlags |= MachineMemOperand::MOStore;
11309 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11310 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11311
11312 if (PtrInfo.V.isNull())
11313 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
11314
11316 MachineMemOperand *MMO = MF.getMachineMemOperand(
11317 PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, Metadata);
11318 return getTruncStore(Chain, dl, Val, Ptr, Offset, SVT, MMO);
11319}
11320
11322 SDValue Ptr, MachinePointerInfo PtrInfo,
11323 EVT SVT, Align Alignment,
11324 MachineMemOperand::Flags MMOFlags,
11325 const MMOMetadata &Metadata) {
11326 return getTruncStore(Chain, dl, Val, Ptr, getPOISON(Ptr.getValueType()),
11327 PtrInfo, SVT, Alignment, MMOFlags, Metadata);
11328}
11329
11331 SDValue Ptr, SDValue Offset, EVT SVT,
11332 MachineMemOperand *MMO) {
11333 return getStore(Chain, dl, Val, Ptr, Offset, SVT, MMO, ISD::UNINDEXED, true);
11334}
11335
11337 SDValue Ptr, EVT SVT,
11338 MachineMemOperand *MMO) {
11339 return getStore(Chain, dl, Val, Ptr, getPOISON(Ptr.getValueType()), SVT, MMO,
11340 ISD::UNINDEXED, true);
11341}
11342
11346 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
11347 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11348 "Store is already a indexed store!");
11349 return getStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
11350 ST->getMemoryVT(), ST->getMemOperand(), AM,
11351 ST->isTruncatingStore());
11352}
11353
11355 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
11356 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
11357 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
11358 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
11359 const MDNode *Ranges, bool IsExpanding) {
11360 MMOFlags |= MachineMemOperand::MOLoad;
11361 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11362 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11363 // clients.
11364 if (PtrInfo.V.isNull())
11365 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
11366
11367 TypeSize Size = MemVT.getStoreSize();
11369 MachineMemOperand *MMO = MF.getMachineMemOperand(
11370 PtrInfo, MMOFlags, Size, Alignment, MMOMetadata(AAInfo, Ranges));
11371 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
11372 MMO, IsExpanding);
11373}
11374
11376 ISD::LoadExtType ExtType, EVT VT,
11377 const SDLoc &dl, SDValue Chain, SDValue Ptr,
11378 SDValue Offset, SDValue Mask, SDValue EVL,
11379 EVT MemVT, MachineMemOperand *MMO,
11380 bool IsExpanding) {
11381 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11382 assert(Mask.getValueType().getVectorElementCount() ==
11383 VT.getVectorElementCount() &&
11384 "Vector width mismatch between mask and data");
11385
11386 bool Indexed = AM != ISD::UNINDEXED;
11387 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11388 "Unindexed load with an offset!");
11389
11390 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
11391 : getVTList(VT, MVT::Other);
11392 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
11393 SDNodeKey ID(ISD::VP_LOAD, VTs, Ops);
11394 ID.AddInteger(MemVT.getRawBits());
11395 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
11396 dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
11397 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11398 ID.AddInteger(MMO->getFlags());
11399 FoldingSetInsertToken InsertToken;
11400 if (auto *E = cast_or_null<VPLoadSDNode>(lookupNode(ID, dl, InsertToken))) {
11401 E->refineAlignment(MMO);
11402 E->refineMMOMetadata(MMO);
11403 return SDValue(E, 0);
11404 }
11405 auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
11406 ExtType, IsExpanding, MemVT, MMO);
11407 createOperands(N, Ops);
11408
11409 CSEMap.insert(N, InsertToken);
11410 InsertNode(N);
11411 SDValue V(N, 0);
11412 NewSDValueDbgMsg(V, "Creating new node: ", this);
11413 return V;
11414}
11415
11417 SDValue Ptr, SDValue Mask, SDValue EVL,
11418 MachinePointerInfo PtrInfo,
11419 MaybeAlign Alignment,
11420 MachineMemOperand::Flags MMOFlags,
11421 const AAMDNodes &AAInfo, const MDNode *Ranges,
11422 bool IsExpanding) {
11424 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
11425 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
11426 IsExpanding);
11427}
11428
11430 SDValue Ptr, SDValue Mask, SDValue EVL,
11431 MachineMemOperand *MMO, bool IsExpanding) {
11433 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
11434 Mask, EVL, VT, MMO, IsExpanding);
11435}
11436
11438 EVT VT, SDValue Chain, SDValue Ptr,
11439 SDValue Mask, SDValue EVL,
11440 MachinePointerInfo PtrInfo, EVT MemVT,
11441 MaybeAlign Alignment,
11442 MachineMemOperand::Flags MMOFlags,
11443 const AAMDNodes &AAInfo, bool IsExpanding) {
11445 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
11446 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,
11447 IsExpanding);
11448}
11449
11451 EVT VT, SDValue Chain, SDValue Ptr,
11452 SDValue Mask, SDValue EVL, EVT MemVT,
11453 MachineMemOperand *MMO, bool IsExpanding) {
11455 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
11456 EVL, MemVT, MMO, IsExpanding);
11457}
11458
11460 SDValue Ptr, SDValue Offset, SDValue Mask,
11461 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
11462 ISD::MemIndexedMode AM, bool IsTruncating,
11463 bool IsCompressing) {
11464 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11465 assert(Mask.getValueType().getVectorElementCount() ==
11467 "Vector width mismatch between mask and data");
11468
11469 bool Indexed = AM != ISD::UNINDEXED;
11470 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11471 "Unindexed vp_store with an offset!");
11472 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
11473 : getVTList(MVT::Other);
11474 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
11475 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11476 ID.AddInteger(MemVT.getRawBits());
11477 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
11478 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
11479 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11480 ID.AddInteger(MMO->getFlags());
11481 FoldingSetInsertToken InsertToken;
11482 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11483 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
11484 return SDValue(E, 0);
11485 }
11486 auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
11487 IsTruncating, IsCompressing, MemVT, MMO);
11488 createOperands(N, Ops);
11489
11490 CSEMap.insert(N, InsertToken);
11491 InsertNode(N);
11492 SDValue V(N, 0);
11493 NewSDValueDbgMsg(V, "Creating new node: ", this);
11494 return V;
11495}
11496
11498 SDValue Val, SDValue Ptr, SDValue Mask,
11499 SDValue EVL, MachinePointerInfo PtrInfo,
11500 EVT SVT, Align Alignment,
11501 MachineMemOperand::Flags MMOFlags,
11502 const AAMDNodes &AAInfo,
11503 bool IsCompressing) {
11504 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11505
11506 MMOFlags |= MachineMemOperand::MOStore;
11507 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11508
11509 if (PtrInfo.V.isNull())
11510 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
11511
11513 MachineMemOperand *MMO = MF.getMachineMemOperand(
11514 PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
11515 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
11516 IsCompressing);
11517}
11518
11520 SDValue Val, SDValue Ptr, SDValue Mask,
11521 SDValue EVL, EVT SVT,
11522 MachineMemOperand *MMO,
11523 bool IsCompressing) {
11524 EVT VT = Val.getValueType();
11525
11526 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11527 if (VT == SVT)
11528 return getStoreVP(Chain, dl, Val, Ptr, getPOISON(Ptr.getValueType()), Mask,
11529 EVL, VT, MMO, ISD::UNINDEXED,
11530 /*IsTruncating*/ false, IsCompressing);
11531
11533 "Should only be a truncating store, not extending!");
11534 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11535 assert(VT.isVector() == SVT.isVector() &&
11536 "Cannot use trunc store to convert to or from a vector!");
11537 assert((!VT.isVector() ||
11539 "Cannot use trunc store to change the number of vector elements!");
11540
11541 SDVTList VTs = getVTList(MVT::Other);
11543 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11544 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11545 ID.AddInteger(SVT.getRawBits());
11546 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
11547 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
11548 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11549 ID.AddInteger(MMO->getFlags());
11550 FoldingSetInsertToken InsertToken;
11551 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11552 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
11553 return SDValue(E, 0);
11554 }
11555 auto *N =
11556 newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
11557 ISD::UNINDEXED, true, IsCompressing, SVT, MMO);
11558 createOperands(N, Ops);
11559
11560 CSEMap.insert(N, InsertToken);
11561 InsertNode(N);
11562 SDValue V(N, 0);
11563 NewSDValueDbgMsg(V, "Creating new node: ", this);
11564 return V;
11565}
11566
11568 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11569 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11570 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11571 bool Indexed = AM != ISD::UNINDEXED;
11572 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11573 "Unindexed load with an offset!");
11574
11575 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11576 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
11577 : getVTList(VT, MVT::Other);
11578 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
11579 ID.AddInteger(MemVT.getRawBits());
11580 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11581 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
11582 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11583
11584 FoldingSetInsertToken InsertToken;
11585 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11586 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
11587 return SDValue(E, 0);
11588 }
11589
11590 auto *N =
11591 newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,
11592 ExtType, IsExpanding, MemVT, MMO);
11593 createOperands(N, Ops);
11594 CSEMap.insert(N, InsertToken);
11595 InsertNode(N);
11596 SDValue V(N, 0);
11597 NewSDValueDbgMsg(V, "Creating new node: ", this);
11598 return V;
11599}
11600
11602 SDValue Ptr, SDValue Stride,
11603 SDValue Mask, SDValue EVL,
11604 MachineMemOperand *MMO,
11605 bool IsExpanding) {
11607 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11608 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
11609}
11610
11612 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11613 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11614 MachineMemOperand *MMO, bool IsExpanding) {
11616 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
11617 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11618}
11619
11621 SDValue Val, SDValue Ptr,
11622 SDValue Offset, SDValue Stride,
11623 SDValue Mask, SDValue EVL, EVT MemVT,
11624 MachineMemOperand *MMO,
11626 bool IsTruncating, bool IsCompressing) {
11627 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11628 bool Indexed = AM != ISD::UNINDEXED;
11629 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11630 "Unindexed vp_store with an offset!");
11631 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
11632 : getVTList(MVT::Other);
11633 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11634 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
11635 ID.AddInteger(MemVT.getRawBits());
11636 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11637 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
11638 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11639 FoldingSetInsertToken InsertToken;
11640 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11641 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
11642 return SDValue(E, 0);
11643 }
11644 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
11645 VTs, AM, IsTruncating,
11646 IsCompressing, MemVT, MMO);
11647 createOperands(N, Ops);
11648
11649 CSEMap.insert(N, InsertToken);
11650 InsertNode(N);
11651 SDValue V(N, 0);
11652 NewSDValueDbgMsg(V, "Creating new node: ", this);
11653 return V;
11654}
11655
11658 ISD::MemIndexType IndexType) {
11659 assert(Ops.size() == 6 && "Incompatible number of operands");
11660
11661 SDNodeKey ID(ISD::VP_GATHER, VTs, Ops);
11662 ID.AddInteger(VT.getRawBits());
11663 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
11664 dl.getIROrder(), VTs, VT, MMO, IndexType));
11665 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11666 ID.AddInteger(MMO->getFlags());
11667 FoldingSetInsertToken InsertToken;
11668 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11669 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
11670 return SDValue(E, 0);
11671 }
11672
11673 auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
11674 VT, MMO, IndexType);
11675 createOperands(N, Ops);
11676
11677 assert(N->getMask().getValueType().getVectorElementCount() ==
11678 N->getValueType(0).getVectorElementCount() &&
11679 "Vector width mismatch between mask and data");
11680 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11681 N->getValueType(0).getVectorElementCount().isScalable() &&
11682 "Scalable flags of index and data do not match");
11684 N->getIndex().getValueType().getVectorElementCount(),
11685 N->getValueType(0).getVectorElementCount()) &&
11686 "Vector width mismatch between index and data");
11687 assert(isa<ConstantSDNode>(N->getScale()) &&
11688 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11689 "Scale should be a constant power of 2");
11690
11691 CSEMap.insert(N, InsertToken);
11692 InsertNode(N);
11693 SDValue V(N, 0);
11694 NewSDValueDbgMsg(V, "Creating new node: ", this);
11695 return V;
11696}
11697
11700 MachineMemOperand *MMO,
11701 ISD::MemIndexType IndexType) {
11702 assert(Ops.size() == 7 && "Incompatible number of operands");
11703
11704 SDNodeKey ID(ISD::VP_SCATTER, VTs, Ops);
11705 ID.AddInteger(VT.getRawBits());
11706 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
11707 dl.getIROrder(), VTs, VT, MMO, IndexType));
11708 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11709 ID.AddInteger(MMO->getFlags());
11710 FoldingSetInsertToken InsertToken;
11711 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11712 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
11713 return SDValue(E, 0);
11714 }
11715 auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
11716 VT, MMO, IndexType);
11717 createOperands(N, Ops);
11718
11719 assert(N->getMask().getValueType().getVectorElementCount() ==
11720 N->getValue().getValueType().getVectorElementCount() &&
11721 "Vector width mismatch between mask and data");
11722 assert(
11723 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11724 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11725 "Scalable flags of index and data do not match");
11727 N->getIndex().getValueType().getVectorElementCount(),
11728 N->getValue().getValueType().getVectorElementCount()) &&
11729 "Vector width mismatch between index and data");
11730 assert(isa<ConstantSDNode>(N->getScale()) &&
11731 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11732 "Scale should be a constant power of 2");
11733
11734 CSEMap.insert(N, InsertToken);
11735 InsertNode(N);
11736 SDValue V(N, 0);
11737 NewSDValueDbgMsg(V, "Creating new node: ", this);
11738 return V;
11739}
11740
11743 SDValue PassThru, EVT MemVT,
11744 MachineMemOperand *MMO,
11746 ISD::LoadExtType ExtTy, bool isExpanding) {
11747 bool Indexed = AM != ISD::UNINDEXED;
11748 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11749 "Unindexed masked load with an offset!");
11750 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
11751 : getVTList(VT, MVT::Other);
11752 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11753 SDNodeKey ID(ISD::MLOAD, VTs, Ops);
11754 ID.AddInteger(MemVT.getRawBits());
11755 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11756 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
11757 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11758 ID.AddInteger(MMO->getFlags());
11759 FoldingSetInsertToken InsertToken;
11760 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11761 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
11762 return SDValue(E, 0);
11763 }
11764 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
11765 AM, ExtTy, isExpanding, MemVT, MMO);
11766 createOperands(N, Ops);
11767
11768 CSEMap.insert(N, InsertToken);
11769 InsertNode(N);
11770 SDValue V(N, 0);
11771 NewSDValueDbgMsg(V, "Creating new node: ", this);
11772 return V;
11773}
11774
11779 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11780 "Masked load is already a indexed load!");
11781 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
11782 Offset, LD->getMask(), LD->getPassThru(),
11783 LD->getMemoryVT(), LD->getMemOperand(), AM,
11784 LD->getExtensionType(), LD->isExpandingLoad());
11785}
11786
11789 SDValue Mask, EVT MemVT,
11790 MachineMemOperand *MMO,
11791 ISD::MemIndexedMode AM, bool IsTruncating,
11792 bool IsCompressing) {
11793 assert(Chain.getValueType() == MVT::Other &&
11794 "Invalid chain type");
11795 bool Indexed = AM != ISD::UNINDEXED;
11796 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11797 "Unindexed masked store with an offset!");
11798 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
11799 : getVTList(MVT::Other);
11800 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11801 SDNodeKey ID(ISD::MSTORE, VTs, Ops);
11802 ID.AddInteger(MemVT.getRawBits());
11803 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11804 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
11805 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11806 ID.AddInteger(MMO->getFlags());
11807 FoldingSetInsertToken InsertToken;
11808 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11809 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
11810 return SDValue(E, 0);
11811 }
11812 auto *N =
11813 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
11814 IsTruncating, IsCompressing, MemVT, MMO);
11815 createOperands(N, Ops);
11816
11817 CSEMap.insert(N, InsertToken);
11818 InsertNode(N);
11819 SDValue V(N, 0);
11820 NewSDValueDbgMsg(V, "Creating new node: ", this);
11821 return V;
11822}
11823
11828 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11829 "Masked store is already a indexed store!");
11830 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
11831 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
11832 AM, ST->isTruncatingStore(), ST->isCompressingStore());
11833}
11834
11837 MachineMemOperand *MMO,
11838 ISD::MemIndexType IndexType,
11839 ISD::LoadExtType ExtTy) {
11840 assert(Ops.size() == 6 && "Incompatible number of operands");
11841
11842 SDNodeKey ID(ISD::MGATHER, VTs, Ops);
11843 ID.AddInteger(MemVT.getRawBits());
11844 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11845 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
11846 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11847 ID.AddInteger(MMO->getFlags());
11848 FoldingSetInsertToken InsertToken;
11849 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11850 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
11851 return SDValue(E, 0);
11852 }
11853
11854 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
11855 VTs, MemVT, MMO, IndexType, ExtTy);
11856 createOperands(N, Ops);
11857
11858 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11859 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11860 assert(N->getMask().getValueType().getVectorElementCount() ==
11861 N->getValueType(0).getVectorElementCount() &&
11862 "Vector width mismatch between mask and data");
11863 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11864 N->getValueType(0).getVectorElementCount().isScalable() &&
11865 "Scalable flags of index and data do not match");
11867 N->getIndex().getValueType().getVectorElementCount(),
11868 N->getValueType(0).getVectorElementCount()) &&
11869 "Vector width mismatch between index and data");
11870 assert(isa<ConstantSDNode>(N->getScale()) &&
11871 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11872 "Scale should be a constant power of 2");
11873
11874 CSEMap.insert(N, InsertToken);
11875 InsertNode(N);
11876 SDValue V(N, 0);
11877 NewSDValueDbgMsg(V, "Creating new node: ", this);
11878 return V;
11879}
11880
11883 MachineMemOperand *MMO,
11884 ISD::MemIndexType IndexType,
11885 bool IsTrunc) {
11886 assert(Ops.size() == 6 && "Incompatible number of operands");
11887
11888 SDNodeKey ID(ISD::MSCATTER, VTs, Ops);
11889 ID.AddInteger(MemVT.getRawBits());
11890 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11891 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
11892 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11893 ID.AddInteger(MMO->getFlags());
11894 FoldingSetInsertToken InsertToken;
11895 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11896 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
11897 return SDValue(E, 0);
11898 }
11899
11900 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
11901 VTs, MemVT, MMO, IndexType, IsTrunc);
11902 createOperands(N, Ops);
11903
11904 assert(N->getMask().getValueType().getVectorElementCount() ==
11905 N->getValue().getValueType().getVectorElementCount() &&
11906 "Vector width mismatch between mask and data");
11907 assert(
11908 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11909 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11910 "Scalable flags of index and data do not match");
11912 N->getIndex().getValueType().getVectorElementCount(),
11913 N->getValue().getValueType().getVectorElementCount()) &&
11914 "Vector width mismatch between index and data");
11915 assert(isa<ConstantSDNode>(N->getScale()) &&
11916 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11917 "Scale should be a constant power of 2");
11918
11919 CSEMap.insert(N, InsertToken);
11920 InsertNode(N);
11921 SDValue V(N, 0);
11922 NewSDValueDbgMsg(V, "Creating new node: ", this);
11923 return V;
11924}
11925
11927 const SDLoc &dl, ArrayRef<SDValue> Ops,
11928 MachineMemOperand *MMO,
11929 ISD::MemIndexType IndexType) {
11930 assert(Ops.size() == 7 && "Incompatible number of operands");
11931
11933 ID.AddInteger(MemVT.getRawBits());
11934 ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11935 dl.getIROrder(), VTs, MemVT, MMO, IndexType));
11936 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11937 ID.AddInteger(MMO->getFlags());
11938 FoldingSetInsertToken InsertToken;
11939 if (SDNode *E = lookupNode(ID, dl, InsertToken)) {
11940 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
11941 return SDValue(E, 0);
11942 }
11943
11944 auto *N = newSDNode<MaskedHistogramSDNode>(dl.getIROrder(), dl.getDebugLoc(),
11945 VTs, MemVT, MMO, IndexType);
11946 createOperands(N, Ops);
11947
11948 assert(N->getMask().getValueType().getVectorElementCount() ==
11949 N->getIndex().getValueType().getVectorElementCount() &&
11950 "Vector width mismatch between mask and data");
11951 assert(isa<ConstantSDNode>(N->getScale()) &&
11952 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11953 "Scale should be a constant power of 2");
11954 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11955
11956 CSEMap.insert(N, InsertToken);
11957 InsertNode(N);
11958 SDValue V(N, 0);
11959 NewSDValueDbgMsg(V, "Creating new node: ", this);
11960 return V;
11961}
11962
11964 SDValue Ptr, SDValue Mask, SDValue EVL,
11965 MachineMemOperand *MMO) {
11966 SDVTList VTs = getVTList(VT, EVL.getValueType(), MVT::Other);
11967 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11968 SDNodeKey ID(ISD::VP_LOAD_FF, VTs, Ops);
11969 ID.AddInteger(VT.getRawBits());
11970 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadFFSDNode>(DL.getIROrder(),
11971 VTs, VT, MMO));
11972 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
11973 ID.AddInteger(MMO->getFlags());
11974 FoldingSetInsertToken InsertToken;
11975 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11976 cast<VPLoadFFSDNode>(E)->refineAlignment(MMO);
11977 return SDValue(E, 0);
11978 }
11979 auto *N = newSDNode<VPLoadFFSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs,
11980 VT, MMO);
11981 createOperands(N, Ops);
11982
11983 CSEMap.insert(N, InsertToken);
11984 InsertNode(N);
11985 SDValue V(N, 0);
11986 NewSDValueDbgMsg(V, "Creating new node: ", this);
11987 return V;
11988}
11989
11991 EVT MemVT, MachineMemOperand *MMO) {
11992 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11993 SDVTList VTs = getVTList(MVT::Other);
11994 SDValue Ops[] = {Chain, Ptr};
11996 ID.AddInteger(MemVT.getRawBits());
11997 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11998 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
11999 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
12000 ID.AddInteger(MMO->getFlags());
12001 FoldingSetInsertToken InsertToken;
12002 if (SDNode *E = lookupNode(ID, dl, InsertToken))
12003 return SDValue(E, 0);
12004
12005 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),
12006 dl.getDebugLoc(), VTs, MemVT, MMO);
12007 createOperands(N, Ops);
12008
12009 CSEMap.insert(N, InsertToken);
12010 InsertNode(N);
12011 SDValue V(N, 0);
12012 NewSDValueDbgMsg(V, "Creating new node: ", this);
12013 return V;
12014}
12015
12017 EVT MemVT, MachineMemOperand *MMO) {
12018 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
12019 SDVTList VTs = getVTList(MVT::Other);
12020 SDValue Ops[] = {Chain, Ptr};
12022 ID.AddInteger(MemVT.getRawBits());
12023 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
12024 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
12025 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
12026 ID.AddInteger(MMO->getFlags());
12027 FoldingSetInsertToken InsertToken;
12028 if (SDNode *E = lookupNode(ID, dl, InsertToken))
12029 return SDValue(E, 0);
12030
12031 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),
12032 dl.getDebugLoc(), VTs, MemVT, MMO);
12033 createOperands(N, Ops);
12034
12035 CSEMap.insert(N, InsertToken);
12036 InsertNode(N);
12037 SDValue V(N, 0);
12038 NewSDValueDbgMsg(V, "Creating new node: ", this);
12039 return V;
12040}
12041
12043 // select undef, T, F --> T (if T is a constant), otherwise F
12044 // select, ?, undef, F --> F
12045 // select, ?, T, undef --> T
12046 if (Cond.isUndef())
12047 return isConstantValueOfAnyType(T) ? T : F;
12048 if (T.isUndef())
12050 if (F.isUndef())
12052
12053 // select true, T, F --> T
12054 // select false, T, F --> F
12055 if (auto C = isBoolConstant(Cond))
12056 return *C ? T : F;
12057
12058 // select ?, T, T --> T
12059 if (T == F)
12060 return T;
12061
12062 return SDValue();
12063}
12064
12066 // shift undef, Y --> 0 (can always assume that the undef value is 0)
12067 if (X.isUndef())
12068 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
12069 // shift X, undef --> undef (because it may shift by the bitwidth)
12070 if (Y.isUndef())
12071 return getUNDEF(X.getValueType());
12072
12073 // shift 0, Y --> 0
12074 // shift X, 0 --> X
12076 return X;
12077
12078 // shift X, C >= bitwidth(X) --> undef
12079 // All vector elements must be too big (or undef) to avoid partial undefs.
12080 auto isShiftTooBig = [X](ConstantSDNode *Val) {
12081 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
12082 };
12083 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
12084 return getUNDEF(X.getValueType());
12085
12086 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
12087 if (X.getValueType().getScalarType() == MVT::i1)
12088 return X;
12089
12090 return SDValue();
12091}
12092
12094 SDNodeFlags Flags) {
12095 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
12096 // (an undef operand can be chosen to be Nan/Inf), then the result of this
12097 // operation is poison. That result can be relaxed to undef.
12098 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
12099 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
12100 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
12101 (YC && YC->getValueAPF().isNaN());
12102 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
12103 (YC && YC->getValueAPF().isInfinity());
12104
12105 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
12106 return getUNDEF(X.getValueType());
12107
12108 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
12109 return getUNDEF(X.getValueType());
12110
12111 if (!YC)
12112 return SDValue();
12113
12114 // X + -0.0 --> X
12115 if (Opcode == ISD::FADD)
12116 if (YC->getValueAPF().isNegZero())
12117 return X;
12118
12119 // X - +0.0 --> X
12120 if (Opcode == ISD::FSUB)
12121 if (YC->getValueAPF().isPosZero())
12122 return X;
12123
12124 // X * 1.0 --> X
12125 // X / 1.0 --> X
12126 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
12127 if (YC->getValueAPF().isOne())
12128 return X;
12129
12130 // X * 0.0 --> 0.0
12131 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
12132 if (YC->getValueAPF().isZero())
12133 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
12134
12135 return SDValue();
12136}
12137
12139 SDValue Ptr, SDValue SV, unsigned Align) {
12140 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
12141 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
12142}
12143
12144SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12146 switch (Ops.size()) {
12147 case 0: return getNode(Opcode, DL, VT);
12148 case 1: return getNode(Opcode, DL, VT, Ops[0].get());
12149 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
12150 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
12151 default: break;
12152 }
12153
12154 // Copy from an SDUse array into an SDValue array for use with
12155 // the regular getNode logic.
12157 return getNode(Opcode, DL, VT, NewOps);
12158}
12159
12160SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12162 SDNodeFlags Flags;
12163 if (Inserter)
12164 Flags = Inserter->getFlags();
12165 return getNode(Opcode, DL, VT, Ops, Flags);
12166}
12167
12168SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12169 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12170 unsigned NumOps = Ops.size();
12171 switch (NumOps) {
12172 case 0: return getNode(Opcode, DL, VT);
12173 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
12174 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
12175 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
12176 default: break;
12177 }
12178
12179#ifndef NDEBUG
12180 for (const auto &Op : Ops)
12181 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12182 "Operand is DELETED_NODE!");
12183#endif
12184
12185 switch (Opcode) {
12186 default: break;
12187 case ISD::BUILD_VECTOR:
12188 // Attempt to simplify BUILD_VECTOR.
12189 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
12190 return V;
12191 break;
12193 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
12194 return V;
12195 break;
12196 case ISD::SELECT_CC:
12197 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
12198 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
12199 "LHS and RHS of condition must have same type!");
12200 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
12201 "True and False arms of SelectCC must have same type!");
12202 assert(Ops[2].getValueType() == VT &&
12203 "select_cc node must be of same type as true and false value!");
12204 assert((!Ops[0].getValueType().isVector() ||
12205 Ops[0].getValueType().getVectorElementCount() ==
12206 VT.getVectorElementCount()) &&
12207 "Expected select_cc with vector result to have the same sized "
12208 "comparison type!");
12209 break;
12210 case ISD::BR_CC:
12211 assert(NumOps == 5 && "BR_CC takes 5 operands!");
12212 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
12213 "LHS/RHS of comparison should match types!");
12214 break;
12215 case ISD::VP_REDUCE_MUL:
12216 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
12217 if (VT == MVT::i1)
12218 Opcode = ISD::VP_REDUCE_AND;
12219 break;
12220 case ISD::VP_REDUCE_ADD:
12221 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
12222 if (VT == MVT::i1)
12223 Opcode = ISD::VP_REDUCE_XOR;
12224 break;
12225 case ISD::VP_REDUCE_SMAX:
12226 case ISD::VP_REDUCE_UMIN:
12227 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
12228 // VP_REDUCE_AND.
12229 if (VT == MVT::i1)
12230 Opcode = ISD::VP_REDUCE_AND;
12231 break;
12232 case ISD::VP_REDUCE_SMIN:
12233 case ISD::VP_REDUCE_UMAX:
12234 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
12235 // VP_REDUCE_OR.
12236 if (VT == MVT::i1)
12237 Opcode = ISD::VP_REDUCE_OR;
12238 break;
12239 }
12240
12241 // Memoize nodes.
12242 SDNode *N;
12243 SDVTList VTs = getVTList(VT);
12244
12245 if (VT != MVT::Glue) {
12246 SDNodeKey ID(Opcode, VTs, Ops);
12247 FoldingSetInsertToken InsertToken;
12248
12249 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
12250 E->intersectFlagsWith(Flags);
12251 return SDValue(E, 0);
12252 }
12253
12254 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
12255 createOperands(N, Ops);
12256
12257 CSEMap.insert(N, InsertToken);
12258 } else {
12259 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
12260 createOperands(N, Ops);
12261 }
12262
12263 N->setFlags(Flags);
12264 InsertNode(N);
12265 SDValue V(N, 0);
12266 NewSDValueDbgMsg(V, "Creating new node: ", this);
12267 return V;
12268}
12269
12270SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12271 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
12272 SDNodeFlags Flags;
12273 if (Inserter)
12274 Flags = Inserter->getFlags();
12275 return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags);
12276}
12277
12278SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12280 const SDNodeFlags Flags) {
12281 return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags);
12282}
12283
12284SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12286 SDNodeFlags Flags;
12287 if (Inserter)
12288 Flags = Inserter->getFlags();
12289 return getNode(Opcode, DL, VTList, Ops, Flags);
12290}
12291
12292SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12293 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12294 if (VTList.NumVTs == 1)
12295 return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
12296
12297#ifndef NDEBUG
12298 for (const auto &Op : Ops)
12299 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12300 "Operand is DELETED_NODE!");
12301#endif
12302
12303 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VTList, Ops, Flags))
12304 return V;
12305
12306 switch (Opcode) {
12307 case ISD::SADDO:
12308 case ISD::UADDO:
12309 case ISD::SSUBO:
12310 case ISD::USUBO: {
12311 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12312 "Invalid add/sub overflow op!");
12313 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12314 Ops[0].getValueType() == Ops[1].getValueType() &&
12315 Ops[0].getValueType() == VTList.VTs[0] &&
12316 "Binary operator types must match!");
12317 SDValue N1 = Ops[0], N2 = Ops[1];
12318 canonicalizeCommutativeBinop(Opcode, N1, N2);
12319
12320 // (X +- 0) -> X with zero-overflow.
12321 ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,
12322 /*AllowTruncation*/ true);
12323 if (N2CV && N2CV->isZero()) {
12324 SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
12325 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
12326 }
12327
12328 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
12329 VTList.VTs[1].getScalarType() == MVT::i1) {
12330 SDValue F1 = getFreeze(N1);
12331 SDValue F2 = getFreeze(N2);
12332 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
12333 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
12334 return getNode(ISD::MERGE_VALUES, DL, VTList,
12335 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
12336 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
12337 Flags);
12338 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
12339 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
12340 SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
12341 return getNode(ISD::MERGE_VALUES, DL, VTList,
12342 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
12343 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
12344 Flags);
12345 }
12346 }
12347 break;
12348 }
12349 case ISD::SADDO_CARRY:
12350 case ISD::UADDO_CARRY:
12351 case ISD::SSUBO_CARRY:
12352 case ISD::USUBO_CARRY:
12353 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
12354 "Invalid add/sub overflow op!");
12355 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12356 Ops[0].getValueType() == Ops[1].getValueType() &&
12357 Ops[0].getValueType() == VTList.VTs[0] &&
12358 Ops[2].getValueType() == VTList.VTs[1] &&
12359 "Binary operator types must match!");
12360 break;
12361 case ISD::SMUL_LOHI:
12362 case ISD::UMUL_LOHI: {
12363 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
12364 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
12365 VTList.VTs[0] == Ops[0].getValueType() &&
12366 VTList.VTs[0] == Ops[1].getValueType() &&
12367 "Binary operator types must match!");
12368 break;
12369 }
12370 case ISD::FFREXP: {
12371 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
12372 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
12373 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
12374
12376 int FrexpExp;
12377 APFloat FrexpMant =
12378 frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);
12379 SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);
12380 SDValue Result1 = getSignedConstant(FrexpMant.isFinite() ? FrexpExp : 0,
12381 DL, VTList.VTs[1]);
12382 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);
12383 }
12384
12385 break;
12386 }
12388 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12389 "Invalid STRICT_FP_EXTEND!");
12390 assert(VTList.VTs[0].isFloatingPoint() &&
12391 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
12392 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12393 "STRICT_FP_EXTEND result type should be vector iff the operand "
12394 "type is vector!");
12395 assert((!VTList.VTs[0].isVector() ||
12396 VTList.VTs[0].getVectorElementCount() ==
12397 Ops[1].getValueType().getVectorElementCount()) &&
12398 "Vector element count mismatch!");
12399 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
12400 "Invalid fpext node, dst <= src!");
12401 break;
12403 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12404 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12405 "STRICT_FP_ROUND result type should be vector iff the operand "
12406 "type is vector!");
12407 assert((!VTList.VTs[0].isVector() ||
12408 VTList.VTs[0].getVectorElementCount() ==
12409 Ops[1].getValueType().getVectorElementCount()) &&
12410 "Vector element count mismatch!");
12411 assert(VTList.VTs[0].isFloatingPoint() &&
12412 Ops[1].getValueType().isFloatingPoint() &&
12413 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12414 Ops[2].getOpcode() == ISD::TargetConstant &&
12415 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12416 "Invalid STRICT_FP_ROUND!");
12417 break;
12418 }
12419
12420 // Memoize the node unless it returns a glue result.
12421 SDNode *N;
12422 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12423 SDNodeKey ID(Opcode, VTList, Ops);
12424 FoldingSetInsertToken InsertToken;
12425 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
12426 E->intersectFlagsWith(Flags);
12427 return SDValue(E, 0);
12428 }
12429
12430 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
12431 createOperands(N, Ops);
12432 CSEMap.insert(N, InsertToken);
12433 } else {
12434 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
12435 createOperands(N, Ops);
12436 }
12437
12438 N->setFlags(Flags);
12439 InsertNode(N);
12440 SDValue V(N, 0);
12441 NewSDValueDbgMsg(V, "Creating new node: ", this);
12442 return V;
12443}
12444
12445SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12446 SDVTList VTList) {
12447 return getNode(Opcode, DL, VTList, ArrayRef<SDValue>());
12448}
12449
12450SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12451 SDValue N1) {
12452 SDValue Ops[] = { N1 };
12453 return getNode(Opcode, DL, VTList, Ops);
12454}
12455
12456SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12457 SDValue N1, SDValue N2) {
12458 SDValue Ops[] = { N1, N2 };
12459 return getNode(Opcode, DL, VTList, Ops);
12460}
12461
12462SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12463 SDValue N1, SDValue N2, SDValue N3) {
12464 SDValue Ops[] = { N1, N2, N3 };
12465 return getNode(Opcode, DL, VTList, Ops);
12466}
12467
12468SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12469 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12470 SDValue Ops[] = { N1, N2, N3, N4 };
12471 return getNode(Opcode, DL, VTList, Ops);
12472}
12473
12474SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12475 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12476 SDValue N5) {
12477 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12478 return getNode(Opcode, DL, VTList, Ops);
12479}
12480
12482 if (!VT.isExtended())
12483 return makeVTList(SDNode::getValueTypeList(VT.getSimpleVT()), 1);
12484
12485 EVT VTs[] = {VT};
12486 return getVTList(VTs);
12487}
12488
12490 EVT VTs[] = {VT1, VT2};
12491 return getVTList(VTs);
12492}
12493
12495 EVT VTs[] = {VT1, VT2, VT3};
12496 return getVTList(VTs);
12497}
12498
12500 EVT VTs[] = {VT1, VT2, VT3, VT4};
12501 return getVTList(VTs);
12502}
12503
12505 auto It = VTLists.find(VTs);
12506 if (It == VTLists.end()) {
12507 EVT *Array = Allocator.Allocate<EVT>(VTs.size());
12508 llvm::copy(VTs, Array);
12509 It = VTLists.insert(ArrayRef(Array, VTs.size())).first;
12510 }
12511 return makeVTList(It->data(), It->size());
12512}
12513
12514/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12515/// specified operands. If the resultant node already exists in the DAG,
12516/// this does not modify the specified node, instead it returns the node that
12517/// already exists. If the resultant node does not exist in the DAG, the
12518/// input node is returned. As a degenerate case, if you specify the same
12519/// input operands as the node already has, the input node is returned.
12521 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12522
12523 // Check to see if there is no change.
12524 if (Op == N->getOperand(0)) return N;
12525
12526 // See if the modified node already exists.
12527 FoldingSetInsertToken InsertToken;
12528 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertToken)) {
12529 Existing->intersectFlagsWith(N->getFlags());
12530 return Existing;
12531 }
12532
12533 // Nope it doesn't. Remove the node from its current place in the maps.
12534 if (InsertToken)
12535 if (!RemoveNodeFromCSEMaps(N))
12536 InsertToken = {};
12537
12538 // Now we update the operands.
12539 N->OperandList[0].set(Op);
12540
12542 // If this gets put into a CSE map, add it.
12543 if (InsertToken)
12544 CSEMap.insert(N, InsertToken);
12545 return N;
12546}
12547
12549 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12550
12551 // Check to see if there is no change.
12552 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
12553 return N; // No operands changed, just return the input node.
12554
12555 // See if the modified node already exists.
12556 FoldingSetInsertToken InsertToken;
12557 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertToken)) {
12558 Existing->intersectFlagsWith(N->getFlags());
12559 return Existing;
12560 }
12561
12562 // Nope it doesn't. Remove the node from its current place in the maps.
12563 if (InsertToken)
12564 if (!RemoveNodeFromCSEMaps(N))
12565 InsertToken = {};
12566
12567 // Now we update the operands.
12568 if (N->OperandList[0] != Op1)
12569 N->OperandList[0].set(Op1);
12570 if (N->OperandList[1] != Op2)
12571 N->OperandList[1].set(Op2);
12572
12574 // If this gets put into a CSE map, add it.
12575 if (InsertToken)
12576 CSEMap.insert(N, InsertToken);
12577 return N;
12578}
12579
12582 SDValue Ops[] = { Op1, Op2, Op3 };
12583 return UpdateNodeOperands(N, Ops);
12584}
12585
12588 SDValue Op3, SDValue Op4) {
12589 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12590 return UpdateNodeOperands(N, Ops);
12591}
12592
12595 SDValue Op3, SDValue Op4, SDValue Op5) {
12596 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12597 return UpdateNodeOperands(N, Ops);
12598}
12599
12602 unsigned NumOps = Ops.size();
12603 assert(N->getNumOperands() == NumOps &&
12604 "Update with wrong number of operands");
12605
12606 // If no operands changed just return the input node.
12607 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
12608 return N;
12609
12610 // See if the modified node already exists.
12611 FoldingSetInsertToken InsertToken;
12612 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertToken)) {
12613 Existing->intersectFlagsWith(N->getFlags());
12614 return Existing;
12615 }
12616
12617 // Nope it doesn't. Remove the node from its current place in the maps.
12618 if (InsertToken)
12619 if (!RemoveNodeFromCSEMaps(N))
12620 InsertToken = {};
12621
12622 // Now we update the operands.
12623 for (unsigned i = 0; i != NumOps; ++i)
12624 if (N->OperandList[i] != Ops[i])
12625 N->OperandList[i].set(Ops[i]);
12626
12628 // If this gets put into a CSE map, add it.
12629 if (InsertToken)
12630 CSEMap.insert(N, InsertToken);
12631 return N;
12632}
12633
12634/// DropOperands - Release the operands and set this node to have
12635/// zero operands.
12637 // Unlike the code in MorphNodeTo that does this, we don't need to
12638 // watch for dead nodes here.
12639 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12640 SDUse &Use = *I++;
12641 Use.set(SDValue());
12642 }
12643}
12644
12646 ArrayRef<MachineMemOperand *> NewMemRefs) {
12647 if (NewMemRefs.empty()) {
12648 N->clearMemRefs();
12649 return;
12650 }
12651
12652 // Check if we can avoid allocating by storing a single reference directly.
12653 if (NewMemRefs.size() == 1) {
12654 N->MemRefs = NewMemRefs[0];
12655 N->NumMemRefs = 1;
12656 return;
12657 }
12658
12659 MachineMemOperand **MemRefsBuffer =
12660 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
12661 llvm::copy(NewMemRefs, MemRefsBuffer);
12662 N->MemRefs = MemRefsBuffer;
12663 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12664}
12665
12666/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12667/// machine opcode.
12668///
12670 EVT VT) {
12671 SDVTList VTs = getVTList(VT);
12672 return SelectNodeTo(N, MachineOpc, VTs, {});
12673}
12674
12676 EVT VT, SDValue Op1) {
12677 SDVTList VTs = getVTList(VT);
12678 SDValue Ops[] = { Op1 };
12679 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12680}
12681
12683 EVT VT, SDValue Op1,
12684 SDValue Op2) {
12685 SDVTList VTs = getVTList(VT);
12686 SDValue Ops[] = { Op1, Op2 };
12687 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12688}
12689
12691 EVT VT, SDValue Op1,
12692 SDValue Op2, SDValue Op3) {
12693 SDVTList VTs = getVTList(VT);
12694 SDValue Ops[] = { Op1, Op2, Op3 };
12695 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12696}
12697
12700 SDVTList VTs = getVTList(VT);
12701 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12702}
12703
12705 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12706 SDVTList VTs = getVTList(VT1, VT2);
12707 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12708}
12709
12711 EVT VT1, EVT VT2) {
12712 SDVTList VTs = getVTList(VT1, VT2);
12713 return SelectNodeTo(N, MachineOpc, VTs, {});
12714}
12715
12717 EVT VT1, EVT VT2, EVT VT3,
12719 SDVTList VTs = getVTList(VT1, VT2, VT3);
12720 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12721}
12722
12724 EVT VT1, EVT VT2,
12725 SDValue Op1, SDValue Op2) {
12726 SDVTList VTs = getVTList(VT1, VT2);
12727 SDValue Ops[] = { Op1, Op2 };
12728 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12729}
12730
12733 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
12734 // Reset the NodeID to -1.
12735 New->setNodeId(-1);
12736 if (New != N) {
12737 ReplaceAllUsesWith(N, New);
12739 }
12740 return New;
12741}
12742
12743/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12744/// the line number information on the merged node since it is not possible to
12745/// preserve the information that operation is associated with multiple lines.
12746/// This will make the debugger work better at -O0, where there is a higher
12747/// probability of having other instructions associated with that line.
12748///
12749/// For IROrder, we keep the smaller of the two
12750SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12751 DebugLoc NLoc = N->getDebugLoc();
12752 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12753 N->setDebugLoc(DebugLoc());
12754 }
12755 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
12756 N->setIROrder(Order);
12757 return N;
12758}
12759
12760/// MorphNodeTo - This *mutates* the specified node to have the specified
12761/// return type, opcode, and operands.
12762///
12763/// Note that MorphNodeTo returns the resultant node. If there is already a
12764/// node of the specified opcode and operands, it returns that node instead of
12765/// the current one. Note that the SDLoc need not be the same.
12766///
12767/// Using MorphNodeTo is faster than creating a new node and swapping it in
12768/// with ReplaceAllUsesWith both because it often avoids allocating a new
12769/// node, and because it doesn't require CSE recalculation for any of
12770/// the node's users.
12771///
12772/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12773/// As a consequence it isn't appropriate to use from within the DAG combiner or
12774/// the legalizer which maintain worklists that would need to be updated when
12775/// deleting things.
12778 // If an identical node already exists, use it.
12779 FoldingSetInsertToken InsertToken;
12780 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12781 SDNodeKey ID(Opc, VTs, Ops);
12782 AddNodeIDCustom(ID.Tail, N, Opc);
12783 if (SDNode *ON = lookupNode(ID, SDLoc(N), InsertToken))
12784 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
12785 }
12786
12787 if (!RemoveNodeFromCSEMaps(N))
12788 InsertToken = {};
12789
12790 // Start the morphing.
12791 N->NodeType = Opc;
12792 N->ValueList = VTs.VTs;
12793 N->NumValues = VTs.NumVTs;
12794
12795 // Clear the operands list, updating used nodes to remove this from their
12796 // use list. Keep track of any operands that become dead as a result.
12797 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12798 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12799 SDUse &Use = *I++;
12800 SDNode *Used = Use.getNode();
12801 Use.set(SDValue());
12802 if (Used->use_empty())
12803 DeadNodeSet.insert(Used);
12804 }
12805
12806 // For MachineNode, initialize the memory references information.
12808 MN->clearMemRefs();
12809
12810 // Swap for an appropriately sized array from the recycler.
12811 removeOperands(N);
12812 createOperands(N, Ops);
12813
12814 // Delete any nodes that are still dead after adding the uses for the
12815 // new operands.
12816 if (!DeadNodeSet.empty()) {
12817 SmallVector<SDNode *, 16> DeadNodes;
12818 for (SDNode *N : DeadNodeSet)
12819 if (N->use_empty())
12820 DeadNodes.push_back(N);
12821 RemoveDeadNodes(DeadNodes);
12822 }
12823
12824 if (InsertToken)
12825 CSEMap.insert(N, InsertToken); // Memoize the new node.
12826 return N;
12827}
12828
12830 unsigned OrigOpc = Node->getOpcode();
12831 unsigned NewOpc;
12832 switch (OrigOpc) {
12833 default:
12834 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12835#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12836 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12837#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12838 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12839#include "llvm/IR/ConstrainedOps.def"
12840 }
12841
12842 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12843
12844 // We're taking this node out of the chain, so we need to re-link things.
12845 SDValue InputChain = Node->getOperand(0);
12846 SDValue OutputChain = SDValue(Node, 1);
12847 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
12848
12850 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12851 Ops.push_back(Node->getOperand(i));
12852
12853 SDVTList VTs = getVTList(Node->getValueType(0));
12854 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
12855
12856 // MorphNodeTo can operate in two ways: if an existing node with the
12857 // specified operands exists, it can just return it. Otherwise, it
12858 // updates the node in place to have the requested operands.
12859 if (Res == Node) {
12860 // If we updated the node in place, reset the node ID. To the isel,
12861 // this should be just like a newly allocated machine node.
12862 Res->setNodeId(-1);
12863 } else {
12866 }
12867
12868 return Res;
12869}
12870
12871/// getMachineNode - These are used for target selectors to create a new node
12872/// with specified return type(s), MachineInstr opcode, and operands.
12873///
12874/// Note that getMachineNode returns the resultant node. If there is already a
12875/// node of the specified opcode and operands, it returns that node instead of
12876/// the current one.
12878 EVT VT) {
12879 SDVTList VTs = getVTList(VT);
12880 return getMachineNode(Opcode, dl, VTs, {});
12881}
12882
12884 EVT VT, SDValue Op1) {
12885 SDVTList VTs = getVTList(VT);
12886 SDValue Ops[] = { Op1 };
12887 return getMachineNode(Opcode, dl, VTs, Ops);
12888}
12889
12891 EVT VT, SDValue Op1, SDValue Op2) {
12892 SDVTList VTs = getVTList(VT);
12893 SDValue Ops[] = { Op1, Op2 };
12894 return getMachineNode(Opcode, dl, VTs, Ops);
12895}
12896
12898 EVT VT, SDValue Op1, SDValue Op2,
12899 SDValue Op3) {
12900 SDVTList VTs = getVTList(VT);
12901 SDValue Ops[] = { Op1, Op2, Op3 };
12902 return getMachineNode(Opcode, dl, VTs, Ops);
12903}
12904
12907 SDVTList VTs = getVTList(VT);
12908 return getMachineNode(Opcode, dl, VTs, Ops);
12909}
12910
12912 EVT VT1, EVT VT2, SDValue Op1,
12913 SDValue Op2) {
12914 SDVTList VTs = getVTList(VT1, VT2);
12915 SDValue Ops[] = { Op1, Op2 };
12916 return getMachineNode(Opcode, dl, VTs, Ops);
12917}
12918
12920 EVT VT1, EVT VT2, SDValue Op1,
12921 SDValue Op2, SDValue Op3) {
12922 SDVTList VTs = getVTList(VT1, VT2);
12923 SDValue Ops[] = { Op1, Op2, Op3 };
12924 return getMachineNode(Opcode, dl, VTs, Ops);
12925}
12926
12928 EVT VT1, EVT VT2,
12930 SDVTList VTs = getVTList(VT1, VT2);
12931 return getMachineNode(Opcode, dl, VTs, Ops);
12932}
12933
12935 EVT VT1, EVT VT2, EVT VT3,
12936 SDValue Op1, SDValue Op2) {
12937 SDVTList VTs = getVTList(VT1, VT2, VT3);
12938 SDValue Ops[] = { Op1, Op2 };
12939 return getMachineNode(Opcode, dl, VTs, Ops);
12940}
12941
12943 EVT VT1, EVT VT2, EVT VT3,
12944 SDValue Op1, SDValue Op2,
12945 SDValue Op3) {
12946 SDVTList VTs = getVTList(VT1, VT2, VT3);
12947 SDValue Ops[] = { Op1, Op2, Op3 };
12948 return getMachineNode(Opcode, dl, VTs, Ops);
12949}
12950
12952 EVT VT1, EVT VT2, EVT VT3,
12954 SDVTList VTs = getVTList(VT1, VT2, VT3);
12955 return getMachineNode(Opcode, dl, VTs, Ops);
12956}
12957
12959 ArrayRef<EVT> ResultTys,
12961 SDVTList VTs = getVTList(ResultTys);
12962 return getMachineNode(Opcode, dl, VTs, Ops);
12963}
12964
12966 SDVTList VTs,
12968 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12970 FoldingSetInsertToken InsertToken;
12971
12972 if (DoCSE) {
12973 SDNodeKey ID(~Opcode, VTs, Ops);
12974 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
12975 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
12976 }
12977 }
12978
12979 // Allocate a new MachineSDNode.
12980 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
12981 createOperands(N, Ops);
12982
12983 if (DoCSE)
12984 CSEMap.insert(N, InsertToken);
12985
12986 InsertNode(N);
12987 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
12988 return N;
12989}
12990
12991/// getTargetExtractSubreg - A convenience function for creating
12992/// TargetOpcode::EXTRACT_SUBREG nodes.
12994 SDValue Operand) {
12995 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
12996 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
12997 VT, Operand, SRIdxVal);
12998 return SDValue(Subreg, 0);
12999}
13000
13001/// getTargetInsertSubreg - A convenience function for creating
13002/// TargetOpcode::INSERT_SUBREG nodes.
13004 SDValue Operand, SDValue Subreg) {
13005 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
13006 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
13007 VT, Operand, Subreg, SRIdxVal);
13008 return SDValue(Result, 0);
13009}
13010
13011/// getNodeIfExists - Get the specified node if it's already available, or
13012/// else return NULL.
13015 bool AllowCommute) {
13016 SDNodeFlags Flags;
13017 if (Inserter)
13018 Flags = Inserter->getFlags();
13019 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
13020}
13021
13024 const SDNodeFlags Flags,
13025 bool AllowCommute) {
13026 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
13027 return nullptr;
13028
13029 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
13030 SDNodeKey ID(Opcode, VTList, LookupOps);
13031 FoldingSetInsertToken InsertToken;
13032 if (SDNode *E = lookupNode(ID, InsertToken)) {
13033 E->intersectFlagsWith(Flags);
13034 return E;
13035 }
13036 return nullptr;
13037 };
13038
13039 if (SDNode *Existing = Lookup(Ops))
13040 return Existing;
13041
13042 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
13043 return Lookup({Ops[1], Ops[0]});
13044
13045 return nullptr;
13046}
13047
13048/// doesNodeExist - Check if a node exists without modifying its flags.
13049bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
13051 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
13052 SDNodeKey ID(Opcode, VTList, Ops);
13053 FoldingSetInsertToken InsertToken;
13054 if (lookupNode(ID, SDLoc(), InsertToken))
13055 return true;
13056 }
13057 return false;
13058}
13059
13060/// getDbgValue - Creates a SDDbgValue node.
13061///
13062/// SDNode
13064 SDNode *N, unsigned R, bool IsIndirect,
13065 const DebugLoc &DL, unsigned O) {
13066 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13067 "Expected inlined-at fields to agree");
13068 return new (DbgInfo->getAlloc())
13069 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
13070 {}, IsIndirect, DL, O,
13071 /*IsVariadic=*/false);
13072}
13073
13074/// Constant
13076 DIExpression *Expr,
13077 const Value *C,
13078 const DebugLoc &DL, unsigned O) {
13079 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13080 "Expected inlined-at fields to agree");
13081 return new (DbgInfo->getAlloc())
13082 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
13083 /*IsIndirect=*/false, DL, O,
13084 /*IsVariadic=*/false);
13085}
13086
13087/// FrameIndex
13089 DIExpression *Expr, unsigned FI,
13090 bool IsIndirect,
13091 const DebugLoc &DL,
13092 unsigned O) {
13093 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13094 "Expected inlined-at fields to agree");
13095 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
13096}
13097
13098/// FrameIndex with dependencies
13100 DIExpression *Expr, unsigned FI,
13101 ArrayRef<SDNode *> Dependencies,
13102 bool IsIndirect,
13103 const DebugLoc &DL,
13104 unsigned O) {
13105 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13106 "Expected inlined-at fields to agree");
13107 return new (DbgInfo->getAlloc())
13108 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
13109 Dependencies, IsIndirect, DL, O,
13110 /*IsVariadic=*/false);
13111}
13112
13113/// VReg
13115 Register VReg, bool IsIndirect,
13116 const DebugLoc &DL, unsigned O) {
13117 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13118 "Expected inlined-at fields to agree");
13119 return new (DbgInfo->getAlloc())
13120 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
13121 {}, IsIndirect, DL, O,
13122 /*IsVariadic=*/false);
13123}
13124
13127 ArrayRef<SDNode *> Dependencies,
13128 bool IsIndirect, const DebugLoc &DL,
13129 unsigned O, bool IsVariadic) {
13130 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13131 "Expected inlined-at fields to agree");
13132 return new (DbgInfo->getAlloc())
13133 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
13134 DL, O, IsVariadic);
13135}
13136
13138 unsigned OffsetInBits, unsigned SizeInBits,
13139 bool InvalidateDbg) {
13140 SDNode *FromNode = From.getNode();
13141 SDNode *ToNode = To.getNode();
13142 assert(FromNode && ToNode && "Can't modify dbg values");
13143
13144 // PR35338
13145 // TODO: assert(From != To && "Redundant dbg value transfer");
13146 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
13147 if (From == To || FromNode == ToNode)
13148 return;
13149
13150 if (!FromNode->getHasDebugValue())
13151 return;
13152
13153 SDDbgOperand FromLocOp =
13154 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
13156
13158 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
13159 if (Dbg->isInvalidated())
13160 continue;
13161
13162 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
13163
13164 // Create a new location ops vector that is equal to the old vector, but
13165 // with each instance of FromLocOp replaced with ToLocOp.
13166 bool Changed = false;
13167 auto NewLocOps = Dbg->copyLocationOps();
13168 std::replace_if(
13169 NewLocOps.begin(), NewLocOps.end(),
13170 [&Changed, FromLocOp](const SDDbgOperand &Op) {
13171 bool Match = Op == FromLocOp;
13172 Changed |= Match;
13173 return Match;
13174 },
13175 ToLocOp);
13176 // Ignore this SDDbgValue if we didn't find a matching location.
13177 if (!Changed)
13178 continue;
13179
13180 DIVariable *Var = Dbg->getVariable();
13181 auto *Expr = Dbg->getExpression();
13182 // If a fragment is requested, update the expression.
13183 if (SizeInBits) {
13184 // When splitting a larger (e.g., sign-extended) value whose
13185 // lower bits are described with an SDDbgValue, do not attempt
13186 // to transfer the SDDbgValue to the upper bits.
13187 if (auto FI = Expr->getFragmentInfo())
13188 if (OffsetInBits + SizeInBits > FI->SizeInBits)
13189 continue;
13190 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
13191 SizeInBits);
13192 if (!Fragment)
13193 continue;
13194 Expr = *Fragment;
13195 }
13196
13197 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
13198 // Clone the SDDbgValue and move it to To.
13199 SDDbgValue *Clone = getDbgValueList(
13200 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
13201 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
13202 Dbg->isVariadic());
13203 ClonedDVs.push_back(Clone);
13204
13205 if (InvalidateDbg) {
13206 // Invalidate value and indicate the SDDbgValue should not be emitted.
13207 Dbg->setIsInvalidated();
13208 Dbg->setIsEmitted();
13209 }
13210 }
13211
13212 for (SDDbgValue *Dbg : ClonedDVs) {
13213 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
13214 "Transferred DbgValues should depend on the new SDNode");
13215 AddDbgValue(Dbg, false);
13216 }
13217}
13218
13220 if (!N.getHasDebugValue())
13221 return;
13222
13223 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
13224 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Node))
13225 return SDDbgOperand::fromFrameIdx(FISDN->getIndex());
13226 return SDDbgOperand::fromNode(Node, ResNo);
13227 };
13228
13230 for (auto *DV : GetDbgValues(&N)) {
13231 if (DV->isInvalidated())
13232 continue;
13233 switch (N.getOpcode()) {
13234 default:
13235 break;
13236 case ISD::ADD: {
13237 SDValue N0 = N.getOperand(0);
13238 SDValue N1 = N.getOperand(1);
13239 if (!isa<ConstantSDNode>(N0)) {
13240 bool RHSConstant = isa<ConstantSDNode>(N1);
13241 uint64_t Offset;
13242 if (RHSConstant)
13243 Offset = N.getConstantOperandVal(1);
13244 // We are not allowed to turn indirect debug values variadic, so
13245 // don't salvage those.
13246 if (!RHSConstant && DV->isIndirect())
13247 continue;
13248
13249 // Rewrite an ADD constant node into a DIExpression. Since we are
13250 // performing arithmetic to compute the variable's *value* in the
13251 // DIExpression, we need to mark the expression with a
13252 // DW_OP_stack_value.
13253 auto *DIExpr = DV->getExpression();
13254 auto NewLocOps = DV->copyLocationOps();
13255 bool Changed = false;
13256 size_t OrigLocOpsSize = NewLocOps.size();
13257 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
13258 // We're not given a ResNo to compare against because the whole
13259 // node is going away. We know that any ISD::ADD only has one
13260 // result, so we can assume any node match is using the result.
13261 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13262 NewLocOps[i].getSDNode() != &N)
13263 continue;
13264 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13265 if (RHSConstant) {
13268 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
13269 } else {
13270 // Convert to a variadic expression (if not already).
13271 // convertToVariadicExpression() returns a const pointer, so we use
13272 // a temporary const variable here.
13273 const auto *TmpDIExpr =
13277 ExprOps.push_back(NewLocOps.size());
13278 ExprOps.push_back(dwarf::DW_OP_plus);
13279 SDDbgOperand RHS =
13281 NewLocOps.push_back(RHS);
13282 DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);
13283 }
13284 Changed = true;
13285 }
13286 (void)Changed;
13287 assert(Changed && "Salvage target doesn't use N");
13288
13289 bool IsVariadic =
13290 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
13291
13292 auto AdditionalDependencies = DV->getAdditionalDependencies();
13293 SDDbgValue *Clone = getDbgValueList(
13294 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
13295 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
13296 ClonedDVs.push_back(Clone);
13297 DV->setIsInvalidated();
13298 DV->setIsEmitted();
13299 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
13300 N0.getNode()->dumprFull(this);
13301 dbgs() << " into " << *DIExpr << '\n');
13302 }
13303 break;
13304 }
13305 case ISD::TRUNCATE: {
13306 SDValue N0 = N.getOperand(0);
13307 TypeSize FromSize = N0.getValueSizeInBits();
13308 TypeSize ToSize = N.getValueSizeInBits(0);
13309
13310 DIExpression *DbgExpression = DV->getExpression();
13311 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);
13312 auto NewLocOps = DV->copyLocationOps();
13313 bool Changed = false;
13314 for (size_t i = 0; i < NewLocOps.size(); ++i) {
13315 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13316 NewLocOps[i].getSDNode() != &N)
13317 continue;
13318
13319 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13320 DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);
13321 Changed = true;
13322 }
13323 assert(Changed && "Salvage target doesn't use N");
13324 (void)Changed;
13325
13326 SDDbgValue *Clone =
13327 getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,
13328 DV->getAdditionalDependencies(), DV->isIndirect(),
13329 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
13330
13331 ClonedDVs.push_back(Clone);
13332 DV->setIsInvalidated();
13333 DV->setIsEmitted();
13334 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
13335 dbgs() << " into " << *DbgExpression << '\n');
13336 break;
13337 }
13338 }
13339 }
13340
13341 for (SDDbgValue *Dbg : ClonedDVs) {
13342 assert((!Dbg->getSDNodes().empty() ||
13343 llvm::any_of(Dbg->getLocationOps(),
13344 [&](const SDDbgOperand &Op) {
13345 return Op.getKind() == SDDbgOperand::FRAMEIX;
13346 })) &&
13347 "Salvaged DbgValue should depend on a new SDNode");
13348 AddDbgValue(Dbg, false);
13349 }
13350}
13351
13352/// Creates a SDDbgLabel node.
13354 const DebugLoc &DL, unsigned O) {
13355 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13356 "Expected inlined-at fields to agree");
13357 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13358}
13359
13360namespace {
13361
13362/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13363/// pointed to by a use iterator is deleted, increment the use iterator
13364/// so that it doesn't dangle.
13365///
13366class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13369
13370 void NodeDeleted(SDNode *N, SDNode *E) override {
13371 // Increment the iterator as needed.
13372 while (UI != UE && N == UI->getUser())
13373 ++UI;
13374 }
13375
13376public:
13377 RAUWUpdateListener(SelectionDAG &d,
13380 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13381};
13382
13383} // end anonymous namespace
13384
13385/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13386/// This can cause recursive merging of nodes in the DAG.
13387///
13388/// This version assumes From has a single result value.
13389///
13391 SDNode *From = FromN.getNode();
13392 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13393 "Cannot replace with this method!");
13394 assert(From != To.getNode() && "Cannot replace uses of with self");
13395
13396 // Preserve Debug Values
13397 transferDbgValues(FromN, To);
13398 // Preserve extra info.
13399 copyExtraInfo(From, To.getNode());
13400
13401 // Iterate over all the existing uses of From. New uses will be added
13402 // to the beginning of the use list, which we avoid visiting.
13403 // This specifically avoids visiting uses of From that arise while the
13404 // replacement is happening, because any such uses would be the result
13405 // of CSE: If an existing node looks like From after one of its operands
13406 // is replaced by To, we don't want to replace of all its users with To
13407 // too. See PR3018 for more info.
13408 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13409 RAUWUpdateListener Listener(*this, UI, UE);
13410 while (UI != UE) {
13411 SDNode *User = UI->getUser();
13412
13413 // This node is about to morph, remove its old self from the CSE maps.
13414 RemoveNodeFromCSEMaps(User);
13415
13416 // A user can appear in a use list multiple times, and when this
13417 // happens the uses are usually next to each other in the list.
13418 // To help reduce the number of CSE recomputations, process all
13419 // the uses of this user that we can find this way.
13420 do {
13421 SDUse &Use = *UI;
13422 ++UI;
13423 Use.set(To);
13424 if (To->isDivergent() != From->isDivergent())
13426 } while (UI != UE && UI->getUser() == User);
13427 // Now that we have modified User, add it back to the CSE maps. If it
13428 // already exists there, recursively merge the results together.
13429 AddModifiedNodeToCSEMaps(User);
13430 }
13431
13432 // If we just RAUW'd the root, take note.
13433 if (FromN == getRoot())
13434 setRoot(To);
13435}
13436
13437/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13438/// This can cause recursive merging of nodes in the DAG.
13439///
13440/// This version assumes that for each value of From, there is a
13441/// corresponding value in To in the same position with the same type.
13442///
13444#ifndef NDEBUG
13445 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13446 assert((!From->hasAnyUseOfValue(i) ||
13447 From->getValueType(i) == To->getValueType(i)) &&
13448 "Cannot use this version of ReplaceAllUsesWith!");
13449#endif
13450
13451 // Handle the trivial case.
13452 if (From == To)
13453 return;
13454
13455 // Preserve Debug Info. Only do this if there's a use.
13456 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13457 if (From->hasAnyUseOfValue(i)) {
13458 assert((i < To->getNumValues()) && "Invalid To location");
13459 transferDbgValues(SDValue(From, i), SDValue(To, i));
13460 }
13461 // Preserve extra info.
13462 copyExtraInfo(From, To);
13463
13464 // Iterate over just the existing users of From. See the comments in
13465 // the ReplaceAllUsesWith above.
13466 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13467 RAUWUpdateListener Listener(*this, UI, UE);
13468 while (UI != UE) {
13469 SDNode *User = UI->getUser();
13470
13471 // This node is about to morph, remove its old self from the CSE maps.
13472 RemoveNodeFromCSEMaps(User);
13473
13474 // A user can appear in a use list multiple times, and when this
13475 // happens the uses are usually next to each other in the list.
13476 // To help reduce the number of CSE recomputations, process all
13477 // the uses of this user that we can find this way.
13478 do {
13479 SDUse &Use = *UI;
13480 ++UI;
13481 Use.setNode(To);
13482 if (To->isDivergent() != From->isDivergent())
13484 } while (UI != UE && UI->getUser() == User);
13485
13486 // Now that we have modified User, add it back to the CSE maps. If it
13487 // already exists there, recursively merge the results together.
13488 AddModifiedNodeToCSEMaps(User);
13489 }
13490
13491 // If we just RAUW'd the root, take note.
13492 if (From == getRoot().getNode())
13493 setRoot(SDValue(To, getRoot().getResNo()));
13494}
13495
13496/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13497/// This can cause recursive merging of nodes in the DAG.
13498///
13499/// This version can replace From with any result values. To must match the
13500/// number and types of values returned by From.
13502 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13503 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
13504
13505 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13506 // Preserve Debug Info.
13507 transferDbgValues(SDValue(From, i), To[i]);
13508 // Preserve extra info.
13509 copyExtraInfo(From, To[i].getNode());
13510 }
13511
13512 // Iterate over just the existing users of From. See the comments in
13513 // the ReplaceAllUsesWith above.
13514 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13515 RAUWUpdateListener Listener(*this, UI, UE);
13516 while (UI != UE) {
13517 SDNode *User = UI->getUser();
13518
13519 // This node is about to morph, remove its old self from the CSE maps.
13520 RemoveNodeFromCSEMaps(User);
13521
13522 // A user can appear in a use list multiple times, and when this happens the
13523 // uses are usually next to each other in the list. To help reduce the
13524 // number of CSE and divergence recomputations, process all the uses of this
13525 // user that we can find this way.
13526 bool To_IsDivergent = false;
13527 do {
13528 SDUse &Use = *UI;
13529 const SDValue &ToOp = To[Use.getResNo()];
13530 ++UI;
13531 Use.set(ToOp);
13532 if (ToOp.getValueType() != MVT::Other)
13533 To_IsDivergent |= ToOp->isDivergent();
13534 } while (UI != UE && UI->getUser() == User);
13535
13536 if (To_IsDivergent != From->isDivergent())
13538
13539 // Now that we have modified User, add it back to the CSE maps. If it
13540 // already exists there, recursively merge the results together.
13541 AddModifiedNodeToCSEMaps(User);
13542 }
13543
13544 // If we just RAUW'd the root, take note.
13545 if (From == getRoot().getNode())
13546 setRoot(SDValue(To[getRoot().getResNo()]));
13547}
13548
13549/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13550/// uses of other values produced by From.getNode() alone. The Deleted
13551/// vector is handled the same way as for ReplaceAllUsesWith.
13553 // Handle the really simple, really trivial case efficiently.
13554 if (From == To) return;
13555
13556 // Handle the simple, trivial, case efficiently.
13557 if (From.getNode()->getNumValues() == 1) {
13558 ReplaceAllUsesWith(From, To);
13559 return;
13560 }
13561
13562 // Preserve Debug Info.
13563 transferDbgValues(From, To);
13564 copyExtraInfo(From.getNode(), To.getNode());
13565
13566 // Iterate over just the existing users of From. See the comments in
13567 // the ReplaceAllUsesWith above.
13568 SDNode::use_iterator UI = From.getNode()->use_begin(),
13569 UE = From.getNode()->use_end();
13570 RAUWUpdateListener Listener(*this, UI, UE);
13571 while (UI != UE) {
13572 SDNode *User = UI->getUser();
13573 bool UserRemovedFromCSEMaps = false;
13574
13575 // A user can appear in a use list multiple times, and when this
13576 // happens the uses are usually next to each other in the list.
13577 // To help reduce the number of CSE recomputations, process all
13578 // the uses of this user that we can find this way.
13579 do {
13580 SDUse &Use = *UI;
13581
13582 // Skip uses of different values from the same node.
13583 if (Use.getResNo() != From.getResNo()) {
13584 ++UI;
13585 continue;
13586 }
13587
13588 // If this node hasn't been modified yet, it's still in the CSE maps,
13589 // so remove its old self from the CSE maps.
13590 if (!UserRemovedFromCSEMaps) {
13591 RemoveNodeFromCSEMaps(User);
13592 UserRemovedFromCSEMaps = true;
13593 }
13594
13595 ++UI;
13596 Use.set(To);
13597 if (To->isDivergent() != From->isDivergent())
13599 } while (UI != UE && UI->getUser() == User);
13600 // We are iterating over all uses of the From node, so if a use
13601 // doesn't use the specific value, no changes are made.
13602 if (!UserRemovedFromCSEMaps)
13603 continue;
13604
13605 // Now that we have modified User, add it back to the CSE maps. If it
13606 // already exists there, recursively merge the results together.
13607 AddModifiedNodeToCSEMaps(User);
13608 }
13609
13610 // If we just RAUW'd the root, take note.
13611 if (From == getRoot())
13612 setRoot(To);
13613}
13614
13615namespace {
13616
13617/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13618/// to record information about a use.
13619struct UseMemo {
13620 SDNode *User;
13621 unsigned Index;
13622 SDUse *Use;
13623};
13624
13625/// operator< - Sort Memos by User.
13626bool operator<(const UseMemo &L, const UseMemo &R) {
13627 return (intptr_t)L.User < (intptr_t)R.User;
13628}
13629
13630/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13631/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13632/// the node already has been taken care of recursively.
13633class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13634 SmallVectorImpl<UseMemo> &Uses;
13635
13636 void NodeDeleted(SDNode *N, SDNode *E) override {
13637 for (UseMemo &Memo : Uses)
13638 if (Memo.User == N)
13639 Memo.User = nullptr;
13640 }
13641
13642public:
13643 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13644 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13645};
13646
13647} // end anonymous namespace
13648
13649/// Return true if a glue output should propagate divergence information.
13651 switch (Node->getOpcode()) {
13652 case ISD::CopyFromReg:
13653 case ISD::CopyToReg:
13654 return false;
13655 default:
13656 return true;
13657 }
13658
13659 llvm_unreachable("covered opcode switch");
13660}
13661
13663 if (TLI->isSDNodeAlwaysUniform(N)) {
13664 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13665 "Conflicting divergence information!");
13666 return false;
13667 }
13668 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13669 return true;
13670 for (const auto &Op : N->ops()) {
13671 EVT VT = Op.getValueType();
13672
13673 // Skip Chain. It does not carry divergence.
13674 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13675 (VT != MVT::Glue || gluePropagatesDivergence(Op.getNode())))
13676 return true;
13677 }
13678 return false;
13679}
13680
13682 SmallVector<SDNode *, 16> Worklist(1, N);
13683 do {
13684 N = Worklist.pop_back_val();
13685 bool IsDivergent = calculateDivergence(N);
13686 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13687 N->SDNodeBits.IsDivergent = IsDivergent;
13688 llvm::append_range(Worklist, N->users());
13689 }
13690 } while (!Worklist.empty());
13691}
13692
13693void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13695 Order.reserve(AllNodes.size());
13696 for (auto &N : allnodes()) {
13697 unsigned NOps = N.getNumOperands();
13698 Degree[&N] = NOps;
13699 if (0 == NOps)
13700 Order.push_back(&N);
13701 }
13702 for (size_t I = 0; I != Order.size(); ++I) {
13703 SDNode *N = Order[I];
13704 for (auto *U : N->users()) {
13705 unsigned &UnsortedOps = Degree[U];
13706 if (0 == --UnsortedOps)
13707 Order.push_back(U);
13708 }
13709 }
13710}
13711
13712#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13713void SelectionDAG::VerifyDAGDivergence() {
13714 std::vector<SDNode *> TopoOrder;
13715 CreateTopologicalOrder(TopoOrder);
13716 for (auto *N : TopoOrder) {
13717 assert(calculateDivergence(N) == N->isDivergent() &&
13718 "Divergence bit inconsistency detected");
13719 }
13720}
13721#endif
13722
13723/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13724/// uses of other values produced by From.getNode() alone. The same value
13725/// may appear in both the From and To list. The Deleted vector is
13726/// handled the same way as for ReplaceAllUsesWith.
13728 const SDValue *To,
13729 unsigned Num){
13730 // Handle the simple, trivial case efficiently.
13731 if (Num == 1)
13732 return ReplaceAllUsesOfValueWith(*From, *To);
13733
13734 transferDbgValues(*From, *To);
13735 copyExtraInfo(From->getNode(), To->getNode());
13736
13737 // Read up all the uses and make records of them. This helps
13738 // processing new uses that are introduced during the
13739 // replacement process.
13741 for (unsigned i = 0; i != Num; ++i) {
13742 unsigned FromResNo = From[i].getResNo();
13743 SDNode *FromNode = From[i].getNode();
13744 for (SDUse &Use : FromNode->uses()) {
13745 if (Use.getResNo() == FromResNo) {
13746 UseMemo Memo = {Use.getUser(), i, &Use};
13747 Uses.push_back(Memo);
13748 }
13749 }
13750 }
13751
13752 // Sort the uses, so that all the uses from a given User are together.
13754 RAUOVWUpdateListener Listener(*this, Uses);
13755
13756 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13757 UseIndex != UseIndexEnd; ) {
13758 // We know that this user uses some value of From. If it is the right
13759 // value, update it.
13760 SDNode *User = Uses[UseIndex].User;
13761 // If the node has been deleted by recursive CSE updates when updating
13762 // another node, then just skip this entry.
13763 if (User == nullptr) {
13764 ++UseIndex;
13765 continue;
13766 }
13767
13768 // This node is about to morph, remove its old self from the CSE maps.
13769 RemoveNodeFromCSEMaps(User);
13770
13771 // The Uses array is sorted, so all the uses for a given User
13772 // are next to each other in the list.
13773 // To help reduce the number of CSE recomputations, process all
13774 // the uses of this user that we can find this way.
13775 do {
13776 unsigned i = Uses[UseIndex].Index;
13777 SDUse &Use = *Uses[UseIndex].Use;
13778 ++UseIndex;
13779
13780 Use.set(To[i]);
13781 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13782
13783 // Now that we have modified User, add it back to the CSE maps. If it
13784 // already exists there, recursively merge the results together.
13785 AddModifiedNodeToCSEMaps(User);
13786 }
13787}
13788
13789/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13790/// based on their topological order. It returns the maximum id and a vector
13791/// of the SDNodes* in assigned order by reference.
13793 unsigned DAGSize = 0;
13794
13795 // SortedPos tracks the progress of the algorithm. Nodes before it are
13796 // sorted, nodes after it are unsorted. When the algorithm completes
13797 // it is at the end of the list.
13798 allnodes_iterator SortedPos = allnodes_begin();
13799
13800 // Visit all the nodes. Move nodes with no operands to the front of
13801 // the list immediately. Annotate nodes that do have operands with their
13802 // operand count. Before we do this, the Node Id fields of the nodes
13803 // may contain arbitrary values. After, the Node Id fields for nodes
13804 // before SortedPos will contain the topological sort index, and the
13805 // Node Id fields for nodes At SortedPos and after will contain the
13806 // count of outstanding operands.
13808 checkForCycles(&N, this);
13809 unsigned Degree = N.getNumOperands();
13810 if (Degree == 0) {
13811 // A node with no uses, add it to the result array immediately.
13812 N.setNodeId(DAGSize++);
13813 allnodes_iterator Q(&N);
13814 if (Q != SortedPos)
13815 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
13816 assert(SortedPos != AllNodes.end() && "Overran node list");
13817 ++SortedPos;
13818 } else {
13819 // Temporarily use the Node Id as scratch space for the degree count.
13820 N.setNodeId(Degree);
13821 }
13822 }
13823
13824 // Visit all the nodes. As we iterate, move nodes into sorted order,
13825 // such that by the time the end is reached all nodes will be sorted.
13826 for (SDNode &Node : allnodes()) {
13827 SDNode *N = &Node;
13828 checkForCycles(N, this);
13829 // N is in sorted position, so all its uses have one less operand
13830 // that needs to be sorted.
13831 for (SDNode *P : N->users()) {
13832 unsigned Degree = P->getNodeId();
13833 assert(Degree != 0 && "Invalid node degree");
13834 --Degree;
13835 if (Degree == 0) {
13836 // All of P's operands are sorted, so P may sorted now.
13837 P->setNodeId(DAGSize++);
13838 if (P->getIterator() != SortedPos)
13839 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
13840 assert(SortedPos != AllNodes.end() && "Overran node list");
13841 ++SortedPos;
13842 } else {
13843 // Update P's outstanding operand count.
13844 P->setNodeId(Degree);
13845 }
13846 }
13847 if (Node.getIterator() == SortedPos) {
13848#ifndef NDEBUG
13850 SDNode *S = &*++I;
13851 dbgs() << "Overran sorted position:\n";
13852 S->dumprFull(this); dbgs() << "\n";
13853 dbgs() << "Checking if this is due to cycles\n";
13854 checkForCycles(this, true);
13855#endif
13856 llvm_unreachable(nullptr);
13857 }
13858 }
13859
13860 assert(SortedPos == AllNodes.end() &&
13861 "Topological sort incomplete!");
13862 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13863 "First node in topological sort is not the entry token!");
13864 assert(AllNodes.front().getNodeId() == 0 &&
13865 "First node in topological sort has non-zero id!");
13866 assert(AllNodes.front().getNumOperands() == 0 &&
13867 "First node in topological sort has operands!");
13868 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13869 "Last node in topologic sort has unexpected id!");
13870 assert(AllNodes.back().use_empty() &&
13871 "Last node in topologic sort has users!");
13872 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13873 return DAGSize;
13874}
13875
13877 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13878 SortedNodes.clear();
13879 // Node -> remaining number of outstanding operands.
13880 DenseMap<const SDNode *, unsigned> RemainingOperands;
13881
13882 // Put nodes without any operands into SortedNodes first.
13883 for (const SDNode &N : allnodes()) {
13884 checkForCycles(&N, this);
13885 unsigned NumOperands = N.getNumOperands();
13886 if (NumOperands == 0)
13887 SortedNodes.push_back(&N);
13888 else
13889 // Record their total number of outstanding operands.
13890 RemainingOperands[&N] = NumOperands;
13891 }
13892
13893 // A node is pushed into SortedNodes when all of its operands (predecessors in
13894 // the graph) are also in SortedNodes.
13895 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13896 const SDNode *N = SortedNodes[i];
13897 for (const SDNode *U : N->users()) {
13898 // HandleSDNode is never part of a DAG and therefore has no entry in
13899 // RemainingOperands.
13900 if (U->getOpcode() == ISD::HANDLENODE)
13901 continue;
13902 unsigned &NumRemOperands = RemainingOperands[U];
13903 assert(NumRemOperands && "Invalid number of remaining operands");
13904 --NumRemOperands;
13905 if (!NumRemOperands)
13906 SortedNodes.push_back(U);
13907 }
13908 }
13909
13910 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13911 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13912 "First node in topological sort is not the entry token");
13913 assert(SortedNodes.front()->getNumOperands() == 0 &&
13914 "First node in topological sort has operands");
13915}
13916
13917/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13918/// value is produced by SD.
13919void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13920 for (SDNode *SD : DB->getSDNodes()) {
13921 if (!SD)
13922 continue;
13923 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13924 SD->setHasDebugValue(true);
13925 }
13926 DbgInfo->add(DB, isParameter);
13927}
13928
13929void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
13930
13932 SDValue NewMemOpChain) {
13933 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13934 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13935 // The new memory operation must have the same position as the old load in
13936 // terms of memory dependency. Create a TokenFactor for the old load and new
13937 // memory operation and update uses of the old load's output chain to use that
13938 // TokenFactor.
13939 if (OldChain == NewMemOpChain || OldChain.use_empty())
13940 return NewMemOpChain;
13941
13942 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
13943 OldChain, NewMemOpChain);
13944 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
13945 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
13946 return TokenFactor;
13947}
13948
13950 SDValue NewMemOp) {
13951 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13952 SDValue OldChain = SDValue(OldLoad, 1);
13953 SDValue NewMemOpChain = NewMemOp.getValue(1);
13954 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13955}
13956
13958 Function **OutFunction) {
13959 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13960
13961 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
13962 auto *Module = MF->getFunction().getParent();
13963 auto *Function = Module->getFunction(Symbol);
13964
13965 if (OutFunction != nullptr)
13966 *OutFunction = Function;
13967
13968 if (Function != nullptr) {
13969 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
13970 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
13971 }
13972
13973 std::string ErrorStr;
13974 raw_string_ostream ErrorFormatter(ErrorStr);
13975 ErrorFormatter << "Undefined external symbol ";
13976 ErrorFormatter << '"' << Symbol << '"';
13977 report_fatal_error(Twine(ErrorStr));
13978}
13979
13980//===----------------------------------------------------------------------===//
13981// SDNode Class
13982//===----------------------------------------------------------------------===//
13983
13986 return Const != nullptr && Const->isZero();
13987}
13988
13990 return V.isUndef() || isNullConstant(V);
13991}
13992
13995 return Const != nullptr && Const->isZero() && !Const->isNegative();
13996}
13997
14000 return Const != nullptr && Const->isAllOnes();
14001}
14002
14005 return Const != nullptr && Const->isOne();
14006}
14007
14010 return Const != nullptr && Const->isMinSignedValue();
14011}
14012
14014 SDValue V, unsigned OperandNo,
14015 unsigned Depth) const {
14016 APInt DemandedElts = getDemandAllEltsMask(V);
14017 return isIdentityElement(Opcode, Flags, V, DemandedElts, OperandNo, Depth);
14018}
14019
14021 SDValue V, const APInt &DemandedElts,
14022 unsigned OperandNo, unsigned Depth) const {
14023 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
14024 // TODO: Target-specific opcodes could be added.
14025 if (V.getValueType().isInteger()) {
14026 KnownBits Known = computeKnownBits(V, DemandedElts, Depth);
14027 if (Known.isConstant()) {
14028 const APInt &Const = Known.getConstant();
14029 switch (Opcode) {
14030 case ISD::ADD:
14031 case ISD::OR:
14032 case ISD::XOR:
14033 case ISD::UMAX:
14034 return Const.isZero();
14035 case ISD::MUL:
14036 return Const.isOne();
14037 case ISD::AND:
14038 case ISD::UMIN:
14039 return Const.isAllOnes();
14040 case ISD::SMAX:
14041 return Const.isMinSignedValue();
14042 case ISD::SMIN:
14043 return Const.isMaxSignedValue();
14044 case ISD::SUB:
14045 case ISD::SHL:
14046 case ISD::SRA:
14047 case ISD::SRL:
14048 return OperandNo == 1 && Const.isZero();
14049 case ISD::UDIV:
14050 case ISD::SDIV:
14051 return OperandNo == 1 && Const.isOne();
14052 }
14053 }
14054 } else if (auto *ConstFP = isConstOrConstSplatFP(V, DemandedElts)) {
14055 switch (Opcode) {
14056 case ISD::FADD:
14057 return ConstFP->isZero() &&
14058 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
14059 case ISD::FSUB:
14060 return OperandNo == 1 && ConstFP->isZero() &&
14061 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
14062 case ISD::FMUL:
14063 return ConstFP->isOne();
14064 case ISD::FDIV:
14065 return OperandNo == 1 && ConstFP->isOne();
14066 case ISD::FMINNUM:
14067 case ISD::FMAXNUM:
14068 case ISD::FMINIMUMNUM:
14069 case ISD::FMAXIMUMNUM: {
14070 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
14071 // depending on fast-math flags (FMF).
14072 EVT VT = V.getValueType();
14073 const fltSemantics &Semantics = VT.getFltSemantics();
14074 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics)
14075 : !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
14076 : APFloat::getLargest(Semantics);
14077 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
14078 NeutralAF.changeSign();
14079
14080 return ConstFP->isExactlyValue(NeutralAF);
14081 }
14082 case ISD::FMINIMUM:
14083 case ISD::FMAXIMUM: {
14084 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
14085 const APFloat &VAPF = ConstFP->getValueAPF();
14086 bool NeutralNegative = (Opcode == ISD::FMAXIMUM);
14087 if (Flags.hasNoInfs())
14088 return VAPF.isLargest() && VAPF.isNegative() == NeutralNegative;
14089 return VAPF.isInfinity() && VAPF.isNegative() == NeutralNegative;
14090 }
14091 }
14092 }
14093 return false;
14094}
14095
14097 while (V.getOpcode() == ISD::BITCAST)
14098 V = V.getOperand(0);
14099 return V;
14100}
14101
14103 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
14104 V = V.getOperand(0);
14105 return V;
14106}
14107
14109 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
14110 V = V.getOperand(0);
14111 return V;
14112}
14113
14115 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
14116 SDValue InVec = V.getOperand(0);
14117 SDValue EltNo = V.getOperand(2);
14118 EVT VT = InVec.getValueType();
14119 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
14120 if (IndexC && VT.isFixedLengthVector() &&
14121 IndexC->getAPIntValue().ult(VT.getVectorNumElements()) &&
14122 !DemandedElts[IndexC->getZExtValue()]) {
14123 V = InVec;
14124 continue;
14125 }
14126 break;
14127 }
14128 return V;
14129}
14130
14132 while (V.getOpcode() == ISD::TRUNCATE)
14133 V = V.getOperand(0);
14134 return V;
14135}
14136
14137bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
14138 if (V.getOpcode() != ISD::XOR)
14139 return false;
14140 V = peekThroughBitcasts(V.getOperand(1));
14141 unsigned NumBits = V.getScalarValueSizeInBits();
14142 ConstantSDNode *C =
14143 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
14144 return C && (C->getAPIntValue().countr_one() >= NumBits);
14145}
14146
14148 bool AllowTruncation) {
14149 APInt DemandedElts = getDemandAllEltsMask(N);
14150 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
14151}
14152
14154 bool AllowUndefs,
14155 bool AllowTruncation) {
14157 return CN;
14158
14159 // SplatVectors can truncate their operands. Ignore that case here unless
14160 // AllowTruncation is set.
14161 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
14162 EVT VecEltVT = N->getValueType(0).getVectorElementType();
14163 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
14164 EVT CVT = CN->getValueType(0);
14165 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
14166 if (AllowTruncation || CVT == VecEltVT)
14167 return CN;
14168 }
14169 }
14170
14172 BitVector UndefElements;
14173 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
14174
14175 // BuildVectors can truncate their operands. Ignore that case here unless
14176 // AllowTruncation is set.
14177 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
14178 if (CN && (UndefElements.none() || AllowUndefs)) {
14179 EVT CVT = CN->getValueType(0);
14180 EVT NSVT = N.getValueType().getScalarType();
14181 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
14182 if (AllowTruncation || (CVT == NSVT))
14183 return CN;
14184 }
14185 }
14186
14187 if (N.getOpcode() == ISD::MERGE_VALUES)
14188 return isConstOrConstSplat(N.getOperand(N.getResNo()), DemandedElts,
14189 AllowUndefs, AllowTruncation);
14190
14191 return nullptr;
14192}
14193
14195 APInt DemandedElts = getDemandAllEltsMask(N);
14196 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
14197}
14198
14200 const APInt &DemandedElts,
14201 bool AllowUndefs) {
14203 return CN;
14204
14206 BitVector UndefElements;
14207 ConstantFPSDNode *CN =
14208 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
14209 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
14210 if (CN && (UndefElements.none() || AllowUndefs))
14211 return CN;
14212 }
14213
14214 if (N.getOpcode() == ISD::SPLAT_VECTOR)
14215 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
14216 return CN;
14217
14218 return nullptr;
14219}
14220
14221bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
14222 // TODO: may want to use peekThroughBitcast() here.
14223 ConstantSDNode *C =
14224 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14225 return C && C->isZero();
14226}
14227
14228bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
14229 ConstantSDNode *C =
14230 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
14231 return C && C->isOne();
14232}
14233
14234bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
14235 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14236 return C && C->isOne();
14237}
14238
14239bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
14241 unsigned BitWidth = N.getScalarValueSizeInBits();
14242 ConstantSDNode *C =
14243 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14244 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
14245}
14246
14247bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
14249 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, true);
14250 return C && C->isZero();
14251}
14252
14253bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
14254 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14255 return C && C->isZero();
14256}
14257
14261
14263 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
14265 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
14266 bool IsVolatile = false;
14267 bool IsNonTemporal = false;
14268 bool IsDereferenceable = true;
14269 bool IsInvariant = true;
14270 for (const MachineMemOperand *MMO : memoperands()) {
14271 IsVolatile |= MMO->isVolatile();
14272 IsNonTemporal |= MMO->isNonTemporal();
14273 IsDereferenceable &= MMO->isDereferenceable();
14274 IsInvariant &= MMO->isInvariant();
14275 }
14276 MemSDNodeBits.IsVolatile = IsVolatile;
14277 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
14278 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
14279 MemSDNodeBits.IsInvariant = IsInvariant;
14280
14281 // For the single-MMO case, we check here that the size of the memory operand
14282 // fits within the size of the MMO. This is because the MMO might indicate
14283 // only a possible address range instead of specifying the affected memory
14284 // addresses precisely.
14287 getMemOperand()->getSize().getValue())) &&
14288 "Size mismatch!");
14289}
14290
14291namespace {
14292
14293 struct EVTArray {
14294 std::vector<EVT> VTs;
14295
14296 EVTArray() {
14297 VTs.reserve(MVT::VALUETYPE_SIZE);
14298 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
14299 VTs.push_back(MVT((MVT::SimpleValueType)i));
14300 }
14301 };
14302
14303} // end anonymous namespace
14304
14305/// getValueTypeList - Return a pointer to the specified value type.
14306///
14307const EVT *SDNode::getValueTypeList(MVT VT) {
14308 static EVTArray SimpleVTArray;
14309
14310 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
14311 return &SimpleVTArray.VTs[VT.SimpleTy];
14312}
14313
14314/// hasAnyUseOfValue - Return true if there are any use of the indicated
14315/// value. This method ignores uses of other values defined by this operation.
14316bool SDNode::hasAnyUseOfValue(unsigned Value) const {
14317 assert(Value < getNumValues() && "Bad value!");
14318
14319 for (SDUse &U : uses())
14320 if (U.getResNo() == Value)
14321 return true;
14322
14323 return false;
14324}
14325
14326/// isOnlyUserOf - Return true if this node is the only use of N.
14327bool SDNode::isOnlyUserOf(const SDNode *N) const {
14328 bool Seen = false;
14329 for (const SDNode *User : N->users()) {
14330 if (User == this)
14331 Seen = true;
14332 else
14333 return false;
14334 }
14335
14336 return Seen;
14337}
14338
14339/// Return true if the only users of N are contained in Nodes.
14341 bool Seen = false;
14342 for (const SDNode *User : N->users()) {
14343 if (llvm::is_contained(Nodes, User))
14344 Seen = true;
14345 else
14346 return false;
14347 }
14348
14349 return Seen;
14350}
14351
14352/// Return true if the referenced return value is an operand of N.
14353bool SDValue::isOperandOf(const SDNode *N) const {
14354 return is_contained(N->op_values(), *this);
14355}
14356
14357bool SDNode::isOperandOf(const SDNode *N) const {
14358 return any_of(N->op_values(),
14359 [this](SDValue Op) { return this == Op.getNode(); });
14360}
14361
14362/// reachesChainWithoutSideEffects - Return true if this operand (which must
14363/// be a chain) reaches the specified operand without crossing any
14364/// side-effecting instructions on any chain path. In practice, this looks
14365/// through token factors and non-volatile loads. In order to remain efficient,
14366/// this only looks a couple of nodes in, it does not do an exhaustive search.
14367///
14368/// Note that we only need to examine chains when we're searching for
14369/// side-effects; SelectionDAG requires that all side-effects are represented
14370/// by chains, even if another operand would force a specific ordering. This
14371/// constraint is necessary to allow transformations like splitting loads.
14373 unsigned Depth) const {
14374 if (*this == Dest) return true;
14375
14376 // Don't search too deeply, we just want to be able to see through
14377 // TokenFactor's etc.
14378 if (Depth == 0) return false;
14379
14380 // If this is a token factor, all inputs to the TF happen in parallel.
14381 if (getOpcode() == ISD::TokenFactor) {
14382 // First, try a shallow search.
14383 if (is_contained((*this)->ops(), Dest)) {
14384 // We found the chain we want as an operand of this TokenFactor.
14385 // Essentially, we reach the chain without side-effects if we could
14386 // serialize the TokenFactor into a simple chain of operations with
14387 // Dest as the last operation. This is automatically true if the
14388 // chain has one use: there are no other ordering constraints.
14389 // If the chain has more than one use, we give up: some other
14390 // use of Dest might force a side-effect between Dest and the current
14391 // node.
14392 if (Dest.hasOneUse())
14393 return true;
14394 }
14395 // Next, try a deep search: check whether every operand of the TokenFactor
14396 // reaches Dest.
14397 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
14398 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
14399 });
14400 }
14401
14402 // Loads don't have side effects, look through them.
14403 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
14404 if (Ld->isUnordered())
14405 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
14406 }
14407 return false;
14408}
14409
14410bool SDNode::hasPredecessor(const SDNode *N) const {
14413 Worklist.push_back(this);
14414 return hasPredecessorHelper(N, Visited, Worklist);
14415}
14416
14418 this->Flags &= Flags;
14419}
14420
14421SDValue
14423 ArrayRef<ISD::NodeType> CandidateBinOps,
14424 bool AllowPartials) {
14425 // The pattern must end in an extract from index 0.
14426 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14427 !isNullConstant(Extract->getOperand(1)))
14428 return SDValue();
14429
14430 // Match against one of the candidate binary ops.
14431 SDValue Op = Extract->getOperand(0);
14432 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
14433 return Op.getOpcode() == unsigned(BinOp);
14434 }))
14435 return SDValue();
14436
14437 // Floating-point reductions may require relaxed constraints on the final step
14438 // of the reduction because they may reorder intermediate operations.
14439 unsigned CandidateBinOp = Op.getOpcode();
14440 if (Op.getValueType().isFloatingPoint()) {
14441 SDNodeFlags Flags = Op->getFlags();
14442 switch (CandidateBinOp) {
14443 case ISD::FADD:
14444 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14445 return SDValue();
14446 break;
14447 default:
14448 llvm_unreachable("Unhandled FP opcode for binop reduction");
14449 }
14450 }
14451
14452 // Matching failed - attempt to see if we did enough stages that a partial
14453 // reduction from a subvector is possible.
14454 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14455 if (!AllowPartials || !Op)
14456 return SDValue();
14457 EVT OpVT = Op.getValueType();
14458 EVT OpSVT = OpVT.getScalarType();
14459 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
14460 if (TLI->getExtractSubvectorCost(SubVT, OpVT, 0) >
14462 return SDValue();
14463 BinOp = (ISD::NodeType)CandidateBinOp;
14464 return getExtractSubvector(SDLoc(Op), SubVT, Op, 0);
14465 };
14466
14467 // At each stage, we're looking for something that looks like:
14468 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14469 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14470 // i32 undef, i32 undef, i32 undef, i32 undef>
14471 // %a = binop <8 x i32> %op, %s
14472 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14473 // we expect something like:
14474 // <4,5,6,7,u,u,u,u>
14475 // <2,3,u,u,u,u,u,u>
14476 // <1,u,u,u,u,u,u,u>
14477 // While a partial reduction match would be:
14478 // <2,3,u,u,u,u,u,u>
14479 // <1,u,u,u,u,u,u,u>
14480 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
14481 SDValue PrevOp;
14482 for (unsigned i = 0; i < Stages; ++i) {
14483 unsigned MaskEnd = (1 << i);
14484
14485 if (Op.getOpcode() != CandidateBinOp)
14486 return PartialReduction(PrevOp, MaskEnd);
14487
14488 SDValue Op0 = Op.getOperand(0);
14489 SDValue Op1 = Op.getOperand(1);
14490
14492 if (Shuffle) {
14493 Op = Op1;
14494 } else {
14495 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
14496 Op = Op0;
14497 }
14498
14499 // The first operand of the shuffle should be the same as the other operand
14500 // of the binop.
14501 if (!Shuffle || Shuffle->getOperand(0) != Op)
14502 return PartialReduction(PrevOp, MaskEnd);
14503
14504 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14505 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14506 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
14507 return PartialReduction(PrevOp, MaskEnd);
14508
14509 PrevOp = Op;
14510 }
14511
14512 // Handle subvector reductions, which tend to appear after the shuffle
14513 // reduction stages.
14514 while (Op.getOpcode() == CandidateBinOp) {
14515 unsigned NumElts = Op.getValueType().getVectorNumElements();
14516 SDValue Op0 = Op.getOperand(0);
14517 SDValue Op1 = Op.getOperand(1);
14518 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14520 Op0.getOperand(0) != Op1.getOperand(0))
14521 break;
14522 SDValue Src = Op0.getOperand(0);
14523 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14524 if (NumSrcElts != (2 * NumElts))
14525 break;
14526 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
14527 Op1.getConstantOperandAPInt(1) == NumElts) &&
14528 !(Op1.getConstantOperandAPInt(1) == 0 &&
14529 Op0.getConstantOperandAPInt(1) == NumElts))
14530 break;
14531 Op = Src;
14532 }
14533
14534 BinOp = (ISD::NodeType)CandidateBinOp;
14535 return Op;
14536}
14537
14539 EVT VT = N->getValueType(0);
14540 EVT EltVT = VT.getVectorElementType();
14541 unsigned NE = getMaxRuntimeNumElements(VT);
14542
14543 if (VT.isScalableVector() && (NE == 0 || ResNE != 0))
14544 reportFatalUsageError("Cannot unroll scalable vector!");
14545
14546 assert(NE && "Nothing to unroll!");
14547 SDLoc dl(N);
14548
14549 // If ResNE is 0, fully unroll the vector op.
14550 if (ResNE == 0)
14551 ResNE = NE;
14552 else if (NE > ResNE)
14553 NE = ResNE;
14554
14555 if (N->getNumValues() == 2) {
14556 SmallVector<SDValue, 8> Scalars0, Scalars1;
14557 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14558 EVT VT1 = N->getValueType(1);
14559 EVT EltVT1 = VT1.getVectorElementType();
14560
14561 unsigned i;
14562 for (i = 0; i != NE; ++i) {
14563 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14564 SDValue Operand = N->getOperand(j);
14565 EVT OperandVT = Operand.getValueType();
14566
14567 // A vector operand; extract a single element.
14568 EVT OperandEltVT = OperandVT.getVectorElementType();
14569 Operands[j] = getExtractVectorElt(dl, OperandEltVT, Operand, i);
14570 }
14571
14572 SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);
14573 Scalars0.push_back(EltOp);
14574 Scalars1.push_back(EltOp.getValue(1));
14575 }
14576
14577 for (; i < ResNE; ++i) {
14578 Scalars0.push_back(getUNDEF(EltVT));
14579 Scalars1.push_back(getUNDEF(EltVT1));
14580 }
14581
14583 : ElementCount::getFixed(ResNE);
14584 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResEC);
14585 EVT VecVT1 = EVT::getVectorVT(*getContext(), EltVT1, ResEC);
14586 SDValue Vec0 = buildVectorFromUnrolledParts(VecVT, dl, Scalars0);
14587 SDValue Vec1 = buildVectorFromUnrolledParts(VecVT1, dl, Scalars1);
14588 return getMergeValues({Vec0, Vec1}, dl);
14589 }
14590
14591 assert(N->getNumValues() == 1 &&
14592 "Can't unroll a vector with multiple results!");
14593
14595 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14596
14597 unsigned i;
14598 for (i= 0; i != NE; ++i) {
14599 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14600 SDValue Operand = N->getOperand(j);
14601 EVT OperandVT = Operand.getValueType();
14602 if (OperandVT.isVector()) {
14603 // A vector operand; extract a single element.
14604 EVT OperandEltVT = OperandVT.getVectorElementType();
14605 Operands[j] = getExtractVectorElt(dl, OperandEltVT, Operand, i);
14606 } else {
14607 // A scalar operand; just use it as is.
14608 Operands[j] = Operand;
14609 }
14610 }
14611
14612 switch (N->getOpcode()) {
14613 default: {
14614 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
14615 N->getFlags()));
14616 break;
14617 }
14618 case ISD::VSELECT: {
14619 SDValue Cond = Operands[0];
14620 EVT CondVT = Cond.getValueType();
14622 if (CondVT != MVT::i1 &&
14623 TLI.getBooleanContents(N->getOperand(1).getValueType()) !=
14624 TLI.getBooleanContents(CondVT)) {
14625 EVT SetCCVT =
14626 TLI.getSetCCResultType(getDataLayout(), *getContext(), CondVT);
14627 Operands[0] =
14628 getSetCC(dl, SetCCVT, Cond, getConstant(0, dl, CondVT), ISD::SETNE);
14629 }
14630 Scalars.push_back(
14631 getNode(ISD::SELECT, dl, EltVT, Operands, N->getFlags()));
14632 break;
14633 }
14634 case ISD::SHL:
14635 case ISD::SRA:
14636 case ISD::SRL:
14637 case ISD::ROTL:
14638 case ISD::ROTR:
14639 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
14641 Operands[1])));
14642 break;
14644 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
14645 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
14646 Operands[0],
14647 getValueType(ExtVT)));
14648 break;
14649 }
14650 case ISD::ADDRSPACECAST: {
14651 const auto *ASC = cast<AddrSpaceCastSDNode>(N);
14652 Scalars.push_back(
14653 getAddrSpaceCast(dl, EltVT, Operands[0], ASC->getSrcAddressSpace(),
14654 ASC->getDestAddressSpace(), ASC->getFlags()));
14655 break;
14656 }
14657 }
14658 }
14659
14660 for (; i < ResNE; ++i)
14661 Scalars.push_back(getUNDEF(EltVT));
14662
14663 EVT VecVT = VT.isScalableVector()
14664 ? VT
14665 : EVT::getVectorVT(*getContext(), EltVT, ResNE);
14666 return buildVectorFromUnrolledParts(VecVT, dl, Scalars);
14667}
14668
14669std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14670 SDNode *N, unsigned ResNE) {
14671 unsigned Opcode = N->getOpcode();
14672 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14673 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14674 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14675 "Expected an overflow opcode");
14676
14677 EVT ResVT = N->getValueType(0);
14678 EVT OvVT = N->getValueType(1);
14679 EVT ResEltVT = ResVT.getVectorElementType();
14680 EVT OvEltVT = OvVT.getVectorElementType();
14681 SDLoc dl(N);
14682
14683 // If ResNE is 0, fully unroll the vector op.
14684 unsigned NE = ResVT.getVectorNumElements();
14685 if (ResNE == 0)
14686 ResNE = NE;
14687 else if (NE > ResNE)
14688 NE = ResNE;
14689
14690 SmallVector<SDValue, 8> LHSScalars;
14691 SmallVector<SDValue, 8> RHSScalars;
14692 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
14693 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
14694
14695 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
14696 SDVTList VTs = getVTList(ResEltVT, SVT);
14697 SmallVector<SDValue, 8> ResScalars;
14698 SmallVector<SDValue, 8> OvScalars;
14699 for (unsigned i = 0; i < NE; ++i) {
14700 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
14701 SDValue Ov =
14702 getSelect(dl, OvEltVT, Res.getValue(1),
14703 getBoolConstant(true, dl, OvEltVT, ResVT),
14704 getConstant(0, dl, OvEltVT));
14705
14706 ResScalars.push_back(Res);
14707 OvScalars.push_back(Ov);
14708 }
14709
14710 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
14711 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
14712
14713 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
14714 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
14715 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
14716 getBuildVector(NewOvVT, dl, OvScalars));
14717}
14718
14721 TypeSize Bytes, int Dist,
14722 const SelectionDAG &DAG) {
14723 if (LS->isVolatile() || Base->isVolatile())
14724 return false;
14725 // TODO: probably too restrictive for atomics, revisit
14726 if (!LS->isSimple())
14727 return false;
14728 if (LS->isIndexed() || Base->isIndexed())
14729 return false;
14730 if (LS->getChain() != Base->getChain())
14731 return false;
14732 EVT VT = LS->getMemoryVT();
14733 if (!VT.getScalarType().isByteSized())
14734 return false;
14735 if (VT.getStoreSize() != Bytes)
14736 return false;
14737
14738 auto BaseLocDecomp = BaseIndexOffset::match(Base, DAG);
14739 auto LocDecomp = BaseIndexOffset::match(LS, DAG);
14741 if (!BaseLocDecomp.equalBaseIndex(LocDecomp, DAG, Offset))
14742 return false;
14743
14744 int64_t ExpectedOffset = Dist * (int64_t)Bytes.getKnownMinValue();
14745 if (Bytes.isFixed())
14746 return Offset == StackOffset::getFixed(ExpectedOffset);
14747 return Offset == StackOffset::getScalable(ExpectedOffset);
14748}
14749
14752 TypeSize Bytes,
14753 int Dist) const {
14754 return areNonVolatileConsecutiveLoadsOrStores(LD, Base, Bytes, Dist, *this);
14755}
14756
14759 TypeSize Bytes,
14760 int Dist) const {
14761 return areNonVolatileConsecutiveLoadsOrStores(ST, Base, Bytes, Dist, *this);
14762}
14763
14764/// InferPtrAlignment - Infer alignment of a load / store address. Return
14765/// std::nullopt if it cannot be inferred.
14767 // If this is a GlobalAddress + cst, return the alignment.
14768 const GlobalValue *GV = nullptr;
14769 int64_t GVOffset = 0;
14770 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
14771 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14772 KnownBits Known(PtrWidth);
14774 unsigned AlignBits = Known.countMinTrailingZeros();
14775 if (AlignBits)
14776 return commonAlignment(Align::fromLog2(std::min(31U, AlignBits)),
14777 GVOffset);
14778 }
14779
14780 // If this is a direct reference to a stack slot, use information about the
14781 // stack slot's alignment.
14782 int FrameIdx = INT_MIN;
14783 int64_t FrameOffset = 0;
14785 FrameIdx = FI->getIndex();
14786 } else if (isBaseWithConstantOffset(Ptr) &&
14788 // Handle FI+Cst
14789 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
14790 FrameOffset = Ptr.getConstantOperandVal(1);
14791 }
14792
14793 if (FrameIdx != INT_MIN) {
14795 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
14796 }
14797
14798 return std::nullopt;
14799}
14800
14801/// Split the scalar node with EXTRACT_ELEMENT using the provided
14802/// VTs and return the low/high part.
14803std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14804 const SDLoc &DL,
14805 const EVT &LoVT,
14806 const EVT &HiVT) {
14807 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14808 "Split node must be a scalar type");
14809 SDValue Lo =
14811 SDValue Hi =
14813 return std::make_pair(Lo, Hi);
14814}
14815
14816/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14817/// which is split (or expanded) into two not necessarily identical pieces.
14818std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14819 // Currently all types are split in half.
14820 EVT LoVT, HiVT;
14821 if (!VT.isVector())
14822 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
14823 else
14824 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
14825
14826 return std::make_pair(LoVT, HiVT);
14827}
14828
14829/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14830/// type, dependent on an enveloping VT that has been split into two identical
14831/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14832std::pair<EVT, EVT>
14834 bool *HiIsEmpty) const {
14835 EVT EltTp = VT.getVectorElementType();
14836 // Examples:
14837 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14838 // custom VL=9 with enveloping VL=8/8 yields 8/1
14839 // custom VL=10 with enveloping VL=8/8 yields 8/2
14840 // etc.
14841 ElementCount VTNumElts = VT.getVectorElementCount();
14842 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14843 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14844 "Mixing fixed width and scalable vectors when enveloping a type");
14845 EVT LoVT, HiVT;
14846 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14847 LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
14848 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
14849 *HiIsEmpty = false;
14850 } else {
14851 // Flag that hi type has zero storage size, but return split envelop type
14852 // (this would be easier if vector types with zero elements were allowed).
14853 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
14854 HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
14855 *HiIsEmpty = true;
14856 }
14857 return std::make_pair(LoVT, HiVT);
14858}
14859
14860/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14861/// low/high part.
14862std::pair<SDValue, SDValue>
14863SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14864 const EVT &HiVT) {
14865 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14866 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14867 "Splitting vector with an invalid mixture of fixed and scalable "
14868 "vector types");
14870 N.getValueType().getVectorMinNumElements() &&
14871 "More vector elements requested than available!");
14872 SDValue Lo, Hi;
14873 Lo = getExtractSubvector(DL, LoVT, N, 0);
14874 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14875 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14876 // IDX with the runtime scaling factor of the result vector type. For
14877 // fixed-width result vectors, that runtime scaling factor is 1.
14879 return std::make_pair(Lo, Hi);
14880}
14881
14882std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14883 const SDLoc &DL) {
14884 // Split the vector length parameter.
14885 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14886 EVT VT = N.getValueType();
14888 "Expecting the mask to be an evenly-sized vector");
14889 SDValue HalfNumElts = getElementCount(
14891 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);
14892 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);
14893 return std::make_pair(Lo, Hi);
14894}
14895
14896/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14898 EVT VT = N.getValueType();
14899 EVT WideVT = VT.changeElementCount(
14900 *getContext(),
14902 return getInsertSubvector(DL, getPOISON(WideVT), N, 0);
14903}
14904
14907 unsigned Start, unsigned Count,
14908 EVT EltVT) {
14909 EVT VT = Op.getValueType();
14910 if (Count == 0)
14912 if (EltVT == EVT())
14913 EltVT = VT.getVectorElementType();
14914 SDLoc SL(Op);
14915 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14916 Args.push_back(getExtractVectorElt(SL, EltVT, Op, i));
14917 }
14918}
14919
14920// getAddressSpace - Return the address space this GlobalAddress belongs to.
14922 return getGlobal()->getType()->getAddressSpace();
14923}
14924
14927 return Val.MachineCPVal->getType();
14928 return Val.ConstVal->getType();
14929}
14930
14931bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14932 unsigned &SplatBitSize,
14933 bool &HasAnyUndefs,
14934 unsigned MinSplatBits,
14935 bool IsBigEndian) const {
14936 EVT VT = getValueType(0);
14937 assert(VT.isVector() && "Expected a vector type");
14938 unsigned VecWidth = VT.getSizeInBits();
14939 if (MinSplatBits > VecWidth)
14940 return false;
14941
14942 // FIXME: The widths are based on this node's type, but build vectors can
14943 // truncate their operands.
14944 SplatValue = APInt(VecWidth, 0);
14945 SplatUndef = APInt(VecWidth, 0);
14946
14947 // Get the bits. Bits with undefined values (when the corresponding element
14948 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14949 // in SplatValue. If any of the values are not constant, give up and return
14950 // false.
14951 unsigned int NumOps = getNumOperands();
14952 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14953 unsigned EltWidth = VT.getScalarSizeInBits();
14954
14955 for (unsigned j = 0; j < NumOps; ++j) {
14956 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14957 SDValue OpVal = getOperand(i);
14958 unsigned BitPos = j * EltWidth;
14959
14960 if (OpVal.isUndef())
14961 SplatUndef.setBits(BitPos, BitPos + EltWidth);
14962 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
14963 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
14964 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
14965 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
14966 else
14967 return false;
14968 }
14969
14970 // The build_vector is all constants or undefs. Find the smallest element
14971 // size that splats the vector.
14972 HasAnyUndefs = (SplatUndef != 0);
14973
14974 // FIXME: This does not work for vectors with elements less than 8 bits.
14975 while (VecWidth > 8) {
14976 // If we can't split in half, stop here.
14977 if (VecWidth & 1)
14978 break;
14979
14980 unsigned HalfSize = VecWidth / 2;
14981 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
14982 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
14983 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
14984 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
14985
14986 // If the two halves do not match (ignoring undef bits), stop here.
14987 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14988 MinSplatBits > HalfSize)
14989 break;
14990
14991 SplatValue = HighValue | LowValue;
14992 SplatUndef = HighUndef & LowUndef;
14993
14994 VecWidth = HalfSize;
14995 }
14996
14997 // FIXME: The loop above only tries to split in halves. But if the input
14998 // vector for example is <3 x i16> it wouldn't be able to detect a
14999 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
15000 // optimizations. I guess that back in the days when this helper was created
15001 // vectors normally was power-of-2 sized.
15002
15003 SplatBitSize = VecWidth;
15004 return true;
15005}
15006
15008 BitVector *UndefElements) const {
15009 unsigned NumOps = getNumOperands();
15010 if (UndefElements) {
15011 UndefElements->clear();
15012 UndefElements->resize(NumOps);
15013 }
15014 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
15015 if (!DemandedElts)
15016 return SDValue();
15017 SDValue Splatted;
15018 for (unsigned i = 0; i != NumOps; ++i) {
15019 if (!DemandedElts[i])
15020 continue;
15021 SDValue Op = getOperand(i);
15022 if (Op.isUndef()) {
15023 if (UndefElements)
15024 (*UndefElements)[i] = true;
15025 } else if (!Splatted) {
15026 Splatted = Op;
15027 } else if (Splatted != Op) {
15028 return SDValue();
15029 }
15030 }
15031
15032 if (!Splatted) {
15033 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
15034 assert(getOperand(FirstDemandedIdx).isUndef() &&
15035 "Can only have a splat without a constant for all undefs.");
15036 return getOperand(FirstDemandedIdx);
15037 }
15038
15039 return Splatted;
15040}
15041
15043 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
15044 return getSplatValue(DemandedElts, UndefElements);
15045}
15046
15048 SmallVectorImpl<SDValue> &Sequence,
15049 BitVector *UndefElements) const {
15050 unsigned NumOps = getNumOperands();
15051 Sequence.clear();
15052 if (UndefElements) {
15053 UndefElements->clear();
15054 UndefElements->resize(NumOps);
15055 }
15056 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
15057 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
15058 return false;
15059
15060 // Set the undefs even if we don't find a sequence (like getSplatValue).
15061 if (UndefElements)
15062 for (unsigned I = 0; I != NumOps; ++I)
15063 if (DemandedElts[I] && getOperand(I).isUndef())
15064 (*UndefElements)[I] = true;
15065
15066 // Iteratively widen the sequence length looking for repetitions.
15067 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
15068 Sequence.append(SeqLen, SDValue());
15069 for (unsigned I = 0; I != NumOps; ++I) {
15070 if (!DemandedElts[I])
15071 continue;
15072 SDValue &SeqOp = Sequence[I % SeqLen];
15074 if (Op.isUndef()) {
15075 if (!SeqOp)
15076 SeqOp = Op;
15077 continue;
15078 }
15079 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
15080 Sequence.clear();
15081 break;
15082 }
15083 SeqOp = Op;
15084 }
15085 if (!Sequence.empty())
15086 return true;
15087 }
15088
15089 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
15090 return false;
15091}
15092
15094 BitVector *UndefElements) const {
15095 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
15096 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
15097}
15098
15101 BitVector *UndefElements) const {
15103 getSplatValue(DemandedElts, UndefElements));
15104}
15105
15108 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
15109}
15110
15113 BitVector *UndefElements) const {
15115 getSplatValue(DemandedElts, UndefElements));
15116}
15117
15122
15123int32_t
15125 uint32_t BitWidth) const {
15126 if (ConstantFPSDNode *CN =
15128 bool IsExact;
15129 APSInt IntVal(BitWidth);
15130 const APFloat &APF = CN->getValueAPF();
15131 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
15132 APFloat::opOK ||
15133 !IsExact)
15134 return -1;
15135
15136 return IntVal.exactLogBase2();
15137 }
15138 return -1;
15139}
15140
15142 bool IsLittleEndian, unsigned DstEltSizeInBits,
15143 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
15144 // Early-out if this contains anything but Undef/Constant/ConstantFP.
15145 if (!isConstant())
15146 return false;
15147
15148 unsigned NumSrcOps = getNumOperands();
15149 unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();
15150 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15151 "Invalid bitcast scale");
15152
15153 // If the elements don't pack cleanly, don't attempt to recast.
15154 // TODO: Add handling (e.g. bitcast <33 x i8> --> <8 x i33>)
15155 if ((SrcEltSizeInBits % DstEltSizeInBits) != 0 &&
15156 (DstEltSizeInBits % SrcEltSizeInBits) != 0)
15157 return false;
15158
15159 // Extract raw src bits.
15160 SmallVector<APInt> SrcBitElements(NumSrcOps,
15161 APInt::getZero(SrcEltSizeInBits));
15162 BitVector SrcUndeElements(NumSrcOps, false);
15163
15164 for (unsigned I = 0; I != NumSrcOps; ++I) {
15166 if (Op.isUndef()) {
15167 SrcUndeElements.set(I);
15168 continue;
15169 }
15170 auto *CInt = dyn_cast<ConstantSDNode>(Op);
15171 auto *CFP = dyn_cast<ConstantFPSDNode>(Op);
15172 assert((CInt || CFP) && "Unknown constant");
15173 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
15174 : CFP->getValueAPF().bitcastToAPInt();
15175 }
15176
15177 // Recast to dst width.
15178 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
15179 SrcBitElements, UndefElements, SrcUndeElements);
15180 return true;
15181}
15182
15183void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
15184 unsigned DstEltSizeInBits,
15185 SmallVectorImpl<APInt> &DstBitElements,
15186 ArrayRef<APInt> SrcBitElements,
15187 BitVector &DstUndefElements,
15188 const BitVector &SrcUndefElements) {
15189 unsigned NumSrcOps = SrcBitElements.size();
15190 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
15191 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15192 "Invalid bitcast scale");
15193 assert(((SrcEltSizeInBits % DstEltSizeInBits) == 0 ||
15194 (DstEltSizeInBits % SrcEltSizeInBits) == 0) &&
15195 "Unsupported bitcast scale");
15196 assert(NumSrcOps == SrcUndefElements.size() && "Vector size mismatch");
15197
15198 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
15199 DstUndefElements.clear();
15200 DstUndefElements.resize(NumDstOps, false);
15201 DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));
15202
15203 // Concatenate src elements constant bits together into dst element.
15204 if (SrcEltSizeInBits <= DstEltSizeInBits) {
15205 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
15206 for (unsigned I = 0; I != NumDstOps; ++I) {
15207 DstUndefElements.set(I);
15208 APInt &DstBits = DstBitElements[I];
15209 for (unsigned J = 0; J != Scale; ++J) {
15210 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15211 if (SrcUndefElements[Idx])
15212 continue;
15213 DstUndefElements.reset(I);
15214 const APInt &SrcBits = SrcBitElements[Idx];
15215 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
15216 "Illegal constant bitwidths");
15217 DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);
15218 }
15219 }
15220 return;
15221 }
15222
15223 // Split src element constant bits into dst elements.
15224 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
15225 for (unsigned I = 0; I != NumSrcOps; ++I) {
15226 if (SrcUndefElements[I]) {
15227 DstUndefElements.set(I * Scale, (I + 1) * Scale);
15228 continue;
15229 }
15230 const APInt &SrcBits = SrcBitElements[I];
15231 for (unsigned J = 0; J != Scale; ++J) {
15232 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15233 APInt &DstBits = DstBitElements[Idx];
15234 DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
15235 }
15236 }
15237}
15238
15240 for (const SDValue &Op : op_values()) {
15241 unsigned Opc = Op.getOpcode();
15242 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
15243 return false;
15244 }
15245 return true;
15246}
15247
15248std::optional<std::pair<APInt, APInt>>
15250 unsigned NumOps = getNumOperands();
15251 if (NumOps < 2)
15252 return std::nullopt;
15253
15254 unsigned EltSize = getValueType(0).getScalarSizeInBits();
15255 APInt Start, Stride;
15256 int FirstIdx = -1, SecondIdx = -1;
15257
15258 // Find the first two non-undef constant elements to determine Start and
15259 // Stride, then verify all remaining elements match the sequence.
15260 for (unsigned I = 0; I < NumOps; ++I) {
15262 if (Op->isUndef())
15263 continue;
15264 if (!isa<ConstantSDNode>(Op))
15265 return std::nullopt;
15266
15267 APInt Val = getConstantOperandAPInt(I).trunc(EltSize);
15268 if (FirstIdx < 0) {
15269 FirstIdx = I;
15270 Start = Val;
15271 } else if (SecondIdx < 0) {
15272 SecondIdx = I;
15273 // Compute stride using modular arithmetic. Simple division would handle
15274 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
15275 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
15276 // Note that modular arithmetic is agnostic to signed/unsigned.
15277 unsigned IdxDiff = I - FirstIdx;
15278 APInt ValDiff = Val - Start;
15279
15280 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
15281 unsigned CommonPow2Bits = llvm::countr_zero(IdxDiff);
15282 if (ValDiff.countr_zero() < CommonPow2Bits)
15283 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
15284 IdxDiff >>= CommonPow2Bits;
15285 ValDiff.lshrInPlace(CommonPow2Bits);
15286
15287 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
15288 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
15289 // one, but we could try all candidates to handle more cases.
15290 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
15291 if (Stride.isZero())
15292 return std::nullopt;
15293
15294 // Step 3: Adjust Start based on the first defined element's index.
15295 Start -= Stride * FirstIdx;
15296 } else {
15297 // Verify this element matches the sequence.
15298 if (Val != Start + Stride * I)
15299 return std::nullopt;
15300 }
15301 }
15302
15303 // Need at least two defined elements.
15304 if (SecondIdx < 0)
15305 return std::nullopt;
15306
15307 return std::make_pair(Start, Stride);
15308}
15309
15311 // Find the first non-undef value in the shuffle mask.
15312 unsigned i, e;
15313 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
15314 /* search */;
15315
15316 // If all elements are undefined, this shuffle can be considered a splat
15317 // (although it should eventually get simplified away completely).
15318 if (i == e)
15319 return true;
15320
15321 // Make sure all remaining elements are either undef or the same as the first
15322 // non-undef value.
15323 for (int Idx = Mask[i]; i != e; ++i)
15324 if (Mask[i] >= 0 && Mask[i] != Idx)
15325 return false;
15326 return true;
15327}
15328
15329// Returns true if it is a constant integer BuildVector or constant integer,
15330// possibly hidden by a bitcast.
15332 SDValue N, bool AllowOpaques) const {
15334
15335 if (auto *C = dyn_cast<ConstantSDNode>(N))
15336 return AllowOpaques || !C->isOpaque();
15337
15339 return true;
15340
15341 // Treat a GlobalAddress supporting constant offset folding as a
15342 // constant integer.
15343 if (auto *GA = dyn_cast<GlobalAddressSDNode>(N))
15344 if (GA->getOpcode() == ISD::GlobalAddress &&
15345 TLI->isOffsetFoldingLegal(GA))
15346 return true;
15347
15348 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15349 isa<ConstantSDNode>(N.getOperand(0)))
15350 return true;
15351 return false;
15352}
15353
15354// Returns true if it is a constant float BuildVector or constant float.
15357 return true;
15358
15360 return true;
15361
15362 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15363 isa<ConstantFPSDNode>(N.getOperand(0)))
15364 return true;
15365
15366 return false;
15367}
15368
15369std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
15370 ConstantSDNode *Const =
15371 isConstOrConstSplat(N, false, /*AllowTruncation=*/true);
15372 if (!Const)
15373 return std::nullopt;
15374
15375 EVT VT = N.getValueType();
15376 const APInt CVal = Const->getAPIntValue().trunc(VT.getScalarSizeInBits());
15377 switch (TLI->getBooleanContents(VT)) {
15379 if (CVal.isOne())
15380 return true;
15381 if (CVal.isZero())
15382 return false;
15383 return std::nullopt;
15385 if (CVal.isAllOnes())
15386 return true;
15387 if (CVal.isZero())
15388 return false;
15389 return std::nullopt;
15391 return CVal[0];
15392 }
15393 llvm_unreachable("Unknown BooleanContent enum");
15394}
15395
15396void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
15397 assert(!Node->OperandList && "Node already has operands");
15399 "too many operands to fit into SDNode");
15400 SDUse *Ops = OperandRecycler.allocate(
15401 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
15402
15403 bool IsDivergent = false;
15404 for (unsigned I = 0; I != Vals.size(); ++I) {
15405 Ops[I].setUser(Node);
15406 Ops[I].setInitial(Vals[I]);
15407 EVT VT = Ops[I].getValueType();
15408
15409 // Skip Chain. It does not carry divergence.
15410 if (VT != MVT::Other &&
15411 (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
15412 Ops[I].getNode()->isDivergent()) {
15413 IsDivergent = true;
15414 }
15415 }
15416 Node->NumOperands = Vals.size();
15417 Node->OperandList = Ops;
15418 if (!TLI->isSDNodeAlwaysUniform(Node)) {
15419 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
15420 Node->SDNodeBits.IsDivergent = IsDivergent;
15421 }
15422 checkForCycles(Node);
15423}
15424
15427 size_t Limit = SDNode::getMaxNumOperands();
15428 while (Vals.size() > Limit) {
15429 unsigned SliceIdx = Vals.size() - Limit;
15430 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
15431 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
15432 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
15433 Vals.emplace_back(NewTF);
15434 }
15435 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
15436}
15437
15439 EVT VT, SDNodeFlags Flags) {
15440 switch (Opcode) {
15441 default:
15442 return SDValue();
15443 case ISD::ADD:
15444 case ISD::OR:
15445 case ISD::XOR:
15446 case ISD::UMAX:
15447 case ISD::MUL:
15448 case ISD::AND:
15449 case ISD::UMIN:
15450 case ISD::SMAX:
15451 case ISD::SMIN:
15453 VT);
15454 case ISD::FADD:
15455 // If flags allow, prefer positive zero since it's generally cheaper
15456 // to materialize on most targets.
15457 return getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15458 case ISD::FMUL:
15459 return getConstantFP(1.0, DL, VT);
15460 case ISD::FMINNUM:
15461 case ISD::FMAXNUM:
15462 case ISD::FMINIMUMNUM:
15463 case ISD::FMAXIMUMNUM: {
15464 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
15465 // depending on fast-math flags (FMF).
15466 const fltSemantics &Semantics = VT.getFltSemantics();
15467 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
15468 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
15469 APFloat::getLargest(Semantics);
15470 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
15471 NeutralAF.changeSign();
15472
15473 return getConstantFP(NeutralAF, DL, VT);
15474 }
15475 case ISD::FMINIMUM:
15476 case ISD::FMAXIMUM: {
15477 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15478 const fltSemantics &Semantics = VT.getFltSemantics();
15479 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
15480 : APFloat::getLargest(Semantics);
15481 if (Opcode == ISD::FMAXIMUM)
15482 NeutralAF.changeSign();
15483
15484 return getConstantFP(NeutralAF, DL, VT);
15485 }
15486
15487 }
15488}
15489
15491 SDValue Acc, SDValue LHS,
15492 SDValue RHS) {
15493 EVT AccVT = Acc.getValueType();
15494 if (AccVT.isFloatingPoint()) {
15495 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15496 SDValue NegRHS = getNode(ISD::FNEG, DL, RHS.getValueType(), RHS);
15497 return getNode(Opc, DL, AccVT, Acc, LHS, NegRHS);
15498 }
15501 "Unexpected opcode");
15502 SDValue NegAcc = getNegative(Acc, DL, AccVT);
15503 SDValue MLA = getNode(Opc, DL, AccVT, NegAcc, LHS, RHS);
15504 return getNegative(MLA, DL, AccVT);
15505}
15506
15507/// Helper used to make a call to a library function that has one argument of
15508/// pointer type.
15509///
15510/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15511/// used to get or set floating-point state. They have one argument of pointer
15512/// type, which points to the memory region containing bits of the
15513/// floating-point state. The value returned by such function is ignored in the
15514/// created call.
15515///
15516/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15517/// \param Ptr Pointer used to save/load state.
15518/// \param InChain Ingoing token chain.
15519/// \param Node Node being legalized
15520/// \returns Outgoing chain token.
15522 SDValue InChain, SDNode *Node) {
15523 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15524 RTLIB::LibcallImpl LibcallImpl =
15525 Libcalls->getLibcallImpl(static_cast<RTLIB::Libcall>(LibFunc));
15526 if (LibcallImpl == RTLIB::Unsupported) {
15527 getContext()->emitError(Twine("no libcall available for ") +
15528 Node->getOperationName(this));
15529 return InChain;
15530 }
15531
15533 Args.emplace_back(Ptr, Ptr.getValueType().getTypeForEVT(*getContext()));
15534 SDValue Callee =
15535 getExternalSymbol(LibcallImpl, TLI->getPointerTy(getDataLayout()));
15537 CLI.setDebugLoc(SDLoc(Node))
15538 .setChain(InChain)
15539 .setLibCallee(Libcalls->getLibcallImplCallingConv(LibcallImpl),
15540 Type::getVoidTy(*getContext()), Callee, std::move(Args));
15541 return TLI->LowerCallTo(CLI).second;
15542}
15543
15545 assert(From && To && "Invalid SDNode; empty source SDValue?");
15546 auto I = SDEI.find(From);
15547 if (I == SDEI.end())
15548 return;
15549
15550 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15551 // the iterator, hence the need to make a copy to prevent a use-after-free.
15552 NodeExtraInfo NEI = I->second;
15553 if (LLVM_LIKELY(!NEI.PCSections)) {
15554 // No deep copy required for the types of extra info set.
15555 //
15556 // FIXME: Investigate if other types of extra info also need deep copy. This
15557 // depends on the types of nodes they can be attached to: if some extra info
15558 // is only ever attached to nodes where a replacement To node is always the
15559 // node where later use and propagation of the extra info has the intended
15560 // semantics, no deep copy is required.
15561 SDEI[To] = std::move(NEI);
15562 return;
15563 }
15564
15565 const SDNode *EntrySDN = getEntryNode().getNode();
15566
15567 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15568 // through the replacement of From with To. Otherwise, replacements of a node
15569 // (From) with more complex nodes (To and its operands) may result in lost
15570 // extra info where the root node (To) is insignificant in further propagating
15571 // and using extra info when further lowering to MIR.
15572 //
15573 // In the first step pre-populate the visited set with the nodes reachable
15574 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15575 // DAG that is not new and should be left untouched.
15576 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15577 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15578 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15579 if (MaxDepth == 0) {
15580 // Remember this node in case we need to increase MaxDepth and continue
15581 // populating FromReach from this node.
15582 Leafs.emplace_back(N);
15583 return;
15584 }
15585 if (!FromReach.insert(N).second)
15586 return;
15587 for (const SDValue &Op : N->op_values())
15588 Self(Self, Op.getNode(), MaxDepth - 1);
15589 };
15590
15591 // Copy extra info to To and all its transitive operands (that are new).
15593 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15594 if (FromReach.contains(N))
15595 return true;
15596 if (!Visited.insert(N).second)
15597 return true;
15598 if (EntrySDN == N)
15599 return false;
15600 for (const SDValue &Op : N->op_values()) {
15601 if (N == To && Op.getNode() == EntrySDN) {
15602 // Special case: New node's operand is the entry node; just need to
15603 // copy extra info to new node.
15604 break;
15605 }
15606 if (!Self(Self, Op.getNode()))
15607 return false;
15608 }
15609 // Copy only if entry node was not reached.
15610 SDEI[N] = std::move(NEI);
15611 return true;
15612 };
15613
15614 // We first try with a lower MaxDepth, assuming that the path to common
15615 // operands between From and To is relatively short. This significantly
15616 // improves performance in the common case. The initial MaxDepth is big
15617 // enough to avoid retry in the common case; the last MaxDepth is large
15618 // enough to avoid having to use the fallback below (and protects from
15619 // potential stack exhaustion from recursion).
15620 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15621 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15622 // StartFrom is the previous (or initial) set of leafs reachable at the
15623 // previous maximum depth.
15625 std::swap(StartFrom, Leafs);
15626 for (const SDNode *N : StartFrom)
15627 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15628 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15629 return;
15630 // This should happen very rarely (reached the entry node).
15631 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15632 assert(!Leafs.empty());
15633 }
15634
15635 // This should not happen - but if it did, that means the subgraph reachable
15636 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15637 // could not visit all reachable common operands. Consequently, we were able
15638 // to reach the entry node.
15639 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15640 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15641 // Best-effort fallback if assertions disabled.
15642 SDEI[To] = std::move(NEI);
15643}
15644
15646 assert(VT.isVector() && "Can only unroll vector types!");
15647 if (VT.isFixedLengthVector())
15648 return VT.getVectorNumElements();
15649
15650 const MachineFunction &MF = getMachineFunction();
15651 const Function &F = MF.getFunction();
15652
15653 APInt MaxVScale = getVScaleRange(&F, sizeof(unsigned) * 8).getUnsignedMax();
15654 if (MaxVScale.ugt(getSelectionDAGOptions().vscale_unroll_limit))
15655 return 0;
15656
15657 bool Overflow;
15658 APInt MinNElts(sizeof(unsigned) * 8, VT.getVectorMinNumElements());
15659 APInt MaxNElts = MinNElts.umul_ov(MaxVScale, Overflow);
15660 if (Overflow)
15661 return 0;
15662
15663 return MaxNElts.getZExtValue();
15664}
15665
15667 ArrayRef<SDValue> Scalars) {
15668 assert(Scalars.size() == getMaxRuntimeNumElements(VT) &&
15669 "Element count mismatch!");
15670 if (VT.isFixedLengthVector())
15671 return getBuildVector(VT, DL, Scalars);
15672
15673 SDValue Vec = getPOISON(VT);
15674 // Iterate in reverse so result remains poison until we encounter a lane that
15675 // exists, after which all lower-numbered lanes must also exist.
15676 for (unsigned IdxVal : reverse(seq(Scalars.size())))
15677 Vec = getInsertVectorElt(DL, Vec, Scalars[IdxVal], IdxVal);
15678
15679 return Vec;
15680}
15681
15682#ifndef NDEBUG
15683static void checkForCyclesHelper(const SDNode *N,
15686 const llvm::SelectionDAG *DAG) {
15687 // If this node has already been checked, don't check it again.
15688 if (Checked.count(N))
15689 return;
15690
15691 // If a node has already been visited on this depth-first walk, reject it as
15692 // a cycle.
15693 if (!Visited.insert(N).second) {
15694 errs() << "Detected cycle in SelectionDAG\n";
15695 dbgs() << "Offending node:\n";
15696 N->dumprFull(DAG); dbgs() << "\n";
15697 abort();
15698 }
15699
15700 for (const SDValue &Op : N->op_values())
15701 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15702
15703 Checked.insert(N);
15704 Visited.erase(N);
15705}
15706#endif
15707
15709 const llvm::SelectionDAG *DAG,
15710 bool force) {
15711#ifndef NDEBUG
15712 bool check = force;
15713#ifdef EXPENSIVE_CHECKS
15714 check = true;
15715#endif // EXPENSIVE_CHECKS
15716 if (check) {
15717 assert(N && "Checking nonexistent SDNode");
15720 checkForCyclesHelper(N, visited, checked, DAG);
15721 }
15722#endif // !NDEBUG
15723}
15724
15725void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15726 checkForCycles(DAG->getRoot().getNode(), DAG, force);
15727}
return SDValue()
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
static bool isConstant(const MachineInstr &MI)
constexpr LLT S1
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
Definition Compiler.h:621
#define LLVM_LIKELY(EXPR)
Definition Compiler.h:347
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
This file defines a hash set that can be used to remove duplication of nodes in a graph.
static MaybeAlign getAlign(Value *Ptr)
iv users
Definition IVUsers.cpp:48
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
#define T
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
#define P(N)
PowerPC Reduce CR logical Operation
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
Contains matchers for matching SelectionDAG nodes and values.
SI Fold Operands
const char * Msg
This file contains some templates that are useful if you are working with the STL at all.
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF, SelectionDAG &DAG)
static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL, EVT VT, Ty Quantity)
static std::pair< SDValue, SDValue > getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl, TargetLowering::ArgListTy &&Args, const CallInst *CI, RTLIB::Libcall Call, SelectionDAG *DAG, const TargetLowering *TLI)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N)
Serialize a node the way SDNodes were uniqued before SDNodeKey, to cross-check the typed comparison a...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static TargetLoweringBase::ArgListEntry getAtomicMemOpSizeArg(SelectionDAG *DAG, const SDLoc &dl, SDValue Size)
static bool hasSpecialNszSemantics(unsigned Opcode)
Return true if the standard nsz semantics may not apply for this opcode.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS, LSBaseSDNode *Base, TypeSize Bytes, int Dist, const SelectionDAG &DAG)
static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth)
static SDValue getMemIntrinsicSize(SelectionDAG &DAG, const SDLoc &dl, SDValue Size, unsigned DstAS, unsigned SrcAS)
The length of a memory intrinsic (e.g.
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, const DataLayout &DL, unsigned AS)
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static bool isInTailCallPositionWrapper(const CallInst *CI, const SelectionDAG *SelDAG, bool AllowReturnsFirstArg)
static bool keyMatches(const SDNodeKey &Key, const SDNode &N)
static bool gluePropagatesDivergence(const SDNode *Node)
Return true if a glue output should propagate divergence information.
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N, unsigned Opc)
If this is an SDNode with special info, add this info to the NodeID data.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static APInt getDemandAllEltsMask(SDValue V)
Construct a DemandedElts mask which demands all elements of V.
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, BatchAAResults *BatchAA, const MDNode *DstMemCacheHint, const MDNode *SrcMemCacheHint)
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This file describes how to lower LLVM code to machine code.
static void removeOperands(MachineInstr &MI, unsigned i)
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static unsigned getSize(unsigned Kind)
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition APFloat.h:351
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static const fltSemantics & BFloat()
Definition APFloat.h:303
static const fltSemantics & IEEEquad()
Definition APFloat.h:306
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:364
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:363
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
opStatus
IEEE-754R 7: Default exception handling.
Definition APFloat.h:377
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
bool isNegative() const
Definition APFloat.h:1583
bool isFinite() const
Definition APFloat.h:1588
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition APFloat.h:1451
bool isZero() const
Definition APFloat.h:1579
LLVM_READONLY bool isOne() const
Definition APFloat.h:1676
bool isLargest() const
Definition APFloat.h:1599
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1242
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition APFloat.h:1436
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
bool isPosZero() const
Definition APFloat.h:1594
bool isNegZero() const
Definition APFloat.h:1595
void changeSign()
Definition APFloat.h:1401
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Definition APFloat.h:1213
bool isInfinity() const
Definition APFloat.h:1580
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt umul_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:2018
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition APInt.h:1426
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition APInt.h:419
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition APInt.cpp:1078
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
APInt abs() const
Get the absolute value.
Definition APInt.h:1815
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
LLVM_ABI APInt reverseBits() const
Definition APInt.cpp:786
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition APInt.h:836
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition APInt.h:1170
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
Definition APInt.h:1648
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition APInt.h:1618
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:648
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
LLVM_ABI APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition APInt.cpp:1086
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition APInt.cpp:393
unsigned logBase2() const
Definition APInt.h:1781
LLVM_ABI APInt multiplicativeInverse() const
Definition APInt.cpp:1303
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:330
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition APInt.h:1387
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:875
LLVM_ABI APInt byteSwap() const
Definition APInt.cpp:764
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
void clearBits(unsigned LoBit, unsigned HiBit)
Clear the bits from LoBit (inclusive) to HiBit (exclusive) to 0.
Definition APInt.h:1437
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition APInt.cpp:478
bool isOne() const
Determine if this is a value of 1.
Definition APInt.h:385
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition APInt.h:860
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
An arbitrary precision integer that knows its signedness.
Definition APSInt.h:24
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
static Capacity get(size_t N)
Get the capacity of an array that can hold at least N elements.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
static LLVM_ABI BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, and fixed/scalable offsets.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
BitVector & reset()
Reset all bits in the bitvector.
Definition BitVector.h:410
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
Definition BitVector.h:356
void clear()
Removes all bits from the bitvector.
Definition BitVector.h:350
BitVector & set()
Set all bits in the bitvector.
Definition BitVector.h:367
bool none() const
Returns true if none of the bits are set.
Definition BitVector.h:208
size_type size() const
Returns the number of bits in this bitvector.
Definition BitVector.h:179
const BlockAddress * getBlockAddress() const
The address of a basic block.
Definition Constants.h:1088
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static LLVM_ABI void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
LLVM_ABI bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
LLVM_ABI ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
LLVM_ABI SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
LLVM_ABI ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
LLVM_ABI int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ABI std::optional< std::pair< APInt, APInt > > isArithmeticSequence() const
If this BuildVector is constant and represents an arithmetic sequence "<a, a+n, a+2n,...
LLVM_ABI bool isConstant() const
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
static LLVM_ABI bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
const APFloat & getValue() const
Definition Constants.h:464
This is the shared class of boolean and integer constants.
Definition Constants.h:87
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition Constants.h:162
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
MachineConstantPoolValue * getMachineCPVal() const
const Constant * getConstVal() const
LLVM_ABI Type * getType() const
This class represents a range of values.
PreferredRangeType
If represented precisely, the result of some range operations may consist of multiple disjoint ranges...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static LLVM_ABI ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
LLVM_ABI OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
LLVM_ABI OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
LLVM_ABI KnownBits toKnownBits() const
Return known bits for values in this range.
LLVM_ABI ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
LLVM_ABI OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
LLVM_ABI ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI ConstantRange multiply(const ConstantRange &Other, unsigned NoWrapKind=0) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
LLVM_ABI APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
LLVM_ABI OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition Constant.h:44
LLVM_ABI Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
LLVM_ABI Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
DWARF expression.
static LLVM_ABI ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isLittleEndian() const
Layout endianness...
Definition DataLayout.h:220
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
LLVM_ABI unsigned getPointerTypeSizeInBits(Type *) const
The pointer representation size in bits for this type.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
A debug info location.
Definition DebugLoc.h:126
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
const char * getSymbol() const
Insertion token: a failed lookup fills it in, the matching insert consumes it.
Definition FoldingSet.h:284
This class is used to gather all the unique data bits of a node.
Definition FoldingSet.h:162
Data structure describing the variable locations in a function.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
LLVM_ABI unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
This class is used to form a handle around another node that is persistent and is updated across invo...
const SDValue & getValue() const
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
bool isUnknown() const
KnownFPClass intersectWith(const KnownFPClass &RHS) const
static KnownFPClass getEmpty()
static LLVM_ABI KnownFPClass bitcast(const fltSemantics &FltSemantics, const KnownBits &Bits)
Report known values for a bitcast into a float with provided semantics.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Base class for LoadSDNode and StoreSDNode.
Tracks which library functions to use for a particular subtarget or function.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
This SDNode is used for LIFETIME_START/LIFETIME_END values.
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
Metadata node.
Definition Metadata.h:1081
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
Machine Value Type.
SimpleValueType SimpleTy
static LLVM_ABI MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
static MVT getIntegerVT(unsigned BitWidth)
Abstract base class for all machine specific constantpool value subclasses.
virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID)=0
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
size_t getNumMemOperands() const
Return the number of memory operands.
LLVM_ABI MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, PointerUnion< MachineMemOperand *, MachineMemOperand ** > memrefs)
Constructor that supports single or multiple MMOs.
PointerUnion< MachineMemOperand *, MachineMemOperand ** > MemRefs
Memory reference information.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
ArrayRef< MachineMemOperand * > memoperands() const
Return the memory operands for this node.
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition Metadata.h:64
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition Module.cpp:254
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
Class to represent pointers.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
unsigned getAddressSpace() const
Return the address space of the Pointer type.
A discriminated union of two or more pointer types, with the discriminator in the low bits of the poi...
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Keeps track of dbg_value information through SDISel.
LLVM_ABI void add(SDDbgValue *V, bool isParameter)
LLVM_ABI void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
LLVM_ABI void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isDivergent() const
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
iterator_range< use_iterator > uses()
MemSDNodeBitfields MemSDNodeBits
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
LLVM_ABI void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static LLVM_ABI bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
LLVM_ABI bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
std::optional< APInt > bitcastToAPInt() const
LLVM_ABI bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
LLVM_ABI bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
LLVM_ABI void DropOperands()
Release the operands and set this node to have zero operands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
Create an SDNode.
Represents a use of a SDNode.
SDNode * getUser()
This returns the SDNode that contains this Use.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
LLVM_ABI bool isOperandOf(const SDNode *N) const
Return true if the referenced return value is an operand of N.
SDValue()=default
LLVM_ABI bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const
Checks that the given target-specific node is valid. Aborts if it is not.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC)
LLVM_ABI Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
LLVM_ABI SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
LLVM_ABI std::pair< SDValue, SDValue > getMemccpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI)
Lower a memccpy operation into a target library call and return the resulting chain and call result a...
LLVM_ABI SDValue emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, const SDLoc &DL, SDValue Chain)
Emit a store/load combination to the stack.
LLVM_ABI bool isKnownNeverLogicalZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Test whether the given floating point SDValue (or all elements of it, if it is a vector) is known to ...
LLVM_ABI SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
LLVM_ABI SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
LLVM_ABI SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
LLVM_ABI std::pair< SDValue, SDValue > getStrlen(SDValue Chain, const SDLoc &dl, SDValue Src, const CallInst *CI)
Lower a strlen operation into a target library call and return the resulting chain and call result as...
LLVM_ABI SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
LLVM_ABI unsigned getHasPredecessorMaxSteps() const
LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl, SDNodeFlags Flags={})
Constant fold a setcc to true or false.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
LLVM_ABI SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, SDNode *Node)
Helper used to make a call to a library function that has one argument of pointer type.
LLVM_ABI std::optional< bool > isBoolConstant(SDValue N) const
Check if a value \op N is a constant using the target's BooleanContent for its type.
LLVM_ABI SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
LLVM_ABI ConstantRange computeConstantRange(SDValue Op, bool ForSigned, unsigned Depth=0) const
Determine the possible constant range of an integer or vector of integers.
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI void updateDivergence(SDNode *N)
LLVM_ABI SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags, bool AllowCommute=false)
Get the specified node if it's already available, or else return NULL.
LLVM_ABI SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
LLVM_ABI const BasicBlock * getBasicBlock() const
LLVM_ABI SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
LLVM_ABI SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
LLVM_ABI SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
LLVM_ABI SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
LLVM_ABI SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
LLVM_ABI SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
LLVM_ABI std::optional< unsigned > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, TypeSize Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
LLVM_ABI bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS, const SDNodeFlags Flags=SDNodeFlags())
Return an AddrSpaceCastSDNode.
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
LLVM_ABI bool cannotBeOrderedNegativeFP(SDValue Op) const
Test whether the given float value is known to be positive.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI bool calculateDivergence(SDNode *N)
LLVM_ABI std::pair< SDValue, SDValue > getStrcmp(SDValue Chain, const SDLoc &dl, SDValue S0, SDValue S1, const CallInst *CI)
Lower a strcmp operation into a target library call and return the resulting chain and call result as...
LLVM_ABI SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
LLVM_ABI SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
LLVM_ABI SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
LLVM_ABI bool canIgnoreSignBitOfZero(const SDUse &Use) const
Check if the user of Use produces the same result regardless of the sign of a zero in the used float ...
LLVM_ABI bool SignBitIsZeroFP(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero, for a floating-point value.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
LLVM_ABI SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
LLVM_ABI bool hasSwiftErrorArg() const
LLVM_ABI SDValue buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Scalars)
Returns a vector constructed from the scalar values in order.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
LLVM_ABI unsigned getMaxRuntimeNumElements(EVT VT) const
Returns the maximum runtime number of elements in VT if known, or 0 otherwise.
LLVM_ABI SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth=0) const
Return true if this function can prove that Op is never poison.
LLVM_ABI SDValue getIdentityElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) identity element for the given opcode, if it exists.
LLVM_ABI SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
LLVM_ABI SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
LLVM_ABI void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
LLVM_ABI SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
LLVM_ABI std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
LLVM_ABI SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI void DeleteNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
LLVM_ABI std::pair< SDValue, SDValue > getStrcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, const CallInst *CI)
Lower a strcpy operation into a target library call and return the resulting chain and call result as...
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI std::optional< unsigned > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
LLVM_ABI bool isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getPartialReduceMLS(unsigned Opc, const SDLoc &DL, SDValue Acc, SDValue LHS, SDValue RHS)
Get an expression that implements a partial multiply-subtract reduction.
LLVM_ABI SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
LLVM_ABI bool areNonVolatileConsecutiveStores(StoreSDNode *ST, StoreSDNode *Base, TypeSize Bytes, int Dist) const
Return true if stores are next to each other and can be merged.
LLVM_ABI ConstantRange computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned, unsigned Depth=0) const
Combine constant ranges from computeConstantRange() and computeKnownBits().
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
LLVM_ABI SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
LLVM_ABI SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
LLVM_ABI SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
LLVM_ABI void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachineMemOperand *MMO)
LLVM_ABI SDValue getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS)
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
LLVM_ABI std::pair< SDValue, SDValue > getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, const CallInst *CI)
Lower a memcmp operation into a target library call and return the resulting chain and call result as...
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
LLVM_ABI SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
LLVM_ABI OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
LLVM_ABI MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
LLVM_ABI void dump() const
Dump the textual format of this DAG.
LLVM_ABI bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
LLVM_ABI bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
SDValue getInsertVectorElt(const SDLoc &DL, SDValue Vec, SDValue Elt, unsigned Idx)
Insert Elt into Vec at offset Idx.
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, Register VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI KnownFPClass computeKnownFPClass(SDValue Op, FPClassTest InterestedClasses, unsigned Depth=0) const
Determine floating-point class information about Op.
LLVM_ABI bool isIdentityElement(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo, unsigned Depth=0) const
Returns true if V is an identity element of Opc with Flags.
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
LLVM_ABI bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
LLVM_ABI SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI std::optional< unsigned > getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
LLVM_ABI std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getMaskFromElementCount(const SDLoc &DL, EVT VT, ElementCount Len)
Return a vector with the first 'Len' lanes set to true and remaining lanes set to false.
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
LLVM_ABI SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
LLVM_ABI SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
LLVM_ABI SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the Fra...
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
LLVM_ABI unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
LLVM_ABI SDValue FoldConstantBuildVector(BuildVectorSDNode *BV, const SDLoc &DL, EVT DstEltVT)
Fold BUILD_VECTOR of constants/undefs to the destination type BUILD_VECTOR of constants/undefs elemen...
LLVM_ABI SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
LLVM_ABI SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
LLVM_ABI SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
LLVM_ABI SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
LLVM_ABI SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
LLVM_ABI SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
LLVM_ABI SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void getTopologicallyOrderedNodes(SmallVectorImpl< const SDNode * > &SortedNodes) const
Get all the nodes in their topological order without modifying any states.
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
LLVM_ABI std::pair< SDValue, SDValue > getStrstr(SDValue Chain, const SDLoc &dl, SDValue S0, SDValue S1, const CallInst *CI)
Lower a strstr operation into a target library call and return the resulting chain and call result as...
LLVM_ABI SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
LLVM_ABI OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getErrorMergeValues(ArrayRef< EVT > ResultTypes, SDValue Chain, const SDLoc &dl)
Return poison values for each of ResultTypes, substituting Chain for any result of type MVT::Other,...
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI bool canIgnoreSignBitOfZeroResult(SDValue Op) const
Check if all uses of Op (limited to checking two for compile-time performance) are insensitive to sig...
LLVM_ABI void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
const SelectionDAGOptions & getSelectionDAGOptions() const
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVM_ABI bool isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth=0) const
Test if the given fp value is known to be an integer power-of-2, either positive or negative.
LLVM_ABI OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
LLVM_ABI void init(MachineFunction &NewMF, const TargetLibraryInfo *LibraryInfo, const LibcallLoweringInfo *LibcallsInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI bool isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero=false, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
LLVM_ABI SDValue getDeactivationSymbol(const GlobalValue *GV)
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
LLVM_ABI bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
LLVM_ABI std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
LLVM_ABI SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
LLVM_ABI std::optional< ConstantRange > getValidShiftAmountRange(SDValue V, const APInt &DemandedElts, unsigned Depth) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
LLVM_ABI SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
LLVM_ABI bool isADDLike(SDValue Op, bool NoWrap=false) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
LLVM_ABI void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
LLVM_ABI SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
LLVM_ABI SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
static LLVM_ABI bool isSplatMask(ArrayRef< int > Mask)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
int64_t getFixed() const
Returns the fixed component of the stack.
Definition TypeSize.h:47
int64_t getScalable() const
Returns the scalable component of the stack.
Definition TypeSize.h:50
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
IntegerType * getSizeTType(const Module &M) const
Returns an IntegerType corresponding to size_t.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
BooleanContent
Enum that describes how the target represents true/false values.
std::vector< ArgListEntry > ArgListTy
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
const Triple & getTargetTriple() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, DriverKit, XROS, or bridgeOS).
Definition Triple.h:726
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:267
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
LLVM_ABI void set(Value *Val)
Definition Value.h:876
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
Value * getOperand(unsigned i) const
Definition User.h:207
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
Definition TypeSize.h:266
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
Definition TypeSize.h:177
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:238
A raw_ostream that writes to an std::string.
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt clmulr(const APInt &LHS, const APInt &RHS)
Perform a reversed carry-less multiply.
Definition APInt.cpp:3254
LLVM_ABI APInt mulhu(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on zero-extended operands.
Definition APInt.cpp:3184
LLVM_ABI APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
Definition APInt.cpp:3161
LLVM_ABI APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
Definition APInt.cpp:3151
LLVM_ABI APInt pext(const APInt &Val, const APInt &Mask)
Perform a "compress" operation, also known as pext or bext.
Definition APInt.cpp:3264
LLVM_ABI APInt fshr(const APInt &Hi, const APInt &Lo, const APInt &Shift)
Perform a funnel shift right.
Definition APInt.cpp:3235
LLVM_ABI APInt mulhs(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on sign-extended operands.
Definition APInt.cpp:3176
LLVM_ABI APInt avgSubFloorS(const APInt &C1, const APInt &C2)
Compute floor((C1 - C2) / 2) where C1 and C2 are treated as signed, so the subtraction is performed w...
Definition APInt.cpp:3166
LLVM_ABI APInt clmul(const APInt &LHS, const APInt &RHS)
Perform a carry-less multiply, also known as XOR multiplication, and return low-bits.
Definition APInt.cpp:3244
LLVM_ABI APInt pdep(const APInt &Val, const APInt &Mask)
Perform an "expand" operation, also known as pdep or bdep.
Definition APInt.cpp:3274
APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
Definition APInt.h:2294
LLVM_ABI APInt fshl(const APInt &Hi, const APInt &Lo, const APInt &Shift)
Perform a funnel shift left.
Definition APInt.cpp:3226
LLVM_ABI APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition APInt.cpp:3052
LLVM_ABI APInt clmulh(const APInt &LHS, const APInt &RHS)
Perform a carry-less multiply, and return high-bits.
Definition APInt.cpp:3259
APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
Definition APInt.h:2299
LLVM_ABI APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
Definition APInt.cpp:3146
LLVM_ABI APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
Definition APInt.cpp:3156
LLVM_ABI APInt avgSubFloorU(const APInt &C1, const APInt &C2)
Compute floor((C1 - C2) / 2) where C1 and C2 are treated as unsigned, so the subtraction is performed...
Definition APInt.cpp:3171
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
ISD namespace - This namespace contains an enum which represents all of the SelectionDAG node types a...
Definition ISDOpcodes.h:26
LLVM_ABI CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
LLVM_ABI bool isBuildVectorOfConstantFPSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
LLVM_ABI bool isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 ...
bool matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false, bool AllowTruncation=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ TargetConstantPool
Definition ISDOpcodes.h:191
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ ATOMIC_LOAD_FMINIMUMNUM
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ VECREDUCE_FMINIMUMNUM
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition ISDOpcodes.h:543
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
Definition ISDOpcodes.h:739
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ TargetBlockAddress
Definition ISDOpcodes.h:193
@ DEACTIVATION_SYMBOL
Untyped node storing deactivation symbol reference (DeactivationSymbolSDNode).
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:296
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ ATOMIC_LOAD_USUB_COND
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ GlobalAddress
Definition ISDOpcodes.h:90
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ STRICT_FMINIMUM
Definition ISDOpcodes.h:476
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
Definition ISDOpcodes.h:665
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:942
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
Definition ISDOpcodes.h:533
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ STRICT_PSEUDO_FMAX
Definition ISDOpcodes.h:465
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ GlobalTLSAddress
Definition ISDOpcodes.h:91
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ ATOMIC_LOAD_USUB_SAT
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ TargetExternalSymbol
Definition ISDOpcodes.h:192
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ TargetJumpTable
Definition ISDOpcodes.h:190
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
Definition ISDOpcodes.h:200
@ PARTIAL_REDUCE_FMLA
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ STRICT_PSEUDO_FMIN
Definition ISDOpcodes.h:464
@ TRUNCATE_SSAT_U
Definition ISDOpcodes.h:905
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition ISDOpcodes.h:859
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition ISDOpcodes.h:707
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition ISDOpcodes.h:71
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition ISDOpcodes.h:187
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ CTLS
Count leading redundant sign bits.
Definition ISDOpcodes.h:824
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ ATOMIC_LOAD_FMAXIMUM
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition ISDOpcodes.h:80
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ STRICT_FMAXIMUM
Definition ISDOpcodes.h:475
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition ISDOpcodes.h:50
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:226
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ TargetConstantFP
Definition ISDOpcodes.h:182
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ ATOMIC_LOAD_FMINIMUM
@ TargetFrameIndex
Definition ISDOpcodes.h:189
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition ISDOpcodes.h:931
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ ATOMIC_LOAD_FMAXIMUMNUM
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ ATOMIC_LOAD_UDEC_WRAP
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition ISDOpcodes.h:181
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition ISDOpcodes.h:510
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:306
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition ISDOpcodes.h:697
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ ExternalSymbol
Definition ISDOpcodes.h:95
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:715
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:953
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ VECREDUCE_FMINIMUM
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ VECREDUCE_SEQ_FMUL
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ ATOMIC_LOAD_UINC_WRAP
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ PARTIAL_REDUCE_SUMLA
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
Definition ISDOpcodes.h:645
@ SET_FPENV_MEM
Sets the current floating point environment.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ TRUNCATE_SSAT_S
TRUNCATE_[SU]SAT_[SU] - Truncate for saturated operand [SU] located in middle, prefix for SAT means i...
Definition ISDOpcodes.h:903
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
@ TRUNCATE_USAT_U
Definition ISDOpcodes.h:907
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:341
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ TargetGlobalTLSAddress
Definition ISDOpcodes.h:188
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
LLVM_ABI NodeType getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns the corresponding opcode with the opposi...
LLVM_ABI NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
LLVM_ABI bool isConstantSplatVectorAllOnes(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0...
LLVM_ABI bool isFreezeUndef(SDValue V)
Return true if the specified value is FREEZE(UNDEF).
bool isExtOpcode(unsigned Opcode)
LLVM_ABI NodeType getUnmaskedBinOpOpcode(unsigned MaskedOpc)
Given a MaskedOpc of ISD::MASKED_(U|S)(DIV|REM), returns the unmasked ISD::(U|S)(DIV|REM).
LLVM_ABI bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI bool allOperandsUndef(SDValue V)
Return true if the value has at least one operand and all operands of the specified value are ISD::UN...
LLVM_ABI std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
LLVM_ABI bool isBuildVectorAllOnes(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are ~0 or undef.
LLVM_ABI std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
LLVM_ABI bool isVectorShrinkable(SDValue V, unsigned NewEltSize, bool Signed)
Returns true if the specified value is a vector where all elements can be truncated to the specified ...
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LLVM_ABI std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
LLVM_ABI bool matchBinaryPredicate(SDValue LHS, SDValue RHS, const APInt &DemandedElts, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI std::optional< unsigned > getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
LLVM_ABI NodeType getInverseMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns ISD::(U|S)MAX and ISD::(U|S)MIN,...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
LLVM_ABI bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
LLVM_ABI bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool matchUnaryPredicate(SDValue Op, const APInt &DemandedElts, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTruncation=false)
Hook for matching ConstantSDNode predicate.
LLVM_ABI CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
LLVM_ABI Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
LLVM_ABI Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
LLVM_ABI Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDValue N, Pattern &&P)
LLVM_ABI unsigned rot(unsigned SrcSignBits, unsigned BitWidth, std::optional< APInt > RotAmt, bool IsRotateRight)
Compute the number of sign bits after rotating a value.
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition Dwarf.h:149
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
@ Offset
Definition DWP.cpp:577
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition Analysis.cpp:248
void fill(R &&Range, T &&Value)
Provide wrappers to std::fill which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1775
LLVM_ABI SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition Utils.cpp:1717
LLVM_ABI SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
LLVM_ABI bool isOneOrOneSplatFP(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant floating-point value, or a splatted vector of a constant float...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition APFloat.h:1816
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
auto cast_or_null(const Y &Val)
Definition Casting.h:714
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition Utils.cpp:1699
LLVM_ABI bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition APFloat.h:1728
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
Definition APFloat.h:1771
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI SDValue peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts)
Recursively peek through INSERT_VECTOR_ELT nodes, returning the source vector operand of V,...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_READONLY APFloat minimumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimumNumber semantics.
Definition APFloat.h:1802
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr std::underlying_type_t< Enum > to_underlying(Enum E)
Returns underlying integer value of an enum.
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
LLVM_ABI SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
bool includesPoison(UndefPoisonKind Kind)
Returns true if Kind includes the Poison bit.
Definition UndefPoison.h:27
LLVM_ABI bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Other
Any other memory.
Definition ModRef.h:68
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
bool includesUndef(UndefPoisonKind Kind)
Returns true if Kind includes the Undef bit.
Definition UndefPoison.h:33
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 minNum semantics.
Definition APFloat.h:1752
@ Sub
Subtraction of integers.
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
LLVM_ABI bool isNullConstantOrUndef(SDValue V)
Returns true if V is a constant integer zero or an UNDEF node.
IntPtrTy
Definition InstrProf.h:82
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:709
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
Definition STLExtras.h:1901
constexpr unsigned BitWidth
LLVM_ABI bool funcReturnsFirstArgOfCall(const CallInst &CI)
Returns true if the parent of CI returns CI's first argument after calling CI.
Definition Analysis.cpp:888
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isZeroOrZeroSplat(SDValue N, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Definition UndefPoison.h:20
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:572
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
LLVM_ABI bool isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs=false)
Return true if the value is a constant (+/-)0.0 floating-point value or a splatted vector thereof (wi...
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
Definition STLExtras.h:2162
LLVM_ABI void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition APFloat.h:1789
LLVM_READONLY APFloat maximumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximumNumber semantics.
Definition APFloat.h:1829
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:368
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:774
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
Definition Metadata.h:794
MDNode * TBAA
The tag for type-based alias analysis.
Definition Metadata.h:791
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
Definition Alignment.h:73
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
intptr_t getRawBits() const
Definition ValueTypes.h:562
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
EVT changeElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a type whose attributes match ourselves with the exception of the element count that ...
Definition ValueTypes.h:121
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition ValueTypes.h:294
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition ValueTypes.h:165
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition ValueTypes.h:346
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:499
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
static LLVM_ABI KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
static LLVM_ABI KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:106
bool isZero() const
Returns true if value is all zero.
Definition KnownBits.h:78
static LLVM_ABI KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
static LLVM_ABI KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition KnownBits.h:288
static LLVM_ABI std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition KnownBits.h:165
KnownBits byteSwap() const
Definition KnownBits.h:559
static LLVM_ABI KnownBits fshl(const KnownBits &LHS, const KnownBits &RHS, const APInt &Amt)
Compute known bits for fshl(LHS, RHS, Amt).
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition KnownBits.h:303
void setAllZero()
Make all bits known to be zero and discard any previous information.
Definition KnownBits.h:84
KnownBits reverseBits() const
Definition KnownBits.h:563
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
Definition KnownBits.h:247
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
static LLVM_ABI KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition KnownBits.h:176
void resetAll()
Resets the known state of all bits.
Definition KnownBits.h:72
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
Definition KnownBits.h:361
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
bool isNonZero() const
Returns true if this value is known to be non-zero.
Definition KnownBits.h:109
static LLVM_ABI KnownBits abdu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for abdu(LHS, RHS).
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
Definition KnownBits.h:239
static LLVM_ABI KnownBits pdep(const KnownBits &Val, const KnownBits &Mask)
Compute known bits for pdep(Val, Mask).
static LLVM_ABI KnownBits avgFloorU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorU.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition KnownBits.h:184
static LLVM_ABI KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
Definition KnownBits.h:200
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:146
static LLVM_ABI KnownBits fshr(const KnownBits &LHS, const KnownBits &RHS, const APInt &Amt)
Compute known bits for fshr(LHS, RHS, Amt).
static LLVM_ABI KnownBits abds(KnownBits LHS, KnownBits RHS)
Compute known bits for abds(LHS, RHS).
static LLVM_ABI KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static LLVM_ABI KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
static LLVM_ABI KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
static LLVM_ABI KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
Definition KnownBits.h:112
static LLVM_ABI KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
static LLVM_ABI KnownBits avgFloorS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorS.
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
Definition KnownBits.h:340
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:103
static LLVM_ABI KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
Definition KnownBits.cpp:54
static KnownBits sub(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from subtraction of LHS and RHS.
Definition KnownBits.h:376
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition KnownBits.h:294
static LLVM_ABI KnownBits avgCeilU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilU.
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
Definition KnownBits.h:171
static LLVM_ABI KnownBits clmul(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for clmul(LHS, RHS).
LLVM_ABI KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
static LLVM_ABI KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
static LLVM_ABI KnownBits pext(const KnownBits &Val, const KnownBits &Mask)
Compute known bits for pext(Val, Mask).
static LLVM_ABI KnownBits avgCeilS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilS.
LLVM IR metadata carried by a MachineMemOperand.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:135
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile)
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.
These are IR-level optimization flags that may be propagated to SDNodes.
static LLVM_ABI bool isEqual(const KeyTy &Key, const SDNode &N)
The key SelectionDAG uniques SDNodes by.
void AddPointer(const void *P)
SmallVector< SDValue, 0 > OpStorage
Backs Ops when the key is built from a node; empty otherwise.
void AddInteger(T I)
const EVT * VTs
ArrayRef< SDValue > Ops
FoldingSetNodeID Tail
SDNodeKey(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned int NumVTs
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)