LCOV - code coverage report
Current view: top level - include/llvm/CodeGen - SelectionDAGNodes.h (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 377 384 98.2 %
Date: 2018-06-17 00:07:59 Functions: 30 31 96.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file declares the SDNode class and derived classes, which are used to
      11             : // represent the nodes and operations present in a SelectionDAG.  These nodes
      12             : // and operations are machine code level operations, with some similarities to
      13             : // the GCC RTL representation.
      14             : //
      15             : // Clients should include the SelectionDAG.h file instead of this file directly.
      16             : //
      17             : //===----------------------------------------------------------------------===//
      18             : 
      19             : #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
      20             : #define LLVM_CODEGEN_SELECTIONDAGNODES_H
      21             : 
      22             : #include "llvm/ADT/APFloat.h"
      23             : #include "llvm/ADT/ArrayRef.h"
      24             : #include "llvm/ADT/BitVector.h"
      25             : #include "llvm/ADT/FoldingSet.h"
      26             : #include "llvm/ADT/GraphTraits.h"
      27             : #include "llvm/ADT/SmallPtrSet.h"
      28             : #include "llvm/ADT/SmallVector.h"
      29             : #include "llvm/ADT/ilist_node.h"
      30             : #include "llvm/ADT/iterator.h"
      31             : #include "llvm/ADT/iterator_range.h"
      32             : #include "llvm/CodeGen/ISDOpcodes.h"
      33             : #include "llvm/CodeGen/MachineMemOperand.h"
      34             : #include "llvm/CodeGen/ValueTypes.h"
      35             : #include "llvm/IR/Constants.h"
      36             : #include "llvm/IR/DebugLoc.h"
      37             : #include "llvm/IR/Instruction.h"
      38             : #include "llvm/IR/Instructions.h"
      39             : #include "llvm/IR/Metadata.h"
      40             : #include "llvm/IR/Operator.h"
      41             : #include "llvm/Support/AlignOf.h"
      42             : #include "llvm/Support/AtomicOrdering.h"
      43             : #include "llvm/Support/Casting.h"
      44             : #include "llvm/Support/ErrorHandling.h"
      45             : #include "llvm/Support/MachineValueType.h"
      46             : #include <algorithm>
      47             : #include <cassert>
      48             : #include <climits>
      49             : #include <cstddef>
      50             : #include <cstdint>
      51             : #include <cstring>
      52             : #include <iterator>
      53             : #include <string>
      54             : #include <tuple>
      55             : 
      56             : namespace llvm {
      57             : 
      58             : class APInt;
      59             : class Constant;
      60             : template <typename T> struct DenseMapInfo;
      61             : class GlobalValue;
      62             : class MachineBasicBlock;
      63             : class MachineConstantPoolValue;
      64             : class MCSymbol;
      65             : class raw_ostream;
      66             : class SDNode;
      67             : class SelectionDAG;
      68             : class Type;
      69             : class Value;
      70             : 
      71             : void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
      72             :                     bool force = false);
      73             : 
      74             : /// This represents a list of ValueType's that has been intern'd by
      75             : /// a SelectionDAG.  Instances of this simple value class are returned by
      76             : /// SelectionDAG::getVTList(...).
      77             : ///
      78             : struct SDVTList {
      79             :   const EVT *VTs;
      80             :   unsigned int NumVTs;
      81             : };
      82             : 
      83             : namespace ISD {
      84             : 
      85             :   /// Node predicates
      86             : 
      87             :   /// If N is a BUILD_VECTOR node whose elements are all the same constant or
      88             :   /// undefined, return true and return the constant value in \p SplatValue.
      89             :   bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
      90             : 
      91             :   /// Return true if the specified node is a BUILD_VECTOR where all of the
      92             :   /// elements are ~0 or undef.
      93             :   bool isBuildVectorAllOnes(const SDNode *N);
      94             : 
      95             :   /// Return true if the specified node is a BUILD_VECTOR where all of the
      96             :   /// elements are 0 or undef.
      97             :   bool isBuildVectorAllZeros(const SDNode *N);
      98             : 
      99             :   /// Return true if the specified node is a BUILD_VECTOR node of all
     100             :   /// ConstantSDNode or undef.
     101             :   bool isBuildVectorOfConstantSDNodes(const SDNode *N);
     102             : 
     103             :   /// Return true if the specified node is a BUILD_VECTOR node of all
     104             :   /// ConstantFPSDNode or undef.
     105             :   bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
     106             : 
     107             :   /// Return true if the node has at least one operand and all operands of the
     108             :   /// specified node are ISD::UNDEF.
     109             :   bool allOperandsUndef(const SDNode *N);
     110             : 
     111             : } // end namespace ISD
     112             : 
     113             : //===----------------------------------------------------------------------===//
     114             : /// Unlike LLVM values, Selection DAG nodes may return multiple
     115             : /// values as the result of a computation.  Many nodes return multiple values,
     116             : /// from loads (which define a token and a return value) to ADDC (which returns
     117             : /// a result and a carry value), to calls (which may return an arbitrary number
     118             : /// of values).
     119             : ///
     120             : /// As such, each use of a SelectionDAG computation must indicate the node that
     121             : /// computes it as well as which return value to use from that node.  This pair
     122             : /// of information is represented with the SDValue value type.
     123             : ///
     124             : class SDValue {
     125             :   friend struct DenseMapInfo<SDValue>;
     126             : 
     127             :   SDNode *Node = nullptr; // The node defining the value we are using.
     128             :   unsigned ResNo = 0;     // Which return value of the node we are using.
     129             : 
     130             : public:
     131    10640166 :   SDValue() = default;
     132             :   SDValue(SDNode *node, unsigned resno);
     133             : 
     134             :   /// get the index which selects a specific result in the SDNode
     135             :   unsigned getResNo() const { return ResNo; }
     136             : 
     137             :   /// get the SDNode which holds the desired result
     138             :   SDNode *getNode() const { return Node; }
     139             : 
     140             :   /// set the SDNode
     141    10008400 :   void setNode(SDNode *N) { Node = N; }
     142             : 
     143             :   inline SDNode *operator->() const { return Node; }
     144             : 
     145             :   bool operator==(const SDValue &O) const {
     146   220165948 :     return Node == O.Node && ResNo == O.ResNo;
     147             :   }
     148             :   bool operator!=(const SDValue &O) const {
     149         243 :     return !operator==(O);
     150             :   }
     151             :   bool operator<(const SDValue &O) const {
     152             :     return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
     153             :   }
     154             :   explicit operator bool() const {
     155        1527 :     return Node != nullptr;
     156             :   }
     157             : 
     158             :   SDValue getValue(unsigned R) const {
     159             :     return SDValue(Node, R);
     160             :   }
     161             : 
     162             :   /// Return true if this node is an operand of N.
     163             :   bool isOperandOf(const SDNode *N) const;
     164             : 
     165             :   /// Return the ValueType of the referenced return value.
     166             :   inline EVT getValueType() const;
     167             : 
     168             :   /// Return the simple ValueType of the referenced return value.
     169             :   MVT getSimpleValueType() const {
     170     6540883 :     return getValueType().getSimpleVT();
     171             :   }
     172             : 
     173             :   /// Returns the size of the value in bits.
     174     3544589 :   unsigned getValueSizeInBits() const {
     175     7089178 :     return getValueType().getSizeInBits();
     176             :   }
     177             : 
     178    33595781 :   unsigned getScalarValueSizeInBits() const {
     179    67191562 :     return getValueType().getScalarType().getSizeInBits();
     180             :   }
     181             : 
     182             :   // Forwarding methods - These forward to the corresponding methods in SDNode.
     183             :   inline unsigned getOpcode() const;
     184             :   inline unsigned getNumOperands() const;
     185             :   inline const SDValue &getOperand(unsigned i) const;
     186             :   inline uint64_t getConstantOperandVal(unsigned i) const;
     187             :   inline bool isTargetMemoryOpcode() const;
     188             :   inline bool isTargetOpcode() const;
     189             :   inline bool isMachineOpcode() const;
     190             :   inline bool isUndef() const;
     191             :   inline unsigned getMachineOpcode() const;
     192             :   inline const DebugLoc &getDebugLoc() const;
     193             :   inline void dump() const;
     194             :   inline void dump(const SelectionDAG *G) const;
     195             :   inline void dumpr() const;
     196             :   inline void dumpr(const SelectionDAG *G) const;
     197             : 
     198             :   /// Return true if this operand (which must be a chain) reaches the
     199             :   /// specified operand without crossing any side-effecting instructions.
     200             :   /// In practice, this looks through token factors and non-volatile loads.
     201             :   /// In order to remain efficient, this only
     202             :   /// looks a couple of nodes in, it does not do an exhaustive search.
     203             :   bool reachesChainWithoutSideEffects(SDValue Dest,
     204             :                                       unsigned Depth = 2) const;
     205             : 
     206             :   /// Return true if there are no nodes using value ResNo of Node.
     207             :   inline bool use_empty() const;
     208             : 
     209             :   /// Return true if there is exactly one node using value ResNo of Node.
     210             :   inline bool hasOneUse() const;
     211             : };
     212             : 
     213             : template<> struct DenseMapInfo<SDValue> {
     214             :   static inline SDValue getEmptyKey() {
     215             :     SDValue V;
     216             :     V.ResNo = -1U;
     217             :     return V;
     218             :   }
     219             : 
     220             :   static inline SDValue getTombstoneKey() {
     221             :     SDValue V;
     222             :     V.ResNo = -2U;
     223             :     return V;
     224             :   }
     225             : 
     226             :   static unsigned getHashValue(const SDValue &Val) {
     227    72863284 :     return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
     228    72863284 :             (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
     229             :   }
     230             : 
     231             :   static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
     232             :     return LHS == RHS;
     233             :   }
     234             : };
     235             : template <> struct isPodLike<SDValue> { static const bool value = true; };
     236             : 
     237             : /// Allow casting operators to work directly on
     238             : /// SDValues as if they were SDNode*'s.
     239             : template<> struct simplify_type<SDValue> {
     240             :   using SimpleType = SDNode *;
     241             : 
     242             :   static SimpleType getSimplifiedValue(SDValue &Val) {
     243      507940 :     return Val.getNode();
     244             :   }
     245             : };
     246             : template<> struct simplify_type<const SDValue> {
     247             :   using SimpleType = /*const*/ SDNode *;
     248             : 
     249             :   static SimpleType getSimplifiedValue(const SDValue &Val) {
     250    95119745 :     return Val.getNode();
     251             :   }
     252             : };
     253             : 
     254             : /// Represents a use of a SDNode. This class holds an SDValue,
     255             : /// which records the SDNode being used and the result number, a
     256             : /// pointer to the SDNode using the value, and Next and Prev pointers,
     257             : /// which link together all the uses of an SDNode.
     258             : ///
     259             : class SDUse {
     260             :   /// Val - The value being used.
     261             :   SDValue Val;
     262             :   /// User - The user of this value.
     263             :   SDNode *User = nullptr;
     264             :   /// Prev, Next - Pointers to the uses list of the SDNode referred by
     265             :   /// this operand.
     266             :   SDUse **Prev = nullptr;
     267             :   SDUse *Next = nullptr;
     268             : 
     269             : public:
     270     4610281 :   SDUse() = default;
     271             :   SDUse(const SDUse &U) = delete;
     272             :   SDUse &operator=(const SDUse &) = delete;
     273             : 
     274             :   /// Normally SDUse will just implicitly convert to an SDValue that it holds.
     275    46877571 :   operator const SDValue&() const { return Val; }
     276             : 
     277             :   /// If implicit conversion to SDValue doesn't work, the get() method returns
     278             :   /// the SDValue.
     279    14317464 :   const SDValue &get() const { return Val; }
     280             : 
     281             :   /// This returns the SDNode that contains this Use.
     282             :   SDNode *getUser() { return User; }
     283             : 
     284             :   /// Get the next SDUse in the use list.
     285             :   SDUse *getNext() const { return Next; }
     286             : 
     287             :   /// Convenience function for get().getNode().
     288   107735210 :   SDNode *getNode() const { return Val.getNode(); }
     289             :   /// Convenience function for get().getResNo().
     290    82663603 :   unsigned getResNo() const { return Val.getResNo(); }
     291             :   /// Convenience function for get().getValueType().
     292             :   EVT getValueType() const { return Val.getValueType(); }
     293             : 
     294             :   /// Convenience function for get().operator==
     295             :   bool operator==(const SDValue &V) const {
     296             :     return Val == V;
     297             :   }
     298             : 
     299             :   /// Convenience function for get().operator!=
     300             :   bool operator!=(const SDValue &V) const {
     301             :     return Val != V;
     302             :   }
     303             : 
     304             :   /// Convenience function for get().operator<
     305             :   bool operator<(const SDValue &V) const {
     306             :     return Val < V;
     307             :   }
     308             : 
     309             : private:
     310             :   friend class SelectionDAG;
     311             :   friend class SDNode;
     312             :   // TODO: unfriend HandleSDNode once we fix its operand handling.
     313             :   friend class HandleSDNode;
     314             : 
     315    50511019 :   void setUser(SDNode *p) { User = p; }
     316             : 
     317             :   /// Remove this use from its existing use list, assign it the
     318             :   /// given value, and add it to the new value's node's use list.
     319             :   inline void set(const SDValue &V);
     320             :   /// Like set, but only supports initializing a newly-allocated
     321             :   /// SDUse with a non-null value.
     322             :   inline void setInitial(const SDValue &V);
     323             :   /// Like set, but only sets the Node portion of the value,
     324             :   /// leaving the ResNo portion unmodified.
     325             :   inline void setNode(SDNode *N);
     326             : 
     327             :   void addToList(SDUse **List) {
     328    57516072 :     Next = *List;
     329    57516072 :     if (Next) Next->Prev = &Next;
     330    57516072 :     Prev = List;
     331    57516072 :     *List = this;
     332             :   }
     333             : 
     334             :   void removeFromList() {
     335    39177903 :     *Prev = Next;
     336    39177903 :     if (Next) Next->Prev = Prev;
     337             :   }
     338             : };
     339             : 
     340             : /// simplify_type specializations - Allow casting operators to work directly on
     341             : /// SDValues as if they were SDNode*'s.
     342             : template<> struct simplify_type<SDUse> {
     343             :   using SimpleType = SDNode *;
     344             : 
     345             :   static SimpleType getSimplifiedValue(SDUse &Val) {
     346             :     return Val.getNode();
     347             :   }
     348             : };
     349             : 
     350             : /// These are IR-level optimization flags that may be propagated to SDNodes.
     351             : /// TODO: This data structure should be shared by the IR optimizer and the
     352             : /// the backend.
     353             : struct SDNodeFlags {
     354             : private:
     355             :   // This bit is used to determine if the flags are in a defined state.
     356             :   // Flag bits can only be masked out during intersection if the masking flags
     357             :   // are defined.
     358             :   bool AnyDefined : 1;
     359             : 
     360             :   bool NoUnsignedWrap : 1;
     361             :   bool NoSignedWrap : 1;
     362             :   bool Exact : 1;
     363             :   bool NoNaNs : 1;
     364             :   bool NoInfs : 1;
     365             :   bool NoSignedZeros : 1;
     366             :   bool AllowReciprocal : 1;
     367             :   bool VectorReduction : 1;
     368             :   bool AllowContract : 1;
     369             :   bool ApproximateFuncs : 1;
     370             :   bool AllowReassociation : 1;
     371             : 
     372             : public:
     373             :   /// Default constructor turns off all optimization flags.
     374             :   SDNodeFlags()
     375    39383010 :       : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
     376             :         Exact(false), NoNaNs(false), NoInfs(false),
     377             :         NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
     378             :         AllowContract(false), ApproximateFuncs(false),
     379    39383010 :         AllowReassociation(false) {}
     380             : 
     381             :   /// Propagate the fast-math-flags from an IR FPMathOperator.
     382      138652 :   void copyFMF(const FPMathOperator &FPMO) {
     383             :     setNoNaNs(FPMO.hasNoNaNs());
     384             :     setNoInfs(FPMO.hasNoInfs());
     385             :     setNoSignedZeros(FPMO.hasNoSignedZeros());
     386             :     setAllowReciprocal(FPMO.hasAllowReciprocal());
     387             :     setAllowContract(FPMO.hasAllowContract());
     388             :     setApproximateFuncs(FPMO.hasApproxFunc());
     389             :     setAllowReassociation(FPMO.hasAllowReassoc());
     390      138652 :   }
     391             : 
     392             :   /// Sets the state of the flags to the defined state.
     393     1380592 :   void setDefined() { AnyDefined = true; }
     394             :   /// Returns true if the flags are in a defined state.
     395      658606 :   bool isDefined() const { return AnyDefined; }
     396             : 
     397             :   // These are mutators for each flag.
     398             :   void setNoUnsignedWrap(bool b) {
     399             :     setDefined();
     400     1237854 :     NoUnsignedWrap = b;
     401             :   }
     402             :   void setNoSignedWrap(bool b) {
     403             :     setDefined();
     404      256965 :     NoSignedWrap = b;
     405             :   }
     406             :   void setExact(bool b) {
     407             :     setDefined();
     408       22542 :     Exact = b;
     409             :   }
     410             :   void setNoNaNs(bool b) {
     411             :     setDefined();
     412      138671 :     NoNaNs = b;
     413             :   }
     414             :   void setNoInfs(bool b) {
     415             :     setDefined();
     416      138671 :     NoInfs = b;
     417             :   }
     418             :   void setNoSignedZeros(bool b) {
     419             :     setDefined();
     420      138652 :     NoSignedZeros = b;
     421             :   }
     422             :   void setAllowReciprocal(bool b) {
     423             :     setDefined();
     424      138652 :     AllowReciprocal = b;
     425             :   }
     426             :   void setVectorReduction(bool b) {
     427             :     setDefined();
     428          61 :     VectorReduction = b;
     429             :   }
     430             :   void setAllowContract(bool b) {
     431             :     setDefined();
     432      138652 :     AllowContract = b;
     433             :   }
     434             :   void setApproximateFuncs(bool b) {
     435             :     setDefined();
     436      138652 :     ApproximateFuncs = b;
     437             :   }
     438             :   void setAllowReassociation(bool b) {
     439             :     setDefined();
     440      138681 :     AllowReassociation = b;
     441             :   }
     442             : 
     443             :   // These are accessors for each flag.
     444             :   bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
     445             :   bool hasNoSignedWrap() const { return NoSignedWrap; }
     446             :   bool hasExact() const { return Exact; }
     447             :   bool hasNoNaNs() const { return NoNaNs; }
     448             :   bool hasNoInfs() const { return NoInfs; }
     449             :   bool hasNoSignedZeros() const { return NoSignedZeros; }
     450             :   bool hasAllowReciprocal() const { return AllowReciprocal; }
     451             :   bool hasVectorReduction() const { return VectorReduction; }
     452             :   bool hasAllowContract() const { return AllowContract; }
     453             :   bool hasApproximateFuncs() const { return ApproximateFuncs; }
     454             :   bool hasAllowReassociation() const { return AllowReassociation; }
     455             : 
     456             :   bool isFast() const {
     457             :     return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs &&
     458             :            AllowContract && ApproximateFuncs && AllowReassociation;
     459             :   }
     460             : 
     461             :   /// Clear any flags in this flag set that aren't also set in Flags.
     462             :   /// If the given Flags are undefined then don't do anything.
     463      658606 :   void intersectWith(const SDNodeFlags Flags) {
     464      658606 :     if (!Flags.isDefined())
     465             :       return;
     466       74204 :     NoUnsignedWrap &= Flags.NoUnsignedWrap;
     467       74204 :     NoSignedWrap &= Flags.NoSignedWrap;
     468       74204 :     Exact &= Flags.Exact;
     469       74204 :     NoNaNs &= Flags.NoNaNs;
     470       74204 :     NoInfs &= Flags.NoInfs;
     471       74204 :     NoSignedZeros &= Flags.NoSignedZeros;
     472       74204 :     AllowReciprocal &= Flags.AllowReciprocal;
     473       74204 :     VectorReduction &= Flags.VectorReduction;
     474       74204 :     AllowContract &= Flags.AllowContract;
     475       74204 :     ApproximateFuncs &= Flags.ApproximateFuncs;
     476       74204 :     AllowReassociation &= Flags.AllowReassociation;
     477             :   }
     478             : };
     479             : 
     480             : /// Represents one node in the SelectionDAG.
     481             : ///
     482             : class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
     483             : private:
     484             :   /// The operation that this node performs.
     485             :   int16_t NodeType;
     486             : 
     487             : protected:
     488             :   // We define a set of mini-helper classes to help us interpret the bits in our
     489             :   // SubclassData.  These are designed to fit within a uint16_t so they pack
     490             :   // with NodeType.
     491             : 
     492             :   class SDNodeBitfields {
     493             :     friend class SDNode;
     494             :     friend class MemIntrinsicSDNode;
     495             :     friend class MemSDNode;
     496             :     friend class SelectionDAG;
     497             : 
     498             :     uint16_t HasDebugValue : 1;
     499             :     uint16_t IsMemIntrinsic : 1;
     500             :     uint16_t IsDivergent : 1;
     501             :   };
     502             :   enum { NumSDNodeBits = 3 };
     503             : 
     504             :   class ConstantSDNodeBitfields {
     505             :     friend class ConstantSDNode;
     506             : 
     507             :     uint16_t : NumSDNodeBits;
     508             : 
     509             :     uint16_t IsOpaque : 1;
     510             :   };
     511             : 
     512             :   class MemSDNodeBitfields {
     513             :     friend class MemSDNode;
     514             :     friend class MemIntrinsicSDNode;
     515             :     friend class AtomicSDNode;
     516             : 
     517             :     uint16_t : NumSDNodeBits;
     518             : 
     519             :     uint16_t IsVolatile : 1;
     520             :     uint16_t IsNonTemporal : 1;
     521             :     uint16_t IsDereferenceable : 1;
     522             :     uint16_t IsInvariant : 1;
     523             :   };
     524             :   enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
     525             : 
     526             :   class LSBaseSDNodeBitfields {
     527             :     friend class LSBaseSDNode;
     528             : 
     529             :     uint16_t : NumMemSDNodeBits;
     530             : 
     531             :     uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode
     532             :   };
     533             :   enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
     534             : 
     535             :   class LoadSDNodeBitfields {
     536             :     friend class LoadSDNode;
     537             :     friend class MaskedLoadSDNode;
     538             : 
     539             :     uint16_t : NumLSBaseSDNodeBits;
     540             : 
     541             :     uint16_t ExtTy : 2; // enum ISD::LoadExtType
     542             :     uint16_t IsExpanding : 1;
     543             :   };
     544             : 
     545             :   class StoreSDNodeBitfields {
     546             :     friend class StoreSDNode;
     547             :     friend class MaskedStoreSDNode;
     548             : 
     549             :     uint16_t : NumLSBaseSDNodeBits;
     550             : 
     551             :     uint16_t IsTruncating : 1;
     552             :     uint16_t IsCompressing : 1;
     553             :   };
     554             : 
     555             :   union {
     556             :     char RawSDNodeBits[sizeof(uint16_t)];
     557             :     SDNodeBitfields SDNodeBits;
     558             :     ConstantSDNodeBitfields ConstantSDNodeBits;
     559             :     MemSDNodeBitfields MemSDNodeBits;
     560             :     LSBaseSDNodeBitfields LSBaseSDNodeBits;
     561             :     LoadSDNodeBitfields LoadSDNodeBits;
     562             :     StoreSDNodeBitfields StoreSDNodeBits;
     563             :   };
     564             : 
     565             :   // RawSDNodeBits must cover the entirety of the union.  This means that all of
     566             :   // the union's members must have size <= RawSDNodeBits.  We write the RHS as
     567             :   // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
     568             :   static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
     569             :   static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
     570             :   static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
     571             :   static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
     572             :   static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
     573             :   static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
     574             : 
     575             : private:
     576             :   friend class SelectionDAG;
     577             :   // TODO: unfriend HandleSDNode once we fix its operand handling.
     578             :   friend class HandleSDNode;
     579             : 
     580             :   /// Unique id per SDNode in the DAG.
     581             :   int NodeId = -1;
     582             : 
     583             :   /// The values that are used by this operation.
     584             :   SDUse *OperandList = nullptr;
     585             : 
     586             :   /// The types of the values this node defines.  SDNode's may
     587             :   /// define multiple values simultaneously.
     588             :   const EVT *ValueList;
     589             : 
     590             :   /// List of uses for this SDNode.
     591             :   SDUse *UseList = nullptr;
     592             : 
     593             :   /// The number of entries in the Operand/Value list.
     594             :   unsigned short NumOperands = 0;
     595             :   unsigned short NumValues;
     596             : 
     597             :   // The ordering of the SDNodes. It roughly corresponds to the ordering of the
     598             :   // original LLVM instructions.
     599             :   // This is used for turning off scheduling, because we'll forgo
     600             :   // the normal scheduling algorithms and output the instructions according to
     601             :   // this ordering.
     602             :   unsigned IROrder;
     603             : 
     604             :   /// Source line information.
     605             :   DebugLoc debugLoc;
     606             : 
     607             :   /// Return a pointer to the specified value type.
     608             :   static const EVT *getValueTypeList(EVT VT);
     609             : 
     610             :   SDNodeFlags Flags;
     611             : 
     612             : public:
     613             :   /// Unique and persistent id per SDNode in the DAG.
     614             :   /// Used for debug printing.
     615             :   uint16_t PersistentId;
     616             : 
     617             :   //===--------------------------------------------------------------------===//
     618             :   //  Accessors
     619             :   //
     620             : 
     621             :   /// Return the SelectionDAG opcode value for this node. For
     622             :   /// pre-isel nodes (those for which isMachineOpcode returns false), these
     623             :   /// are the opcode values in the ISD and <target>ISD namespaces. For
     624             :   /// post-isel opcodes, see getMachineOpcode.
     625   799746771 :   unsigned getOpcode()  const { return (unsigned short)NodeType; }
     626             : 
     627             :   /// Test if this node has a target-specific opcode (in the
     628             :   /// \<target\>ISD namespace).
     629      306778 :   bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
     630             : 
     631             :   /// Test if this node has a target-specific
     632             :   /// memory-referencing opcode (in the \<target\>ISD namespace and
     633             :   /// greater than FIRST_TARGET_MEMORY_OPCODE).
     634             :   bool isTargetMemoryOpcode() const {
     635             :     return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
     636             :   }
     637             : 
     638             :   /// Return true if the type of the node type undefined.
     639      145235 :   bool isUndef() const { return NodeType == ISD::UNDEF; }
     640             : 
     641             :   /// Test if this node is a memory intrinsic (with valid pointer information).
     642             :   /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
     643             :   /// non-memory intrinsics (with chains) that are not really instances of
     644             :   /// MemSDNode. For such nodes, we need some extra state to determine the
     645             :   /// proper classof relationship.
     646             :   bool isMemIntrinsic() const {
     647      887707 :     return (NodeType == ISD::INTRINSIC_W_CHAIN ||
     648      897761 :             NodeType == ISD::INTRINSIC_VOID) &&
     649       10054 :            SDNodeBits.IsMemIntrinsic;
     650             :   }
     651             : 
     652             :   /// Test if this node is a strict floating point pseudo-op.
     653             :   bool isStrictFPOpcode() {
     654     7281408 :     switch (NodeType) {
     655             :       default:
     656             :         return false;
     657             :       case ISD::STRICT_FADD:
     658             :       case ISD::STRICT_FSUB:
     659             :       case ISD::STRICT_FMUL:
     660             :       case ISD::STRICT_FDIV:
     661             :       case ISD::STRICT_FREM:
     662             :       case ISD::STRICT_FMA:
     663             :       case ISD::STRICT_FSQRT:
     664             :       case ISD::STRICT_FPOW:
     665             :       case ISD::STRICT_FPOWI:
     666             :       case ISD::STRICT_FSIN:
     667             :       case ISD::STRICT_FCOS:
     668             :       case ISD::STRICT_FEXP:
     669             :       case ISD::STRICT_FEXP2:
     670             :       case ISD::STRICT_FLOG:
     671             :       case ISD::STRICT_FLOG10:
     672             :       case ISD::STRICT_FLOG2:
     673             :       case ISD::STRICT_FRINT:
     674             :       case ISD::STRICT_FNEARBYINT:
     675             :         return true;
     676             :     }
     677             :   }
     678             : 
     679             :   /// Test if this node has a post-isel opcode, directly
     680             :   /// corresponding to a MachineInstr opcode.
     681             :   bool isMachineOpcode() const { return NodeType < 0; }
     682             : 
     683             :   /// This may only be called if isMachineOpcode returns
     684             :   /// true. It returns the MachineInstr opcode value that the node's opcode
     685             :   /// corresponds to.
     686             :   unsigned getMachineOpcode() const {
     687             :     assert(isMachineOpcode() && "Not a MachineInstr opcode!");
     688    27408520 :     return ~NodeType;
     689             :   }
     690             : 
     691     8222582 :   bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
     692       21740 :   void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
     693             : 
     694    41873897 :   bool isDivergent() const { return SDNodeBits.IsDivergent; }
     695             : 
     696             :   /// Return true if there are no uses of this node.
     697             :   bool use_empty() const { return UseList == nullptr; }
     698             : 
     699             :   /// Return true if there is exactly one use of this node.
     700             :   bool hasOneUse() const {
     701    28131763 :     return !use_empty() && std::next(use_begin()) == use_end();
     702             :   }
     703             : 
     704             :   /// Return the number of uses of this node. This method takes
     705             :   /// time proportional to the number of uses.
     706        2026 :   size_t use_size() const { return std::distance(use_begin(), use_end()); }
     707             : 
     708             :   /// Return the unique node id.
     709             :   int getNodeId() const { return NodeId; }
     710             : 
     711             :   /// Set unique node id.
     712   123140735 :   void setNodeId(int Id) { NodeId = Id; }
     713             : 
     714             :   /// Return the node ordering.
     715             :   unsigned getIROrder() const { return IROrder; }
     716             : 
     717             :   /// Set the node ordering.
     718       92317 :   void setIROrder(unsigned Order) { IROrder = Order; }
     719             : 
     720             :   /// Return the source location info.
     721     4187902 :   const DebugLoc &getDebugLoc() const { return debugLoc; }
     722             : 
     723             :   /// Set source location info.  Try to avoid this, putting
     724             :   /// it in the constructor is preferable.
     725             :   void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
     726             : 
     727             :   /// This class provides iterator support for SDUse
     728             :   /// operands that use a specific SDNode.
     729             :   class use_iterator
     730             :     : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
     731             :     friend class SDNode;
     732             : 
     733             :     SDUse *Op = nullptr;
     734             : 
     735     5654251 :     explicit use_iterator(SDUse *op) : Op(op) {}
     736             : 
     737             :   public:
     738             :     using reference = std::iterator<std::forward_iterator_tag,
     739             :                                     SDUse, ptrdiff_t>::reference;
     740             :     using pointer = std::iterator<std::forward_iterator_tag,
     741             :                                   SDUse, ptrdiff_t>::pointer;
     742             : 
     743             :     use_iterator() = default;
     744     1519733 :     use_iterator(const use_iterator &I) : Op(I.Op) {}
     745             : 
     746             :     bool operator==(const use_iterator &x) const {
     747    21278190 :       return Op == x.Op;
     748             :     }
     749             :     bool operator!=(const use_iterator &x) const {
     750             :       return !operator==(x);
     751             :     }
     752             : 
     753             :     /// Return true if this iterator is at the end of uses list.
     754             :     bool atEnd() const { return Op == nullptr; }
     755             : 
     756             :     // Iterator traversal: forward iteration only.
     757             :     use_iterator &operator++() {          // Preincrement
     758             :       assert(Op && "Cannot increment end iterator!");
     759   123810015 :       Op = Op->getNext();
     760             :       return *this;
     761             :     }
     762             : 
     763             :     use_iterator operator++(int) {        // Postincrement
     764             :       use_iterator tmp = *this; ++*this; return tmp;
     765             :     }
     766             : 
     767             :     /// Retrieve a pointer to the current user node.
     768             :     SDNode *operator*() const {
     769             :       assert(Op && "Cannot dereference end iterator!");
     770    96413801 :       return Op->getUser();
     771             :     }
     772             : 
     773             :     SDNode *operator->() const { return operator*(); }
     774             : 
     775     7863267 :     SDUse &getUse() const { return *Op; }
     776             : 
     777             :     /// Retrieve the operand # of this use in its user.
     778             :     unsigned getOperandNo() const {
     779             :       assert(Op && "Cannot dereference end iterator!");
     780     1870540 :       return (unsigned)(Op - Op->getUser()->OperandList);
     781             :     }
     782             :   };
     783             : 
     784             :   /// Provide iteration support to walk over all uses of an SDNode.
     785             :   use_iterator use_begin() const {
     786             :     return use_iterator(UseList);
     787             :   }
     788             : 
     789             :   static use_iterator use_end() { return use_iterator(nullptr); }
     790             : 
     791             :   inline iterator_range<use_iterator> uses() {
     792    11314178 :     return make_range(use_begin(), use_end());
     793             :   }
     794             :   inline iterator_range<use_iterator> uses() const {
     795        6370 :     return make_range(use_begin(), use_end());
     796             :   }
     797             : 
     798             :   /// Return true if there are exactly NUSES uses of the indicated value.
     799             :   /// This method ignores uses of other values defined by this operation.
     800             :   bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
     801             : 
     802             :   /// Return true if there are any use of the indicated value.
     803             :   /// This method ignores uses of other values defined by this operation.
     804             :   bool hasAnyUseOfValue(unsigned Value) const;
     805             : 
     806             :   /// Return true if this node is the only use of N.
     807             :   bool isOnlyUserOf(const SDNode *N) const;
     808             : 
     809             :   /// Return true if this node is an operand of N.
     810             :   bool isOperandOf(const SDNode *N) const;
     811             : 
     812             :   /// Return true if this node is a predecessor of N.
     813             :   /// NOTE: Implemented on top of hasPredecessor and every bit as
     814             :   /// expensive. Use carefully.
     815             :   bool isPredecessorOf(const SDNode *N) const {
     816        4138 :     return N->hasPredecessor(this);
     817             :   }
     818             : 
     819             :   /// Return true if N is a predecessor of this node.
     820             :   /// N is either an operand of this node, or can be reached by recursively
     821             :   /// traversing up the operands.
     822             :   /// NOTE: This is an expensive method. Use it carefully.
     823             :   bool hasPredecessor(const SDNode *N) const;
     824             : 
     825             :   /// Returns true if N is a predecessor of any node in Worklist. This
     826             :   /// helper keeps Visited and Worklist sets externally to allow unions
     827             :   /// searches to be performed in parallel, caching of results across
     828             :   /// queries and incremental addition to Worklist. Stops early if N is
     829             :   /// found but will resume. Remember to clear Visited and Worklists
     830             :   /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
     831             :   /// giving up. The TopologicalPrune flag signals that positive NodeIds are
     832             :   /// topologically ordered (Operands have strictly smaller node id) and search
     833             :   /// can be pruned leveraging this.
     834      354970 :   static bool hasPredecessorHelper(const SDNode *N,
     835             :                                    SmallPtrSetImpl<const SDNode *> &Visited,
     836             :                                    SmallVectorImpl<const SDNode *> &Worklist,
     837             :                                    unsigned int MaxSteps = 0,
     838             :                                    bool TopologicalPrune = false) {
     839             :     SmallVector<const SDNode *, 8> DeferredNodes;
     840      354970 :     if (Visited.count(N))
     841             :       return true;
     842             : 
     843             :     // Node Id's are assigned in three places: As a topological
     844             :     // ordering (> 0), during legalization (results in values set to
     845             :     // 0), new nodes (set to -1). If N has a topolgical id then we
     846             :     // know that all nodes with ids smaller than it cannot be
     847             :     // successors and we need not check them. Filter out all node
     848             :     // that can't be matches. We add them to the worklist before exit
     849             :     // in case of multiple calls. Note that during selection the topological id
     850             :     // may be violated if a node's predecessor is selected before it. We mark
     851             :     // this at selection negating the id of unselected successors and
     852             :     // restricting topological pruning to positive ids.
     853             : 
     854      354496 :     int NId = N->getNodeId();
     855             :     // If we Invalidated the Id, reconstruct original NId.
     856      354496 :     if (NId < -1)
     857         755 :       NId = -(NId + 1);
     858             : 
     859             :     bool Found = false;
     860     1512599 :     while (!Worklist.empty()) {
     861     1161532 :       const SDNode *M = Worklist.pop_back_val();
     862     1161532 :       int MId = M->getNodeId();
     863     3028010 :       if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
     864     3258947 :           (MId > 0) && (MId < NId)) {
     865      906940 :         DeferredNodes.push_back(M);
     866      906940 :         continue;
     867             :       }
     868      795320 :       for (const SDValue &OpV : M->op_values()) {
     869      540728 :         SDNode *Op = OpV.getNode();
     870      540728 :         if (Visited.insert(Op).second)
     871      441440 :           Worklist.push_back(Op);
     872      540728 :         if (Op == N)
     873             :           Found = true;
     874             :       }
     875      254592 :       if (Found)
     876             :         break;
     877      376103 :       if (MaxSteps != 0 && Visited.size() >= MaxSteps)
     878             :         break;
     879             :     }
     880             :     // Push deferred nodes back on worklist.
     881      354496 :     Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
     882             :     // If we bailed early, conservatively return found.
     883      550520 :     if (MaxSteps != 0 && Visited.size() >= MaxSteps)
     884             :       return true;
     885             :     return Found;
     886             :   }
     887             : 
     888             :   /// Return true if all the users of N are contained in Nodes.
     889             :   /// NOTE: Requires at least one match, but doesn't require them all.
     890             :   static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
     891             : 
     892             :   /// Return the number of values used by this operation.
     893   134122703 :   unsigned getNumOperands() const { return NumOperands; }
     894             : 
     895             :   /// Helper method returns the integer value of a ConstantSDNode operand.
     896             :   inline uint64_t getConstantOperandVal(unsigned Num) const;
     897             : 
     898             :   const SDValue &getOperand(unsigned Num) const {
     899             :     assert(Num < NumOperands && "Invalid child # of SDNode!");
     900   186408213 :     return OperandList[Num];
     901             :   }
     902             : 
     903             :   using op_iterator = SDUse *;
     904             : 
     905             :   op_iterator op_begin() const { return OperandList; }
     906   118756830 :   op_iterator op_end() const { return OperandList+NumOperands; }
     907    40740151 :   ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
     908             : 
     909             :   /// Iterator for directly iterating over the operand SDValue's.
     910             :   struct value_op_iterator
     911             :       : iterator_adaptor_base<value_op_iterator, op_iterator,
     912             :                               std::random_access_iterator_tag, SDValue,
     913             :                               ptrdiff_t, value_op_iterator *,
     914             :                               value_op_iterator *> {
     915             :     explicit value_op_iterator(SDUse *U = nullptr)
     916             :       : iterator_adaptor_base(U) {}
     917             : 
     918       16600 :     const SDValue &operator*() const { return I->get(); }
     919             :   };
     920             : 
     921             :   iterator_range<value_op_iterator> op_values() const {
     922             :     return make_range(value_op_iterator(op_begin()),
     923    59017353 :                       value_op_iterator(op_end()));
     924             :   }
     925             : 
     926             :   SDVTList getVTList() const {
     927      373297 :     SDVTList X = { ValueList, NumValues };
     928             :     return X;
     929             :   }
     930             : 
     931             :   /// If this node has a glue operand, return the node
     932             :   /// to which the glue operand points. Otherwise return NULL.
     933             :   SDNode *getGluedNode() const {
     934    41995636 :     if (getNumOperands() != 0 &&
     935    41353288 :         getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
     936             :       return getOperand(getNumOperands()-1).getNode();
     937             :     return nullptr;
     938             :   }
     939             : 
     940             :   /// If this node has a glue value with a user, return
     941             :   /// the user (there is at most one). Otherwise return NULL.
     942             :   SDNode *getGluedUser() const {
     943      929125 :     for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
     944     1325865 :       if (UI.getUse().get().getValueType() == MVT::Glue)
     945             :         return *UI;
     946             :     return nullptr;
     947             :   }
     948             : 
     949     5626664 :   const SDNodeFlags getFlags() const { return Flags; }
     950     6634418 :   void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
     951             :   bool isFast() { return Flags.isFast(); }
     952             : 
     953             :   /// Clear any flags in this node that aren't also set in Flags.
     954             :   /// If Flags is not in a defined state then this has no effect.
     955             :   void intersectFlagsWith(const SDNodeFlags Flags);
     956             : 
     957             :   /// Return the number of values defined/returned by this operator.
     958    49631359 :   unsigned getNumValues() const { return NumValues; }
     959             : 
     960             :   /// Return the type of a specified result.
     961             :   EVT getValueType(unsigned ResNo) const {
     962             :     assert(ResNo < NumValues && "Illegal result number!");
     963   360429748 :     return ValueList[ResNo];
     964             :   }
     965             : 
     966             :   /// Return the type of a specified result as a simple type.
     967             :   MVT getSimpleValueType(unsigned ResNo) const {
     968    12487294 :     return getValueType(ResNo).getSimpleVT();
     969             :   }
     970             : 
     971             :   /// Returns MVT::getSizeInBits(getValueType(ResNo)).
     972             :   unsigned getValueSizeInBits(unsigned ResNo) const {
     973      545983 :     return getValueType(ResNo).getSizeInBits();
     974             :   }
     975             : 
     976             :   using value_iterator = const EVT *;
     977             : 
     978             :   value_iterator value_begin() const { return ValueList; }
     979    10699523 :   value_iterator value_end() const { return ValueList+NumValues; }
     980             : 
     981             :   /// Return the opcode of this operation for printing.
     982             :   std::string getOperationName(const SelectionDAG *G = nullptr) const;
     983             :   static const char* getIndexedModeName(ISD::MemIndexedMode AM);
     984             :   void print_types(raw_ostream &OS, const SelectionDAG *G) const;
     985             :   void print_details(raw_ostream &OS, const SelectionDAG *G) const;
     986             :   void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
     987             :   void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
     988             : 
     989             :   /// Print a SelectionDAG node and all children down to
     990             :   /// the leaves.  The given SelectionDAG allows target-specific nodes
     991             :   /// to be printed in human-readable form.  Unlike printr, this will
     992             :   /// print the whole DAG, including children that appear multiple
     993             :   /// times.
     994             :   ///
     995             :   void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
     996             : 
     997             :   /// Print a SelectionDAG node and children up to
     998             :   /// depth "depth."  The given SelectionDAG allows target-specific
     999             :   /// nodes to be printed in human-readable form.  Unlike printr, this
    1000             :   /// will print children that appear multiple times wherever they are
    1001             :   /// used.
    1002             :   ///
    1003             :   void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
    1004             :                        unsigned depth = 100) const;
    1005             : 
    1006             :   /// Dump this node, for debugging.
    1007             :   void dump() const;
    1008             : 
    1009             :   /// Dump (recursively) this node and its use-def subgraph.
    1010             :   void dumpr() const;
    1011             : 
    1012             :   /// Dump this node, for debugging.
    1013             :   /// The given SelectionDAG allows target-specific nodes to be printed
    1014             :   /// in human-readable form.
    1015             :   void dump(const SelectionDAG *G) const;
    1016             : 
    1017             :   /// Dump (recursively) this node and its use-def subgraph.
    1018             :   /// The given SelectionDAG allows target-specific nodes to be printed
    1019             :   /// in human-readable form.
    1020             :   void dumpr(const SelectionDAG *G) const;
    1021             : 
    1022             :   /// printrFull to dbgs().  The given SelectionDAG allows
    1023             :   /// target-specific nodes to be printed in human-readable form.
    1024             :   /// Unlike dumpr, this will print the whole DAG, including children
    1025             :   /// that appear multiple times.
    1026             :   void dumprFull(const SelectionDAG *G = nullptr) const;
    1027             : 
    1028             :   /// printrWithDepth to dbgs().  The given
    1029             :   /// SelectionDAG allows target-specific nodes to be printed in
    1030             :   /// human-readable form.  Unlike dumpr, this will print children
    1031             :   /// that appear multiple times wherever they are used.
    1032             :   ///
    1033             :   void dumprWithDepth(const SelectionDAG *G = nullptr,
    1034             :                       unsigned depth = 100) const;
    1035             : 
    1036             :   /// Gather unique data for the node.
    1037             :   void Profile(FoldingSetNodeID &ID) const;
    1038             : 
    1039             :   /// This method should only be used by the SDUse class.
    1040    57516072 :   void addUse(SDUse &U) { U.addToList(&UseList); }
    1041             : 
    1042             : protected:
    1043             :   static SDVTList getSDVTList(EVT VT) {
    1044    12518633 :     SDVTList Ret = { getValueTypeList(VT), 1 };
    1045             :     return Ret;
    1046             :   }
    1047             : 
    1048             :   /// Create an SDNode.
    1049             :   ///
    1050             :   /// SDNodes are created without any operands, and never own the operand
    1051             :   /// storage. To add operands, see SelectionDAG::createOperands.
    1052    27007408 :   SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    1053    81022224 :       : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
    1054    81022224 :         IROrder(Order), debugLoc(std::move(dl)) {
    1055    27007408 :     memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
    1056             :     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
    1057             :     assert(NumValues == VTs.NumVTs &&
    1058             :            "NumValues wasn't wide enough for its operands!");
    1059    27007408 :   }
    1060             : 
    1061             :   /// Release the operands and set this node to have zero operands.
    1062             :   void DropOperands();
    1063             : };
    1064             : 
    1065             : /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
    1066             : /// into SDNode creation functions.
    1067             : /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
    1068             : /// from the original Instruction, and IROrder is the ordinal position of
    1069             : /// the instruction.
    1070             : /// When an SDNode is created after the DAG is being built, both DebugLoc and
    1071             : /// the IROrder are propagated from the original SDNode.
    1072             : /// So SDLoc class provides two constructors besides the default one, one to
    1073             : /// be used by the DAGBuilder, the other to be used by others.
    1074    10651586 : class SDLoc {
    1075             : private:
    1076             :   DebugLoc DL;
    1077             :   int IROrder = 0;
    1078             : 
    1079             : public:
    1080      181711 :   SDLoc() = default;
    1081   114823361 :   SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
    1082             :   SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
    1083     5183694 :   SDLoc(const Instruction *I, int Order) : IROrder(Order) {
    1084             :     assert(Order >= 0 && "bad IROrder");
    1085     5183694 :     if (I)
    1086             :       DL = I->getDebugLoc();
    1087             :   }
    1088             : 
    1089    23200385 :   unsigned getIROrder() const { return IROrder; }
    1090    17714221 :   const DebugLoc &getDebugLoc() const { return DL; }
    1091             : };
    1092             : 
    1093             : // Define inline functions from the SDValue class.
    1094             : 
    1095    38277380 : inline SDValue::SDValue(SDNode *node, unsigned resno)
    1096    37816109 :     : Node(node), ResNo(resno) {
    1097             :   // Explicitly check for !ResNo to avoid use-after-free, because there are
    1098             :   // callers that use SDValue(N, 0) with a deleted N to indicate successful
    1099             :   // combines.
    1100             :   assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&
    1101             :          "Invalid result number for the given node!");
    1102             :   assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.");
    1103             : }
    1104             : 
    1105             : inline unsigned SDValue::getOpcode() const {
    1106   177505718 :   return Node->getOpcode();
    1107             : }
    1108             : 
    1109             : inline EVT SDValue::getValueType() const {
    1110   262900511 :   return Node->getValueType(ResNo);
    1111             : }
    1112             : 
    1113             : inline unsigned SDValue::getNumOperands() const {
    1114    44908081 :   return Node->getNumOperands();
    1115             : }
    1116             : 
    1117             : inline const SDValue &SDValue::getOperand(unsigned i) const {
    1118    82130845 :   return Node->getOperand(i);
    1119             : }
    1120             : 
    1121             : inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
    1122             :   return Node->getConstantOperandVal(i);
    1123             : }
    1124             : 
    1125             : inline bool SDValue::isTargetOpcode() const {
    1126             :   return Node->isTargetOpcode();
    1127             : }
    1128             : 
    1129             : inline bool SDValue::isTargetMemoryOpcode() const {
    1130             :   return Node->isTargetMemoryOpcode();
    1131             : }
    1132             : 
    1133             : inline bool SDValue::isMachineOpcode() const {
    1134    13920163 :   return Node->isMachineOpcode();
    1135             : }
    1136             : 
    1137             : inline unsigned SDValue::getMachineOpcode() const {
    1138          21 :   return Node->getMachineOpcode();
    1139             : }
    1140             : 
    1141             : inline bool SDValue::isUndef() const {
    1142    34012484 :   return Node->isUndef();
    1143             : }
    1144             : 
    1145             : inline bool SDValue::use_empty() const {
    1146      499427 :   return !Node->hasAnyUseOfValue(ResNo);
    1147             : }
    1148             : 
    1149             : inline bool SDValue::hasOneUse() const {
    1150     8607578 :   return Node->hasNUsesOfValue(1, ResNo);
    1151             : }
    1152             : 
    1153             : inline const DebugLoc &SDValue::getDebugLoc() const {
    1154             :   return Node->getDebugLoc();
    1155             : }
    1156             : 
    1157             : inline void SDValue::dump() const {
    1158             :   return Node->dump();
    1159             : }
    1160             : 
    1161             : inline void SDValue::dump(const SelectionDAG *G) const {
    1162             :   return Node->dump(G);
    1163             : }
    1164             : 
    1165             : inline void SDValue::dumpr() const {
    1166             :   return Node->dumpr();
    1167             : }
    1168             : 
    1169             : inline void SDValue::dumpr(const SelectionDAG *G) const {
    1170             :   return Node->dumpr(G);
    1171             : }
    1172             : 
    1173             : // Define inline functions from the SDUse class.
    1174             : 
    1175    37116049 : inline void SDUse::set(const SDValue &V) {
    1176    37116049 :   if (Val.getNode()) removeFromList();
    1177    37116049 :   Val = V;
    1178    37116049 :   if (V.getNode()) V.getNode()->addUse(*this);
    1179    37116049 : }
    1180             : 
    1181             : inline void SDUse::setInitial(const SDValue &V) {
    1182    50511019 :   Val = V;
    1183    50511019 :   V.getNode()->addUse(*this);
    1184             : }
    1185             : 
    1186             : inline void SDUse::setNode(SDNode *N) {
    1187     2061854 :   if (Val.getNode()) removeFromList();
    1188             :   Val.setNode(N);
    1189     2061854 :   if (N) N->addUse(*this);
    1190             : }
    1191             : 
    1192             : /// This class is used to form a handle around another node that
    1193             : /// is persistent and is updated across invocations of replaceAllUsesWith on its
    1194             : /// operand.  This node should be directly created by end-users and not added to
    1195             : /// the AllNodes list.
    1196             : class HandleSDNode : public SDNode {
    1197             :   SDUse Op;
    1198             : 
    1199             : public:
    1200     4610281 :   explicit HandleSDNode(SDValue X)
    1201    13830843 :     : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
    1202             :     // HandleSDNodes are never inserted into the DAG, so they won't be
    1203             :     // auto-numbered. Use ID 65535 as a sentinel.
    1204     4610281 :     PersistentId = 0xffff;
    1205             : 
    1206             :     // Manually set up the operand list. This node type is special in that it's
    1207             :     // always stack allocated and SelectionDAG does not manage its operands.
    1208             :     // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
    1209             :     // be so special.
    1210             :     Op.setUser(this);
    1211             :     Op.setInitial(X);
    1212     4610281 :     NumOperands = 1;
    1213     4610281 :     OperandList = &Op;
    1214     4610281 :   }
    1215             :   ~HandleSDNode();
    1216             : 
    1217             :   const SDValue &getValue() const { return Op; }
    1218             : };
    1219             : 
    1220             : class AddrSpaceCastSDNode : public SDNode {
    1221             : private:
    1222             :   unsigned SrcAddrSpace;
    1223             :   unsigned DestAddrSpace;
    1224             : 
    1225             : public:
    1226             :   AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
    1227             :                       unsigned SrcAS, unsigned DestAS);
    1228             : 
    1229             :   unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
    1230             :   unsigned getDestAddressSpace() const { return DestAddrSpace; }
    1231             : 
    1232             :   static bool classof(const SDNode *N) {
    1233             :     return N->getOpcode() == ISD::ADDRSPACECAST;
    1234             :   }
    1235             : };
    1236             : 
    1237             : /// This is an abstract virtual class for memory operations.
    1238             : class MemSDNode : public SDNode {
    1239             : private:
    1240             :   // VT of in-memory value.
    1241             :   EVT MemoryVT;
    1242             : 
    1243             : protected:
    1244             :   /// Memory reference information.
    1245             :   MachineMemOperand *MMO;
    1246             : 
    1247             : public:
    1248             :   MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
    1249             :             EVT MemoryVT, MachineMemOperand *MMO);
    1250             : 
    1251        7302 :   bool readMem() const { return MMO->isLoad(); }
    1252       31585 :   bool writeMem() const { return MMO->isStore(); }
    1253             : 
    1254             :   /// Returns alignment and volatility of the memory access
    1255             :   unsigned getOriginalAlignment() const {
    1256     4965265 :     return MMO->getBaseAlignment();
    1257             :   }
    1258             :   unsigned getAlignment() const {
    1259    12130663 :     return MMO->getAlignment();
    1260             :   }
    1261             : 
    1262             :   /// Return the SubclassData value, without HasDebugValue. This contains an
    1263             :   /// encoding of the volatile flag, as well as bits used by subclasses. This
    1264             :   /// function should only be used to compute a FoldingSetNodeID value.
    1265             :   /// The HasDebugValue bit is masked out because CSE map needs to match
    1266             :   /// nodes with debug info with nodes without debug info. Same is about
    1267             :   /// isDivergent bit.
    1268             :   unsigned getRawSubclassData() const {
    1269             :     uint16_t Data;
    1270             :     union {
    1271             :       char RawSDNodeBits[sizeof(uint16_t)];
    1272             :       SDNodeBitfields SDNodeBits;
    1273             :     };
    1274     6127490 :     memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
    1275     6127490 :     SDNodeBits.HasDebugValue = 0;
    1276     6127490 :     SDNodeBits.IsDivergent = false;
    1277     6127490 :     memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
    1278     4138535 :     return Data;
    1279             :   }
    1280             : 
    1281    23118527 :   bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
    1282      888627 :   bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
    1283       16487 :   bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
    1284    11335988 :   bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
    1285             : 
    1286             :   // Returns the offset from the location of the access.
    1287     4945714 :   int64_t getSrcValueOffset() const { return MMO->getOffset(); }
    1288             : 
    1289             :   /// Returns the AA info that describes the dereference.
    1290             :   AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
    1291             : 
    1292             :   /// Returns the Ranges that describes the dereference.
    1293     2349962 :   const MDNode *getRanges() const { return MMO->getRanges(); }
    1294             : 
    1295             :   /// Returns the synchronization scope ID for this memory operation.
    1296             :   SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
    1297             : 
    1298             :   /// Return the atomic ordering requirements for this memory operation. For
    1299             :   /// cmpxchg atomic operations, return the atomic ordering requirements when
    1300             :   /// store occurs.
    1301             :   AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
    1302             : 
    1303             :   /// Return the type of the in-memory value.
    1304             :   EVT getMemoryVT() const { return MemoryVT; }
    1305             : 
    1306             :   /// Return a MachineMemOperand object describing the memory
    1307             :   /// reference performed by operation.
    1308             :   MachineMemOperand *getMemOperand() const { return MMO; }
    1309             : 
    1310             :   const MachinePointerInfo &getPointerInfo() const {
    1311             :     return MMO->getPointerInfo();
    1312             :   }
    1313             : 
    1314             :   /// Return the address space for the associated pointer
    1315             :   unsigned getAddressSpace() const {
    1316    15044914 :     return getPointerInfo().getAddrSpace();
    1317             :   }
    1318             : 
    1319             :   /// Update this MemSDNode's MachineMemOperand information
    1320             :   /// to reflect the alignment of NewMMO, if it has a greater alignment.
    1321             :   /// This must only be used when the new alignment applies to all users of
    1322             :   /// this MachineMemOperand.
    1323             :   void refineAlignment(const MachineMemOperand *NewMMO) {
    1324       29525 :     MMO->refineAlignment(NewMMO);
    1325             :   }
    1326             : 
    1327    33921747 :   const SDValue &getChain() const { return getOperand(0); }
    1328             :   const SDValue &getBasePtr() const {
    1329    34151047 :     return getOperand(getOpcode() == ISD::STORE ? 2 : 1);
    1330             :   }
    1331             : 
    1332             :   // Methods to support isa and dyn_cast
    1333    10357862 :   static bool classof(const SDNode *N) {
    1334             :     // For some targets, we lower some target intrinsics to a MemIntrinsicNode
    1335             :     // with either an intrinsic or a target opcode.
    1336    16424845 :     return N->getOpcode() == ISD::LOAD                ||
    1337      902194 :            N->getOpcode() == ISD::STORE               ||
    1338      901657 :            N->getOpcode() == ISD::PREFETCH            ||
    1339      901344 :            N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
    1340      899717 :            N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
    1341      898200 :            N->getOpcode() == ISD::ATOMIC_SWAP         ||
    1342      896805 :            N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
    1343      896229 :            N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
    1344      895842 :            N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
    1345      895666 :            N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
    1346      895139 :            N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
    1347      894621 :            N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
    1348      894531 :            N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
    1349      894043 :            N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
    1350      893568 :            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
    1351      893097 :            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
    1352      892626 :            N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
    1353      889607 :            N->getOpcode() == ISD::ATOMIC_LOAD         ||
    1354      888698 :            N->getOpcode() == ISD::ATOMIC_STORE        ||
    1355      887892 :            N->getOpcode() == ISD::MLOAD               ||
    1356      887541 :            N->getOpcode() == ISD::MSTORE              ||
    1357      887541 :            N->getOpcode() == ISD::MGATHER             ||
    1358             :            N->getOpcode() == ISD::MSCATTER            ||
    1359    11237497 :            N->isMemIntrinsic()                        ||
    1360    10357862 :            N->isTargetMemoryOpcode();
    1361             :   }
    1362             : };
    1363             : 
    1364             : /// This is an SDNode representing atomic operations.
    1365             : class AtomicSDNode : public MemSDNode {
    1366             : public:
    1367             :   AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
    1368             :                EVT MemVT, MachineMemOperand *MMO)
    1369       11963 :       : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {}
    1370             : 
    1371        1597 :   const SDValue &getBasePtr() const { return getOperand(1); }
    1372          36 :   const SDValue &getVal() const { return getOperand(2); }
    1373             : 
    1374             :   /// Returns true if this SDNode represents cmpxchg atomic operation, false
    1375             :   /// otherwise.
    1376             :   bool isCompareAndSwap() const {
    1377             :     unsigned Op = getOpcode();
    1378             :     return Op == ISD::ATOMIC_CMP_SWAP ||
    1379             :            Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
    1380             :   }
    1381             : 
    1382             :   /// For cmpxchg atomic operations, return the atomic ordering requirements
    1383             :   /// when store does not occur.
    1384             :   AtomicOrdering getFailureOrdering() const {
    1385             :     assert(isCompareAndSwap() && "Must be cmpxchg operation");
    1386             :     return MMO->getFailureOrdering();
    1387             :   }
    1388             : 
    1389             :   // Methods to support isa and dyn_cast
    1390      484930 :   static bool classof(const SDNode *N) {
    1391      969828 :     return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
    1392      484898 :            N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
    1393      484728 :            N->getOpcode() == ISD::ATOMIC_SWAP         ||
    1394      484532 :            N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
    1395      484374 :            N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
    1396      484256 :            N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
    1397      484256 :            N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
    1398      484138 :            N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
    1399      484020 :            N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
    1400      484020 :            N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
    1401      483902 :            N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
    1402      483782 :            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
    1403      483664 :            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
    1404      483546 :            N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
    1405      968350 :            N->getOpcode() == ISD::ATOMIC_LOAD         ||
    1406      484930 :            N->getOpcode() == ISD::ATOMIC_STORE;
    1407             :   }
    1408             : };
    1409             : 
    1410             : /// This SDNode is used for target intrinsics that touch
    1411             : /// memory and need an associated MachineMemOperand. Its opcode may be
    1412             : /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
    1413             : /// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
    1414             : class MemIntrinsicSDNode : public MemSDNode {
    1415             : public:
    1416             :   MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
    1417             :                      SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    1418       30473 :       : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
    1419       30473 :     SDNodeBits.IsMemIntrinsic = true;
    1420             :   }
    1421             : 
    1422             :   // Methods to support isa and dyn_cast
    1423             :   static bool classof(const SDNode *N) {
    1424             :     // We lower some target intrinsics to their target opcode
    1425             :     // early a node with a target opcode can be of this class
    1426         186 :     return N->isMemIntrinsic()             ||
    1427           0 :            N->getOpcode() == ISD::PREFETCH ||
    1428             :            N->isTargetMemoryOpcode();
    1429             :   }
    1430             : };
    1431             : 
    1432             : /// This SDNode is used to implement the code generator
    1433             : /// support for the llvm IR shufflevector instruction.  It combines elements
    1434             : /// from two input vectors into a new input vector, with the selection and
    1435             : /// ordering of elements determined by an array of integers, referred to as
    1436             : /// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
    1437             : /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
    1438             : /// An index of -1 is treated as undef, such that the code generator may put
    1439             : /// any value in the corresponding element of the result.
    1440             : class ShuffleVectorSDNode : public SDNode {
    1441             :   // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
    1442             :   // is freed when the SelectionDAG object is destroyed.
    1443             :   const int *Mask;
    1444             : 
    1445             : protected:
    1446             :   friend class SelectionDAG;
    1447             : 
    1448       97798 :   ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    1449      293394 :       : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
    1450             : 
    1451             : public:
    1452             :   ArrayRef<int> getMask() const {
    1453      623179 :     EVT VT = getValueType(0);
    1454      312407 :     return makeArrayRef(Mask, VT.getVectorNumElements());
    1455             :   }
    1456             : 
    1457             :   int getMaskElt(unsigned Idx) const {
    1458             :     assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!");
    1459     2452442 :     return Mask[Idx];
    1460             :   }
    1461             : 
    1462      148820 :   bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
    1463             : 
    1464       16505 :   int  getSplatIndex() const {
    1465             :     assert(isSplat() && "Cannot get splat index for non-splat!");
    1466       33010 :     EVT VT = getValueType(0);
    1467       17323 :     for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
    1468       17323 :       if (Mask[i] >= 0)
    1469       16505 :         return Mask[i];
    1470             :     }
    1471           0 :     llvm_unreachable("Splat with all undef indices?");
    1472             :   }
    1473             : 
    1474             :   static bool isSplatMask(const int *Mask, EVT VT);
    1475             : 
    1476             :   /// Change values in a shuffle permute mask assuming
    1477             :   /// the two vector operands have swapped position.
    1478             :   static void commuteMask(MutableArrayRef<int> Mask) {
    1479      158742 :     unsigned NumElems = Mask.size();
    1480     4422234 :     for (unsigned i = 0; i != NumElems; ++i) {
    1481     4263492 :       int idx = Mask[i];
    1482     2131746 :       if (idx < 0)
    1483             :         continue;
    1484     1812787 :       else if (idx < (int)NumElems)
    1485     1156161 :         Mask[i] = idx + NumElems;
    1486             :       else
    1487      656626 :         Mask[i] = idx - NumElems;
    1488             :     }
    1489             :   }
    1490             : 
    1491             :   static bool classof(const SDNode *N) {
    1492      411715 :     return N->getOpcode() == ISD::VECTOR_SHUFFLE;
    1493             :   }
    1494             : };
    1495             : 
    1496             : class ConstantSDNode : public SDNode {
    1497             :   friend class SelectionDAG;
    1498             : 
    1499             :   const ConstantInt *Value;
    1500             : 
    1501     3581912 :   ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val,
    1502             :                  const DebugLoc &DL, EVT VT)
    1503     3581912 :       : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DL,
    1504             :                getSDVTList(VT)),
    1505    10745736 :         Value(val) {
    1506     3581912 :     ConstantSDNodeBits.IsOpaque = isOpaque;
    1507     3581912 :   }
    1508             : 
    1509             : public:
    1510             :   const ConstantInt *getConstantIntValue() const { return Value; }
    1511             :   const APInt &getAPIntValue() const { return Value->getValue(); }
    1512             :   uint64_t getZExtValue() const { return Value->getZExtValue(); }
    1513             :   int64_t getSExtValue() const { return Value->getSExtValue(); }
    1514             :   uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) {
    1515             :     return Value->getLimitedValue(Limit);
    1516             :   }
    1517             : 
    1518             :   bool isOne() const { return Value->isOne(); }
    1519             :   bool isNullValue() const { return Value->isZero(); }
    1520             :   bool isAllOnesValue() const { return Value->isMinusOne(); }
    1521             : 
    1522    14323896 :   bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
    1523             : 
    1524             :   static bool classof(const SDNode *N) {
    1525   298563234 :     return N->getOpcode() == ISD::Constant ||
    1526             :            N->getOpcode() == ISD::TargetConstant;
    1527             :   }
    1528             : };
    1529             : 
    1530             : uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
    1531      954642 :   return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
    1532             : }
    1533             : 
    1534             : class ConstantFPSDNode : public SDNode {
    1535             :   friend class SelectionDAG;
    1536             : 
    1537             :   const ConstantFP *Value;
    1538             : 
    1539       27330 :   ConstantFPSDNode(bool isTarget, const ConstantFP *val, const DebugLoc &DL,
    1540             :                    EVT VT)
    1541       27330 :       : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, DL,
    1542             :                getSDVTList(VT)),
    1543       81990 :         Value(val) {}
    1544             : 
    1545             : public:
    1546             :   const APFloat& getValueAPF() const { return Value->getValueAPF(); }
    1547             :   const ConstantFP *getConstantFPValue() const { return Value; }
    1548             : 
    1549             :   /// Return true if the value is positive or negative zero.
    1550             :   bool isZero() const { return Value->isZero(); }
    1551             : 
    1552             :   /// Return true if the value is a NaN.
    1553             :   bool isNaN() const { return Value->isNaN(); }
    1554             : 
    1555             :   /// Return true if the value is an infinity
    1556             :   bool isInfinity() const { return Value->isInfinity(); }
    1557             : 
    1558             :   /// Return true if the value is negative.
    1559             :   bool isNegative() const { return Value->isNegative(); }
    1560             : 
    1561             :   /// We don't rely on operator== working on double values, as
    1562             :   /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
    1563             :   /// As such, this method can be used to do an exact bit-for-bit comparison of
    1564             :   /// two floating point values.
    1565             : 
    1566             :   /// We leave the version with the double argument here because it's just so
    1567             :   /// convenient to write "2.0" and the like.  Without this function we'd
    1568             :   /// have to duplicate its logic everywhere it's called.
    1569             :   bool isExactlyValue(double V) const {
    1570       12383 :     return Value->getValueAPF().isExactlyValue(V);
    1571             :   }
    1572             :   bool isExactlyValue(const APFloat& V) const;
    1573             : 
    1574             :   static bool isValueValidForType(EVT VT, const APFloat& Val);
    1575             : 
    1576             :   static bool classof(const SDNode *N) {
    1577    80107527 :     return N->getOpcode() == ISD::ConstantFP ||
    1578             :            N->getOpcode() == ISD::TargetConstantFP;
    1579             :   }
    1580             : };
    1581             : 
    1582             : /// Returns true if \p V is a constant integer zero.
    1583             : bool isNullConstant(SDValue V);
    1584             : 
    1585             : /// Returns true if \p V is an FP constant with a value of positive zero.
    1586             : bool isNullFPConstant(SDValue V);
    1587             : 
    1588             : /// Returns true if \p V is an integer constant with all bits set.
    1589             : bool isAllOnesConstant(SDValue V);
    1590             : 
    1591             : /// Returns true if \p V is a constant integer one.
    1592             : bool isOneConstant(SDValue V);
    1593             : 
    1594             : /// Returns true if \p V is a bitwise not operation. Assumes that an all ones
    1595             : /// constant is canonicalized to be operand 1.
    1596             : bool isBitwiseNot(SDValue V);
    1597             : 
    1598             : /// Returns the SDNode if it is a constant splat BuildVector or constant int.
    1599             : ConstantSDNode *isConstOrConstSplat(SDValue V);
    1600             : 
    1601             : /// Returns the SDNode if it is a constant splat BuildVector or constant float.
    1602             : ConstantFPSDNode *isConstOrConstSplatFP(SDValue V);
    1603             : 
    1604             : class GlobalAddressSDNode : public SDNode {
    1605             :   friend class SelectionDAG;
    1606             : 
    1607             :   const GlobalValue *TheGlobal;
    1608             :   int64_t Offset;
    1609             :   unsigned char TargetFlags;
    1610             : 
    1611             :   GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
    1612             :                       const GlobalValue *GA, EVT VT, int64_t o,
    1613             :                       unsigned char TargetFlags);
    1614             : 
    1615             : public:
    1616             :   const GlobalValue *getGlobal() const { return TheGlobal; }
    1617             :   int64_t getOffset() const { return Offset; }
    1618             :   unsigned char getTargetFlags() const { return TargetFlags; }
    1619             :   // Return the address space this GlobalAddress belongs to.
    1620             :   unsigned getAddressSpace() const;
    1621             : 
    1622             :   static bool classof(const SDNode *N) {
    1623    84567242 :     return N->getOpcode() == ISD::GlobalAddress ||
    1624    45212572 :            N->getOpcode() == ISD::TargetGlobalAddress ||
    1625   105535246 :            N->getOpcode() == ISD::GlobalTLSAddress ||
    1626             :            N->getOpcode() == ISD::TargetGlobalTLSAddress;
    1627             :   }
    1628             : };
    1629             : 
    1630             : class FrameIndexSDNode : public SDNode {
    1631             :   friend class SelectionDAG;
    1632             : 
    1633             :   int FI;
    1634             : 
    1635      356249 :   FrameIndexSDNode(int fi, EVT VT, bool isTarg)
    1636      356249 :     : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    1637     1068747 :       0, DebugLoc(), getSDVTList(VT)), FI(fi) {
    1638      356249 :   }
    1639             : 
    1640             : public:
    1641             :   int getIndex() const { return FI; }
    1642             : 
    1643             :   static bool classof(const SDNode *N) {
    1644    53169883 :     return N->getOpcode() == ISD::FrameIndex ||
    1645             :            N->getOpcode() == ISD::TargetFrameIndex;
    1646             :   }
    1647             : };
    1648             : 
    1649             : class JumpTableSDNode : public SDNode {
    1650             :   friend class SelectionDAG;
    1651             : 
    1652             :   int JTI;
    1653             :   unsigned char TargetFlags;
    1654             : 
    1655         666 :   JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned char TF)
    1656         666 :     : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    1657        1998 :       0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
    1658         666 :   }
    1659             : 
    1660             : public:
    1661             :   int getIndex() const { return JTI; }
    1662             :   unsigned char getTargetFlags() const { return TargetFlags; }
    1663             : 
    1664             :   static bool classof(const SDNode *N) {
    1665    14039666 :     return N->getOpcode() == ISD::JumpTable ||
    1666             :            N->getOpcode() == ISD::TargetJumpTable;
    1667             :   }
    1668             : };
    1669             : 
    1670             : class ConstantPoolSDNode : public SDNode {
    1671             :   friend class SelectionDAG;
    1672             : 
    1673             :   union {
    1674             :     const Constant *ConstVal;
    1675             :     MachineConstantPoolValue *MachineCPVal;
    1676             :   } Val;
    1677             :   int Offset;  // It's a MachineConstantPoolValue if top bit is set.
    1678             :   unsigned Alignment;  // Minimum alignment requirement of CP (not log2 value).
    1679             :   unsigned char TargetFlags;
    1680             : 
    1681       68937 :   ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
    1682             :                      unsigned Align, unsigned char TF)
    1683       68937 :     : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
    1684      137874 :              DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
    1685      206811 :              TargetFlags(TF) {
    1686             :     assert(Offset >= 0 && "Offset is too large");
    1687       68937 :     Val.ConstVal = c;
    1688       68937 :   }
    1689             : 
    1690         273 :   ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v,
    1691             :                      EVT VT, int o, unsigned Align, unsigned char TF)
    1692         273 :     : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
    1693         546 :              DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
    1694         819 :              TargetFlags(TF) {
    1695             :     assert(Offset >= 0 && "Offset is too large");
    1696         273 :     Val.MachineCPVal = v;
    1697         273 :     Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1);
    1698         273 :   }
    1699             : 
    1700             : public:
    1701             :   bool isMachineConstantPoolEntry() const {
    1702           0 :     return Offset < 0;
    1703             :   }
    1704             : 
    1705             :   const Constant *getConstVal() const {
    1706             :     assert(!isMachineConstantPoolEntry() && "Wrong constantpool type");
    1707             :     return Val.ConstVal;
    1708             :   }
    1709             : 
    1710             :   MachineConstantPoolValue *getMachineCPVal() const {
    1711             :     assert(isMachineConstantPoolEntry() && "Wrong constantpool type");
    1712             :     return Val.MachineCPVal;
    1713             :   }
    1714             : 
    1715             :   int getOffset() const {
    1716       95685 :     return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT-1));
    1717             :   }
    1718             : 
    1719             :   // Return the alignment of this constant pool object, which is either 0 (for
    1720             :   // default alignment) or the desired value.
    1721             :   unsigned getAlignment() const { return Alignment; }
    1722             :   unsigned char getTargetFlags() const { return TargetFlags; }
    1723             : 
    1724             :   Type *getType() const;
    1725             : 
    1726             :   static bool classof(const SDNode *N) {
    1727    23656748 :     return N->getOpcode() == ISD::ConstantPool ||
    1728             :            N->getOpcode() == ISD::TargetConstantPool;
    1729             :   }
    1730             : };
    1731             : 
    1732             : /// Completely target-dependent object reference.
    1733             : class TargetIndexSDNode : public SDNode {
    1734             :   friend class SelectionDAG;
    1735             : 
    1736             :   unsigned char TargetFlags;
    1737             :   int Index;
    1738             :   int64_t Offset;
    1739             : 
    1740             : public:
    1741           0 :   TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF)
    1742           0 :     : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
    1743           0 :       TargetFlags(TF), Index(Idx), Offset(Ofs) {}
    1744             : 
    1745             :   unsigned char getTargetFlags() const { return TargetFlags; }
    1746             :   int getIndex() const { return Index; }
    1747             :   int64_t getOffset() const { return Offset; }
    1748             : 
    1749             :   static bool classof(const SDNode *N) {
    1750             :     return N->getOpcode() == ISD::TargetIndex;
    1751             :   }
    1752             : };
    1753             : 
    1754             : class BasicBlockSDNode : public SDNode {
    1755             :   friend class SelectionDAG;
    1756             : 
    1757             :   MachineBasicBlock *MBB;
    1758             : 
    1759             :   /// Debug info is meaningful and potentially useful here, but we create
    1760             :   /// blocks out of order when they're jumped to, which makes it a bit
    1761             :   /// harder.  Let's see if we need it first.
    1762      163719 :   explicit BasicBlockSDNode(MachineBasicBlock *mbb)
    1763      491157 :     : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
    1764      163719 :   {}
    1765             : 
    1766             : public:
    1767             :   MachineBasicBlock *getBasicBlock() const { return MBB; }
    1768             : 
    1769             :   static bool classof(const SDNode *N) {
    1770             :     return N->getOpcode() == ISD::BasicBlock;
    1771             :   }
    1772             : };
    1773             : 
    1774             : /// A "pseudo-class" with methods for operating on BUILD_VECTORs.
    1775             : class BuildVectorSDNode : public SDNode {
    1776             : public:
    1777             :   // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
    1778             :   explicit BuildVectorSDNode() = delete;
    1779             : 
    1780             :   /// Check if this is a constant splat, and if so, find the
    1781             :   /// smallest element size that splats the vector.  If MinSplatBits is
    1782             :   /// nonzero, the element size must be at least that large.  Note that the
    1783             :   /// splat element may be the entire vector (i.e., a one element vector).
    1784             :   /// Returns the splat element value in SplatValue.  Any undefined bits in
    1785             :   /// that value are zero, and the corresponding bits in the SplatUndef mask
    1786             :   /// are set.  The SplatBitSize value is set to the splat element size in
    1787             :   /// bits.  HasAnyUndefs is set to true if any bits in the vector are
    1788             :   /// undefined.  isBigEndian describes the endianness of the target.
    1789             :   bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
    1790             :                        unsigned &SplatBitSize, bool &HasAnyUndefs,
    1791             :                        unsigned MinSplatBits = 0,
    1792             :                        bool isBigEndian = false) const;
    1793             : 
    1794             :   /// Returns the splatted value or a null value if this is not a splat.
    1795             :   ///
    1796             :   /// If passed a non-null UndefElements bitvector, it will resize it to match
    1797             :   /// the vector width and set the bits where elements are undef.
    1798             :   SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
    1799             : 
    1800             :   /// Returns the splatted constant or null if this is not a constant
    1801             :   /// splat.
    1802             :   ///
    1803             :   /// If passed a non-null UndefElements bitvector, it will resize it to match
    1804             :   /// the vector width and set the bits where elements are undef.
    1805             :   ConstantSDNode *
    1806             :   getConstantSplatNode(BitVector *UndefElements = nullptr) const;
    1807             : 
    1808             :   /// Returns the splatted constant FP or null if this is not a constant
    1809             :   /// FP splat.
    1810             :   ///
    1811             :   /// If passed a non-null UndefElements bitvector, it will resize it to match
    1812             :   /// the vector width and set the bits where elements are undef.
    1813             :   ConstantFPSDNode *
    1814             :   getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
    1815             : 
    1816             :   /// If this is a constant FP splat and the splatted constant FP is an
    1817             :   /// exact power or 2, return the log base 2 integer value.  Otherwise,
    1818             :   /// return -1.
    1819             :   ///
    1820             :   /// The BitWidth specifies the necessary bit precision.
    1821             :   int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
    1822             :                                           uint32_t BitWidth) const;
    1823             : 
    1824             :   bool isConstant() const;
    1825             : 
    1826             :   static bool classof(const SDNode *N) {
    1827    13675845 :     return N->getOpcode() == ISD::BUILD_VECTOR;
    1828             :   }
    1829             : };
    1830             : 
    1831             : /// An SDNode that holds an arbitrary LLVM IR Value. This is
    1832             : /// used when the SelectionDAG needs to make a simple reference to something
    1833             : /// in the LLVM IR representation.
    1834             : ///
    1835             : class SrcValueSDNode : public SDNode {
    1836             :   friend class SelectionDAG;
    1837             : 
    1838             :   const Value *V;
    1839             : 
    1840             :   /// Create a SrcValue for a general value.
    1841         455 :   explicit SrcValueSDNode(const Value *v)
    1842        1365 :     : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
    1843             : 
    1844             : public:
    1845             :   /// Return the contained Value.
    1846             :   const Value *getValue() const { return V; }
    1847             : 
    1848             :   static bool classof(const SDNode *N) {
    1849             :     return N->getOpcode() == ISD::SRCVALUE;
    1850             :   }
    1851             : };
    1852             : 
    1853             : class MDNodeSDNode : public SDNode {
    1854             :   friend class SelectionDAG;
    1855             : 
    1856             :   const MDNode *MD;
    1857             : 
    1858       16844 :   explicit MDNodeSDNode(const MDNode *md)
    1859       50532 :   : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
    1860       16844 :   {}
    1861             : 
    1862             : public:
    1863             :   const MDNode *getMD() const { return MD; }
    1864             : 
    1865             :   static bool classof(const SDNode *N) {
    1866         333 :     return N->getOpcode() == ISD::MDNODE_SDNODE;
    1867             :   }
    1868             : };
    1869             : 
    1870             : class RegisterSDNode : public SDNode {
    1871             :   friend class SelectionDAG;
    1872             : 
    1873             :   unsigned Reg;
    1874             : 
    1875     1758387 :   RegisterSDNode(unsigned reg, EVT VT)
    1876     5275161 :     : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
    1877             : 
    1878             : public:
    1879             :   unsigned getReg() const { return Reg; }
    1880             : 
    1881             :   static bool classof(const SDNode *N) {
    1882     3095462 :     return N->getOpcode() == ISD::Register;
    1883             :   }
    1884             : };
    1885             : 
    1886             : class RegisterMaskSDNode : public SDNode {
    1887             :   friend class SelectionDAG;
    1888             : 
    1889             :   // The memory for RegMask is not owned by the node.
    1890             :   const uint32_t *RegMask;
    1891             : 
    1892      117796 :   RegisterMaskSDNode(const uint32_t *mask)
    1893      353388 :     : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    1894      353388 :       RegMask(mask) {}
    1895             : 
    1896             : public:
    1897             :   const uint32_t *getRegMask() const { return RegMask; }
    1898             : 
    1899             :   static bool classof(const SDNode *N) {
    1900     3774200 :     return N->getOpcode() == ISD::RegisterMask;
    1901             :   }
    1902             : };
    1903             : 
    1904             : class BlockAddressSDNode : public SDNode {
    1905             :   friend class SelectionDAG;
    1906             : 
    1907             :   const BlockAddress *BA;
    1908             :   int64_t Offset;
    1909             :   unsigned char TargetFlags;
    1910             : 
    1911         262 :   BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
    1912             :                      int64_t o, unsigned char Flags)
    1913         786 :     : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
    1914         786 :              BA(ba), Offset(o), TargetFlags(Flags) {}
    1915             : 
    1916             : public:
    1917             :   const BlockAddress *getBlockAddress() const { return BA; }
    1918             :   int64_t getOffset() const { return Offset; }
    1919             :   unsigned char getTargetFlags() const { return TargetFlags; }
    1920             : 
    1921             :   static bool classof(const SDNode *N) {
    1922    13955092 :     return N->getOpcode() == ISD::BlockAddress ||
    1923             :            N->getOpcode() == ISD::TargetBlockAddress;
    1924             :   }
    1925             : };
    1926             : 
    1927             : class LabelSDNode : public SDNode {
    1928             :   friend class SelectionDAG;
    1929             : 
    1930             :   MCSymbol *Label;
    1931             : 
    1932       85481 :   LabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L)
    1933      256443 :       : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {}
    1934             : 
    1935             : public:
    1936             :   MCSymbol *getLabel() const { return Label; }
    1937             : 
    1938             :   static bool classof(const SDNode *N) {
    1939             :     return N->getOpcode() == ISD::EH_LABEL ||
    1940             :            N->getOpcode() == ISD::ANNOTATION_LABEL;
    1941             :   }
    1942             : };
    1943             : 
    1944             : class ExternalSymbolSDNode : public SDNode {
    1945             :   friend class SelectionDAG;
    1946             : 
    1947             :   const char *Symbol;
    1948             :   unsigned char TargetFlags;
    1949             : 
    1950       23509 :   ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned char TF, EVT VT)
    1951       23509 :     : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol,
    1952       70527 :              0, DebugLoc(), getSDVTList(VT)), Symbol(Sym), TargetFlags(TF) {}
    1953             : 
    1954             : public:
    1955             :   const char *getSymbol() const { return Symbol; }
    1956             :   unsigned char getTargetFlags() const { return TargetFlags; }
    1957             : 
    1958             :   static bool classof(const SDNode *N) {
    1959    14036836 :     return N->getOpcode() == ISD::ExternalSymbol ||
    1960             :            N->getOpcode() == ISD::TargetExternalSymbol;
    1961             :   }
    1962             : };
    1963             : 
    1964             : class MCSymbolSDNode : public SDNode {
    1965             :   friend class SelectionDAG;
    1966             : 
    1967             :   MCSymbol *Symbol;
    1968             : 
    1969          53 :   MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    1970         159 :       : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
    1971             : 
    1972             : public:
    1973             :   MCSymbol *getMCSymbol() const { return Symbol; }
    1974             : 
    1975             :   static bool classof(const SDNode *N) {
    1976          16 :     return N->getOpcode() == ISD::MCSymbol;
    1977             :   }
    1978             : };
    1979             : 
    1980             : class CondCodeSDNode : public SDNode {
    1981             :   friend class SelectionDAG;
    1982             : 
    1983             :   ISD::CondCode Condition;
    1984             : 
    1985      116813 :   explicit CondCodeSDNode(ISD::CondCode Cond)
    1986      350439 :     : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    1987      350439 :       Condition(Cond) {}
    1988             : 
    1989             : public:
    1990             :   ISD::CondCode get() const { return Condition; }
    1991             : 
    1992             :   static bool classof(const SDNode *N) {
    1993             :     return N->getOpcode() == ISD::CONDCODE;
    1994             :   }
    1995             : };
    1996             : 
    1997             : /// This class is used to represent EVT's, which are used
    1998             : /// to parameterize some operations.
    1999             : class VTSDNode : public SDNode {
    2000             :   friend class SelectionDAG;
    2001             : 
    2002             :   EVT ValueType;
    2003             : 
    2004       44946 :   explicit VTSDNode(EVT VT)
    2005      134838 :     : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    2006      134838 :       ValueType(VT) {}
    2007             : 
    2008             : public:
    2009             :   EVT getVT() const { return ValueType; }
    2010             : 
    2011             :   static bool classof(const SDNode *N) {
    2012           6 :     return N->getOpcode() == ISD::VALUETYPE;
    2013             :   }
    2014             : };
    2015             : 
    2016             : /// Base class for LoadSDNode and StoreSDNode
    2017             : class LSBaseSDNode : public MemSDNode {
    2018             : public:
    2019             :   LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
    2020             :                SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
    2021             :                MachineMemOperand *MMO)
    2022     3914869 :       : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
    2023     3914869 :     LSBaseSDNodeBits.AddressingMode = AM;
    2024             :     assert(getAddressingMode() == AM && "Value truncated");
    2025             :   }
    2026             : 
    2027             :   const SDValue &getOffset() const {
    2028        6396 :     return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
    2029             :   }
    2030             : 
    2031             :   /// Return the addressing mode for this load or store:
    2032             :   /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
    2033             :   ISD::MemIndexedMode getAddressingMode() const {
    2034    46249380 :     return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
    2035             :   }
    2036             : 
    2037             :   /// Return true if this is a pre/post inc/dec load/store.
    2038             :   bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
    2039             : 
    2040             :   /// Return true if this is NOT a pre/post inc/dec load/store.
    2041             :   bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
    2042             : 
    2043             :   static bool classof(const SDNode *N) {
    2044       17160 :     return N->getOpcode() == ISD::LOAD ||
    2045             :            N->getOpcode() == ISD::STORE;
    2046             :   }
    2047             : };
    2048             : 
    2049             : /// This class is used to represent ISD::LOAD nodes.
    2050             : class LoadSDNode : public LSBaseSDNode {
    2051             :   friend class SelectionDAG;
    2052             : 
    2053             :   LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
    2054             :              ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
    2055             :              MachineMemOperand *MMO)
    2056             :       : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
    2057     1806177 :     LoadSDNodeBits.ExtTy = ETy;
    2058             :     assert(readMem() && "Load MachineMemOperand is not a load!");
    2059             :     assert(!writeMem() && "Load MachineMemOperand is a store!");
    2060             :   }
    2061             : 
    2062             : public:
    2063             :   /// Return whether this is a plain node,
    2064             :   /// or one of the varieties of value-extending loads.
    2065             :   ISD::LoadExtType getExtensionType() const {
    2066    15697262 :     return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
    2067             :   }
    2068             : 
    2069      613260 :   const SDValue &getBasePtr() const { return getOperand(1); }
    2070        4415 :   const SDValue &getOffset() const { return getOperand(2); }
    2071             : 
    2072             :   static bool classof(const SDNode *N) {
    2073    20548244 :     return N->getOpcode() == ISD::LOAD;
    2074             :   }
    2075             : };
    2076             : 
    2077             : /// This class is used to represent ISD::STORE nodes.
    2078             : class StoreSDNode : public LSBaseSDNode {
    2079             :   friend class SelectionDAG;
    2080             : 
    2081             :   StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
    2082             :               ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
    2083             :               MachineMemOperand *MMO)
    2084             :       : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
    2085     2108692 :     StoreSDNodeBits.IsTruncating = isTrunc;
    2086             :     assert(!readMem() && "Store MachineMemOperand is a load!");
    2087             :     assert(writeMem() && "Store MachineMemOperand is not a store!");
    2088             :   }
    2089             : 
    2090             : public:
    2091             :   /// Return true if the op does a truncation before store.
    2092             :   /// For integers this is the same as doing a TRUNCATE and storing the result.
    2093             :   /// For floats, it is the same as doing an FP_ROUND and storing the result.
    2094    14265693 :   bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
    2095             :   void setTruncatingStore(bool Truncating) {
    2096         213 :     StoreSDNodeBits.IsTruncating = Truncating;
    2097             :   }
    2098             : 
    2099    14366518 :   const SDValue &getValue() const { return getOperand(1); }
    2100      823660 :   const SDValue &getBasePtr() const { return getOperand(2); }
    2101           0 :   const SDValue &getOffset() const { return getOperand(3); }
    2102             : 
    2103             :   static bool classof(const SDNode *N) {
    2104    13541021 :     return N->getOpcode() == ISD::STORE;
    2105             :   }
    2106             : };
    2107             : 
    2108             : /// This base class is used to represent MLOAD and MSTORE nodes
    2109             : class MaskedLoadStoreSDNode : public MemSDNode {
    2110             : public:
    2111             :   friend class SelectionDAG;
    2112             : 
    2113             :   MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
    2114             :                         const DebugLoc &dl, SDVTList VTs, EVT MemVT,
    2115             :                         MachineMemOperand *MMO)
    2116        1948 :       : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
    2117             : 
    2118             :   // In the both nodes address is Op1, mask is Op2:
    2119             :   // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value
    2120             :   // MaskedStoreSDNode (Chain, ptr, mask, data)
    2121             :   // Mask is a vector of i1 elements
    2122         148 :   const SDValue &getBasePtr() const { return getOperand(1); }
    2123        2750 :   const SDValue &getMask() const    { return getOperand(2); }
    2124             : 
    2125             :   static bool classof(const SDNode *N) {
    2126             :     return N->getOpcode() == ISD::MLOAD ||
    2127             :            N->getOpcode() == ISD::MSTORE;
    2128             :   }
    2129             : };
    2130             : 
    2131             : /// This class is used to represent an MLOAD node
    2132             : class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
    2133             : public:
    2134             :   friend class SelectionDAG;
    2135             : 
    2136             :   MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
    2137             :                    ISD::LoadExtType ETy, bool IsExpanding, EVT MemVT,
    2138             :                    MachineMemOperand *MMO)
    2139             :       : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) {
    2140        1224 :     LoadSDNodeBits.ExtTy = ETy;
    2141        1224 :     LoadSDNodeBits.IsExpanding = IsExpanding;
    2142             :   }
    2143             : 
    2144             :   ISD::LoadExtType getExtensionType() const {
    2145        2333 :     return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
    2146             :   }
    2147             : 
    2148         141 :   const SDValue &getSrc0() const { return getOperand(3); }
    2149             :   static bool classof(const SDNode *N) {
    2150         569 :     return N->getOpcode() == ISD::MLOAD;
    2151             :   }
    2152             : 
    2153        2952 :   bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
    2154             : };
    2155             : 
    2156             : /// This class is used to represent an MSTORE node
    2157             : class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
    2158             : public:
    2159             :   friend class SelectionDAG;
    2160             : 
    2161             :   MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
    2162             :                     bool isTrunc, bool isCompressing, EVT MemVT,
    2163             :                     MachineMemOperand *MMO)
    2164             :       : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) {
    2165         724 :     StoreSDNodeBits.IsTruncating = isTrunc;
    2166         724 :     StoreSDNodeBits.IsCompressing = isCompressing;
    2167             :   }
    2168             : 
    2169             :   /// Return true if the op does a truncation before store.
    2170             :   /// For integers this is the same as doing a TRUNCATE and storing the result.
    2171             :   /// For floats, it is the same as doing an FP_ROUND and storing the result.
    2172        1001 :   bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
    2173             : 
    2174             :   /// Returns true if the op does a compression to the vector before storing.
    2175             :   /// The node contiguously stores the active elements (integers or floats)
    2176             :   /// in src (those with their respective bit set in writemask k) to unaligned
    2177             :   /// memory at base_addr.
    2178        1342 :   bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
    2179             : 
    2180         416 :   const SDValue &getValue() const { return getOperand(3); }
    2181             : 
    2182             :   static bool classof(const SDNode *N) {
    2183         305 :     return N->getOpcode() == ISD::MSTORE;
    2184             :   }
    2185             : };
    2186             : 
    2187             : /// This is a base class used to represent
    2188             : /// MGATHER and MSCATTER nodes
    2189             : ///
    2190             : class MaskedGatherScatterSDNode : public MemSDNode {
    2191             : public:
    2192             :   friend class SelectionDAG;
    2193             : 
    2194             :   MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
    2195             :                             const DebugLoc &dl, SDVTList VTs, EVT MemVT,
    2196             :                             MachineMemOperand *MMO)
    2197        1156 :       : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
    2198             : 
    2199             :   // In the both nodes address is Op1, mask is Op2:
    2200             :   // MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
    2201             :   // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
    2202             :   // Mask is a vector of i1 elements
    2203             :   const SDValue &getBasePtr() const { return getOperand(3); }
    2204         393 :   const SDValue &getIndex()   const { return getOperand(4); }
    2205         600 :   const SDValue &getMask()    const { return getOperand(2); }
    2206         275 :   const SDValue &getValue()   const { return getOperand(1); }
    2207         156 :   const SDValue &getScale()   const { return getOperand(5); }
    2208             : 
    2209             :   static bool classof(const SDNode *N) {
    2210             :     return N->getOpcode() == ISD::MGATHER ||
    2211             :            N->getOpcode() == ISD::MSCATTER;
    2212             :   }
    2213             : };
    2214             : 
    2215             : /// This class is used to represent an MGATHER node
    2216             : ///
    2217             : class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
    2218             : public:
    2219             :   friend class SelectionDAG;
    2220             : 
    2221             :   MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
    2222             :                      EVT MemVT, MachineMemOperand *MMO)
    2223             :       : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {}
    2224             : 
    2225             :   static bool classof(const SDNode *N) {
    2226             :     return N->getOpcode() == ISD::MGATHER;
    2227             :   }
    2228             : };
    2229             : 
    2230             : /// This class is used to represent an MSCATTER node
    2231             : ///
    2232             : class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
    2233             : public:
    2234             :   friend class SelectionDAG;
    2235             : 
    2236             :   MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
    2237             :                       EVT MemVT, MachineMemOperand *MMO)
    2238             :       : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {}
    2239             : 
    2240             :   static bool classof(const SDNode *N) {
    2241             :     return N->getOpcode() == ISD::MSCATTER;
    2242             :   }
    2243             : };
    2244             : 
    2245             : /// An SDNode that represents everything that will be needed
    2246             : /// to construct a MachineInstr. These nodes are created during the
    2247             : /// instruction selection proper phase.
    2248             : class MachineSDNode : public SDNode {
    2249             : public:
    2250             :   using mmo_iterator = MachineMemOperand **;
    2251             : 
    2252             : private:
    2253             :   friend class SelectionDAG;
    2254             : 
    2255      413720 :   MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    2256     1241160 :       : SDNode(Opc, Order, DL, VTs) {}
    2257             : 
    2258             :   /// Memory reference descriptions for this instruction.
    2259             :   mmo_iterator MemRefs = nullptr;
    2260             :   mmo_iterator MemRefsEnd = nullptr;
    2261             : 
    2262             : public:
    2263             :   mmo_iterator memoperands_begin() const { return MemRefs; }
    2264             :   mmo_iterator memoperands_end() const { return MemRefsEnd; }
    2265             :   bool memoperands_empty() const { return MemRefsEnd == MemRefs; }
    2266             : 
    2267             :   /// Assign this MachineSDNodes's memory reference descriptor
    2268             :   /// list. This does not transfer ownership.
    2269             :   void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
    2270             :     for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI)
    2271             :       assert(*MMI && "Null mem ref detected!");
    2272     3811597 :     MemRefs = NewMemRefs;
    2273     3811597 :     MemRefsEnd = NewMemRefsEnd;
    2274             :   }
    2275             : 
    2276             :   static bool classof(const SDNode *N) {
    2277     4069270 :     return N->isMachineOpcode();
    2278             :   }
    2279             : };
    2280             : 
    2281             : class SDNodeIterator : public std::iterator<std::forward_iterator_tag,
    2282             :                                             SDNode, ptrdiff_t> {
    2283             :   const SDNode *Node;
    2284             :   unsigned Operand;
    2285             : 
    2286             :   SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
    2287             : 
    2288             : public:
    2289             :   bool operator==(const SDNodeIterator& x) const {
    2290             :     return Operand == x.Operand;
    2291             :   }
    2292             :   bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
    2293             : 
    2294             :   pointer operator*() const {
    2295       27768 :     return Node->getOperand(Operand).getNode();
    2296             :   }
    2297             :   pointer operator->() const { return operator*(); }
    2298             : 
    2299             :   SDNodeIterator& operator++() {                // Preincrement
    2300       13862 :     ++Operand;
    2301             :     return *this;
    2302             :   }
    2303             :   SDNodeIterator operator++(int) { // Postincrement
    2304             :     SDNodeIterator tmp = *this; ++*this; return tmp;
    2305             :   }
    2306             :   size_t operator-(SDNodeIterator Other) const {
    2307             :     assert(Node == Other.Node &&
    2308             :            "Cannot compare iterators of two different nodes!");
    2309             :     return Operand - Other.Operand;
    2310             :   }
    2311             : 
    2312             :   static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
    2313             :   static SDNodeIterator end  (const SDNode *N) {
    2314        7746 :     return SDNodeIterator(N, N->getNumOperands());
    2315             :   }
    2316             : 
    2317             :   unsigned getOperand() const { return Operand; }
    2318             :   const SDNode *getNode() const { return Node; }
    2319             : };
    2320             : 
    2321             : template <> struct GraphTraits<SDNode*> {
    2322             :   using NodeRef = SDNode *;
    2323             :   using ChildIteratorType = SDNodeIterator;
    2324             : 
    2325             :   static NodeRef getEntryNode(SDNode *N) { return N; }
    2326             : 
    2327             :   static ChildIteratorType child_begin(NodeRef N) {
    2328             :     return SDNodeIterator::begin(N);
    2329             :   }
    2330             : 
    2331             :   static ChildIteratorType child_end(NodeRef N) {
    2332             :     return SDNodeIterator::end(N);
    2333             :   }
    2334             : };
    2335             : 
    2336             : /// A representation of the largest SDNode, for use in sizeof().
    2337             : ///
    2338             : /// This needs to be a union because the largest node differs on 32 bit systems
    2339             : /// with 4 and 8 byte pointer alignment, respectively.
    2340             : using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
    2341             :                                             BlockAddressSDNode,
    2342             :                                             GlobalAddressSDNode>;
    2343             : 
    2344             : /// The SDNode class with the greatest alignment requirement.
    2345             : using MostAlignedSDNode = GlobalAddressSDNode;
    2346             : 
    2347             : namespace ISD {
    2348             : 
    2349             :   /// Returns true if the specified node is a non-extending and unindexed load.
    2350             :   inline bool isNormalLoad(const SDNode *N) {
    2351             :     const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
    2352     5918877 :     return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    2353             :       Ld->getAddressingMode() == ISD::UNINDEXED;
    2354             :   }
    2355             : 
    2356             :   /// Returns true if the specified node is a non-extending load.
    2357             :   inline bool isNON_EXTLoad(const SDNode *N) {
    2358      303140 :     return isa<LoadSDNode>(N) &&
    2359             :       cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
    2360             :   }
    2361             : 
    2362             :   /// Returns true if the specified node is a EXTLOAD.
    2363             :   inline bool isEXTLoad(const SDNode *N) {
    2364      379528 :     return isa<LoadSDNode>(N) &&
    2365             :       cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
    2366             :   }
    2367             : 
    2368             :   /// Returns true if the specified node is a SEXTLOAD.
    2369             :   inline bool isSEXTLoad(const SDNode *N) {
    2370      193049 :     return isa<LoadSDNode>(N) &&
    2371             :       cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
    2372             :   }
    2373             : 
    2374             :   /// Returns true if the specified node is a ZEXTLOAD.
    2375             :   inline bool isZEXTLoad(const SDNode *N) {
    2376     5014139 :     return isa<LoadSDNode>(N) &&
    2377             :       cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
    2378             :   }
    2379             : 
    2380             :   /// Returns true if the specified node is an unindexed load.
    2381             :   inline bool isUNINDEXEDLoad(const SDNode *N) {
    2382     1424633 :     return isa<LoadSDNode>(N) &&
    2383             :       cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
    2384             :   }
    2385             : 
    2386             :   /// Returns true if the specified node is a non-truncating
    2387             :   /// and unindexed store.
    2388             :   inline bool isNormalStore(const SDNode *N) {
    2389             :     const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
    2390     6450002 :     return St && !St->isTruncatingStore() &&
    2391             :       St->getAddressingMode() == ISD::UNINDEXED;
    2392             :   }
    2393             : 
    2394             :   /// Returns true if the specified node is a non-truncating store.
    2395             :   inline bool isNON_TRUNCStore(const SDNode *N) {
    2396     1899472 :     return isa<StoreSDNode>(N) && !cast<StoreSDNode>(N)->isTruncatingStore();
    2397             :   }
    2398             : 
    2399             :   /// Returns true if the specified node is a truncating store.
    2400             :   inline bool isTRUNCStore(const SDNode *N) {
    2401             :     return isa<StoreSDNode>(N) && cast<StoreSDNode>(N)->isTruncatingStore();
    2402             :   }
    2403             : 
    2404             :   /// Returns true if the specified node is an unindexed store.
    2405             :   inline bool isUNINDEXEDStore(const SDNode *N) {
    2406             :     return isa<StoreSDNode>(N) &&
    2407             :       cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
    2408             :   }
    2409             : 
    2410             :   /// Attempt to match a unary predicate against a scalar/splat constant or
    2411             :   /// every element of a constant BUILD_VECTOR.
    2412             :   bool matchUnaryPredicate(SDValue Op,
    2413             :                            std::function<bool(ConstantSDNode *)> Match);
    2414             : 
    2415             :   /// Attempt to match a binary predicate against a pair of scalar/splat
    2416             :   /// constants or every element of a pair of constant BUILD_VECTORs.
    2417             :   bool matchBinaryPredicate(
    2418             :       SDValue LHS, SDValue RHS,
    2419             :       std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match);
    2420             : 
    2421             : } // end namespace ISD
    2422             : 
    2423             : } // end namespace llvm
    2424             : 
    2425             : #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H

Generated by: LCOV version 1.13