LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - DAGCombiner.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 7726 8134 95.0 %
Date: 2018-06-17 00:07:59 Functions: 287 290 99.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
      11             : // both before and after the DAG is legalized.
      12             : //
      13             : // This pass is not a substitute for the LLVM IR instcombine pass. This pass is
      14             : // primarily intended to handle simplification opportunities that are implicit
      15             : // in the LLVM IR and exposed by the various codegen lowering phases.
      16             : //
      17             : //===----------------------------------------------------------------------===//
      18             : 
      19             : #include "llvm/ADT/APFloat.h"
      20             : #include "llvm/ADT/APInt.h"
      21             : #include "llvm/ADT/ArrayRef.h"
      22             : #include "llvm/ADT/DenseMap.h"
      23             : #include "llvm/ADT/None.h"
      24             : #include "llvm/ADT/Optional.h"
      25             : #include "llvm/ADT/STLExtras.h"
      26             : #include "llvm/ADT/SetVector.h"
      27             : #include "llvm/ADT/SmallBitVector.h"
      28             : #include "llvm/ADT/SmallPtrSet.h"
      29             : #include "llvm/ADT/SmallSet.h"
      30             : #include "llvm/ADT/SmallVector.h"
      31             : #include "llvm/ADT/Statistic.h"
      32             : #include "llvm/Analysis/AliasAnalysis.h"
      33             : #include "llvm/Analysis/MemoryLocation.h"
      34             : #include "llvm/CodeGen/DAGCombine.h"
      35             : #include "llvm/CodeGen/ISDOpcodes.h"
      36             : #include "llvm/CodeGen/MachineFrameInfo.h"
      37             : #include "llvm/CodeGen/MachineFunction.h"
      38             : #include "llvm/CodeGen/MachineMemOperand.h"
      39             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      40             : #include "llvm/CodeGen/SelectionDAG.h"
      41             : #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
      42             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      43             : #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
      44             : #include "llvm/CodeGen/TargetLowering.h"
      45             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      46             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      47             : #include "llvm/CodeGen/ValueTypes.h"
      48             : #include "llvm/IR/Attributes.h"
      49             : #include "llvm/IR/Constant.h"
      50             : #include "llvm/IR/DataLayout.h"
      51             : #include "llvm/IR/DerivedTypes.h"
      52             : #include "llvm/IR/Function.h"
      53             : #include "llvm/IR/LLVMContext.h"
      54             : #include "llvm/IR/Metadata.h"
      55             : #include "llvm/Support/Casting.h"
      56             : #include "llvm/Support/CodeGen.h"
      57             : #include "llvm/Support/CommandLine.h"
      58             : #include "llvm/Support/Compiler.h"
      59             : #include "llvm/Support/Debug.h"
      60             : #include "llvm/Support/ErrorHandling.h"
      61             : #include "llvm/Support/KnownBits.h"
      62             : #include "llvm/Support/MachineValueType.h"
      63             : #include "llvm/Support/MathExtras.h"
      64             : #include "llvm/Support/raw_ostream.h"
      65             : #include "llvm/Target/TargetMachine.h"
      66             : #include "llvm/Target/TargetOptions.h"
      67             : #include <algorithm>
      68             : #include <cassert>
      69             : #include <cstdint>
      70             : #include <functional>
      71             : #include <iterator>
      72             : #include <string>
      73             : #include <tuple>
      74             : #include <utility>
      75             : #include <vector>
      76             : 
      77             : using namespace llvm;
      78             : 
      79             : #define DEBUG_TYPE "dagcombine"
      80             : 
      81             : STATISTIC(NodesCombined   , "Number of dag nodes combined");
      82             : STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
      83             : STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
      84             : STATISTIC(OpsNarrowed     , "Number of load/op/store narrowed");
      85             : STATISTIC(LdStFP2Int      , "Number of fp load/store pairs transformed to int");
      86             : STATISTIC(SlicedLoads, "Number of load sliced");
      87             : 
      88             : static cl::opt<bool>
      89      101169 : CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
      90      101169 :                  cl::desc("Enable DAG combiner's use of IR alias analysis"));
      91             : 
      92             : static cl::opt<bool>
      93      303507 : UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
      94      202338 :         cl::desc("Enable DAG combiner's use of TBAA"));
      95             : 
      96             : #ifndef NDEBUG
      97             : static cl::opt<std::string>
      98             : CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
      99             :                    cl::desc("Only use DAG-combiner alias analysis in this"
     100             :                             " function"));
     101             : #endif
     102             : 
     103             : /// Hidden option to stress test load slicing, i.e., when this option
     104             : /// is enabled, load slicing bypasses most of its profitability guards.
     105             : static cl::opt<bool>
     106      101169 : StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
     107      101169 :                   cl::desc("Bypass the profitability model of load slicing"),
     108      303507 :                   cl::init(false));
     109             : 
     110             : static cl::opt<bool>
     111      303507 :   MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
     112      202338 :                     cl::desc("DAG combiner may split indexing from loads"));
     113             : 
     114             : namespace {
     115             : 
     116     1725464 :   class DAGCombiner {
     117             :     SelectionDAG &DAG;
     118             :     const TargetLowering &TLI;
     119             :     CombineLevel Level;
     120             :     CodeGenOpt::Level OptLevel;
     121             :     bool LegalOperations = false;
     122             :     bool LegalTypes = false;
     123             :     bool ForCodeSize;
     124             : 
     125             :     /// Worklist of all of the nodes that need to be simplified.
     126             :     ///
     127             :     /// This must behave as a stack -- new nodes to process are pushed onto the
     128             :     /// back and when processing we pop off of the back.
     129             :     ///
     130             :     /// The worklist will not contain duplicates but may contain null entries
     131             :     /// due to nodes being deleted from the underlying DAG.
     132             :     SmallVector<SDNode *, 64> Worklist;
     133             : 
     134             :     /// Mapping from an SDNode to its position on the worklist.
     135             :     ///
     136             :     /// This is used to find and remove nodes from the worklist (by nulling
     137             :     /// them) when they are deleted from the underlying DAG. It relies on
     138             :     /// stable indices of nodes within the worklist.
     139             :     DenseMap<SDNode *, unsigned> WorklistMap;
     140             : 
     141             :     /// Set of nodes which have been combined (at least once).
     142             :     ///
     143             :     /// This is used to allow us to reliably add any operands of a DAG node
     144             :     /// which have not yet been combined to the worklist.
     145             :     SmallPtrSet<SDNode *, 32> CombinedNodes;
     146             : 
     147             :     // AA - Used for DAG load/store alias analysis.
     148             :     AliasAnalysis *AA;
     149             : 
     150             :     /// When an instruction is simplified, add all users of the instruction to
     151             :     /// the work lists because they might get more simplified now.
     152             :     void AddUsersToWorklist(SDNode *N) {
     153     6154160 :       for (SDNode *Node : N->uses())
     154     4053358 :         AddToWorklist(Node);
     155             :     }
     156             : 
     157             :     /// Call the node-specific routine that folds each particular type of node.
     158             :     SDValue visit(SDNode *N);
     159             : 
     160             :   public:
     161      862732 :     DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
     162     1725464 :         : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
     163     2588196 :           OptLevel(OL), AA(AA) {
     164      862732 :       ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
     165             : 
     166      862732 :       MaximumLegalStoreInBits = 0;
     167    98351448 :       for (MVT VT : MVT::all_valuetypes())
     168    97488716 :         if (EVT(VT).isSimple() && VT != MVT::Other &&
     169   109823421 :             TLI.isTypeLegal(EVT(VT)) &&
     170    13197437 :             VT.getSizeInBits() >= MaximumLegalStoreInBits)
     171     9719410 :           MaximumLegalStoreInBits = VT.getSizeInBits();
     172      862732 :     }
     173             : 
     174             :     /// Add to the worklist making sure its instance is at the back (next to be
     175             :     /// processed.)
     176    85895091 :     void AddToWorklist(SDNode *N) {
     177             :       assert(N->getOpcode() != ISD::DELETED_NODE &&
     178             :              "Deleted Node added to Worklist");
     179             : 
     180             :       // Skip handle nodes as they can't usefully be combined and confuse the
     181             :       // zero-use deletion strategy.
     182    85895091 :       if (N->getOpcode() == ISD::HANDLENODE)
     183             :         return;
     184             : 
     185   171778918 :       if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
     186    32419380 :         Worklist.push_back(N);
     187             :     }
     188             : 
     189             :     /// Remove all instances of N from the worklist.
     190     4797095 :     void removeFromWorklist(SDNode *N) {
     191             :       CombinedNodes.erase(N);
     192             : 
     193     4797095 :       auto It = WorklistMap.find(N);
     194     4797095 :       if (It == WorklistMap.end())
     195     2904816 :         return; // Not in the worklist.
     196             : 
     197             :       // Null out the entry rather than erasing it to avoid a linear operation.
     198     3784558 :       Worklist[It->second] = nullptr;
     199             :       WorklistMap.erase(It);
     200             :     }
     201             : 
     202             :     void deleteAndRecombine(SDNode *N);
     203             :     bool recursivelyDeleteUnusedNodes(SDNode *N);
     204             : 
     205             :     /// Replaces all uses of the results of one DAG node with new values.
     206             :     SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
     207             :                       bool AddTo = true);
     208             : 
     209             :     /// Replaces all uses of the results of one DAG node with new values.
     210             :     SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
     211      399010 :       return CombineTo(N, &Res, 1, AddTo);
     212             :     }
     213             : 
     214             :     /// Replaces all uses of the results of one DAG node with new values.
     215             :     SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
     216             :                       bool AddTo = true) {
     217      152367 :       SDValue To[] = { Res0, Res1 };
     218      152367 :       return CombineTo(N, To, 2, AddTo);
     219             :     }
     220             : 
     221             :     void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
     222             : 
     223             :   private:
     224             :     unsigned MaximumLegalStoreInBits;
     225             : 
     226             :     /// Check the specified integer node value to see if it can be simplified or
     227             :     /// if things it uses can be simplified by bit propagation.
     228             :     /// If so, return true.
     229     2588440 :     bool SimplifyDemandedBits(SDValue Op) {
     230     2588440 :       unsigned BitWidth = Op.getScalarValueSizeInBits();
     231     2588440 :       APInt Demanded = APInt::getAllOnesValue(BitWidth);
     232     5176880 :       return SimplifyDemandedBits(Op, Demanded);
     233             :     }
     234             : 
     235             :     /// Check the specified vector node value to see if it can be simplified or
     236             :     /// if things it uses can be simplified as it only uses some of the
     237             :     /// elements. If so, return true.
     238       65481 :     bool SimplifyDemandedVectorElts(SDValue Op) {
     239      130962 :       unsigned NumElts = Op.getValueType().getVectorNumElements();
     240       65481 :       APInt Demanded = APInt::getAllOnesValue(NumElts);
     241      130962 :       return SimplifyDemandedVectorElts(Op, Demanded);
     242             :     }
     243             : 
     244             :     bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
     245             :     bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded);
     246             : 
     247             :     bool CombineToPreIndexedLoadStore(SDNode *N);
     248             :     bool CombineToPostIndexedLoadStore(SDNode *N);
     249             :     SDValue SplitIndexingFromLoad(LoadSDNode *LD);
     250             :     bool SliceUpLoad(SDNode *N);
     251             : 
     252             :     /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
     253             :     ///   load.
     254             :     ///
     255             :     /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
     256             :     /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
     257             :     /// \param EltNo index of the vector element to load.
     258             :     /// \param OriginalLoad load that EVE came from to be replaced.
     259             :     /// \returns EVE on success SDValue() on failure.
     260             :     SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
     261             :         SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
     262             :     void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
     263             :     SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
     264             :     SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
     265             :     SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
     266             :     SDValue PromoteIntBinOp(SDValue Op);
     267             :     SDValue PromoteIntShiftOp(SDValue Op);
     268             :     SDValue PromoteExtend(SDValue Op);
     269             :     bool PromoteLoad(SDValue Op);
     270             : 
     271             :     /// Call the node-specific routine that knows how to fold each
     272             :     /// particular type of node. If that doesn't do anything, try the
     273             :     /// target-specific DAG combines.
     274             :     SDValue combine(SDNode *N);
     275             : 
     276             :     // Visitation implementation - Implement dag node combining for different
     277             :     // node types.  The semantics are as follows:
     278             :     // Return Value:
     279             :     //   SDValue.getNode() == 0 - No change was made
     280             :     //   SDValue.getNode() == N - N was replaced, is dead and has been handled.
     281             :     //   otherwise              - N should be replaced by the returned Operand.
     282             :     //
     283             :     SDValue visitTokenFactor(SDNode *N);
     284             :     SDValue visitMERGE_VALUES(SDNode *N);
     285             :     SDValue visitADD(SDNode *N);
     286             :     SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
     287             :     SDValue visitSUB(SDNode *N);
     288             :     SDValue visitADDC(SDNode *N);
     289             :     SDValue visitUADDO(SDNode *N);
     290             :     SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
     291             :     SDValue visitSUBC(SDNode *N);
     292             :     SDValue visitUSUBO(SDNode *N);
     293             :     SDValue visitADDE(SDNode *N);
     294             :     SDValue visitADDCARRY(SDNode *N);
     295             :     SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
     296             :     SDValue visitSUBE(SDNode *N);
     297             :     SDValue visitSUBCARRY(SDNode *N);
     298             :     SDValue visitMUL(SDNode *N);
     299             :     SDValue useDivRem(SDNode *N);
     300             :     SDValue visitSDIV(SDNode *N);
     301             :     SDValue visitUDIV(SDNode *N);
     302             :     SDValue visitREM(SDNode *N);
     303             :     SDValue visitMULHU(SDNode *N);
     304             :     SDValue visitMULHS(SDNode *N);
     305             :     SDValue visitSMUL_LOHI(SDNode *N);
     306             :     SDValue visitUMUL_LOHI(SDNode *N);
     307             :     SDValue visitSMULO(SDNode *N);
     308             :     SDValue visitUMULO(SDNode *N);
     309             :     SDValue visitIMINMAX(SDNode *N);
     310             :     SDValue visitAND(SDNode *N);
     311             :     SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *LocReference);
     312             :     SDValue visitOR(SDNode *N);
     313             :     SDValue visitORLike(SDValue N0, SDValue N1, SDNode *LocReference);
     314             :     SDValue visitXOR(SDNode *N);
     315             :     SDValue SimplifyVBinOp(SDNode *N);
     316             :     SDValue visitSHL(SDNode *N);
     317             :     SDValue visitSRA(SDNode *N);
     318             :     SDValue visitSRL(SDNode *N);
     319             :     SDValue visitRotate(SDNode *N);
     320             :     SDValue visitABS(SDNode *N);
     321             :     SDValue visitBSWAP(SDNode *N);
     322             :     SDValue visitBITREVERSE(SDNode *N);
     323             :     SDValue visitCTLZ(SDNode *N);
     324             :     SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
     325             :     SDValue visitCTTZ(SDNode *N);
     326             :     SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
     327             :     SDValue visitCTPOP(SDNode *N);
     328             :     SDValue visitSELECT(SDNode *N);
     329             :     SDValue visitVSELECT(SDNode *N);
     330             :     SDValue visitSELECT_CC(SDNode *N);
     331             :     SDValue visitSETCC(SDNode *N);
     332             :     SDValue visitSETCCCARRY(SDNode *N);
     333             :     SDValue visitSIGN_EXTEND(SDNode *N);
     334             :     SDValue visitZERO_EXTEND(SDNode *N);
     335             :     SDValue visitANY_EXTEND(SDNode *N);
     336             :     SDValue visitAssertExt(SDNode *N);
     337             :     SDValue visitSIGN_EXTEND_INREG(SDNode *N);
     338             :     SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
     339             :     SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
     340             :     SDValue visitTRUNCATE(SDNode *N);
     341             :     SDValue visitBITCAST(SDNode *N);
     342             :     SDValue visitBUILD_PAIR(SDNode *N);
     343             :     SDValue visitFADD(SDNode *N);
     344             :     SDValue visitFSUB(SDNode *N);
     345             :     SDValue visitFMUL(SDNode *N);
     346             :     SDValue visitFMA(SDNode *N);
     347             :     SDValue visitFDIV(SDNode *N);
     348             :     SDValue visitFREM(SDNode *N);
     349             :     SDValue visitFSQRT(SDNode *N);
     350             :     SDValue visitFCOPYSIGN(SDNode *N);
     351             :     SDValue visitSINT_TO_FP(SDNode *N);
     352             :     SDValue visitUINT_TO_FP(SDNode *N);
     353             :     SDValue visitFP_TO_SINT(SDNode *N);
     354             :     SDValue visitFP_TO_UINT(SDNode *N);
     355             :     SDValue visitFP_ROUND(SDNode *N);
     356             :     SDValue visitFP_ROUND_INREG(SDNode *N);
     357             :     SDValue visitFP_EXTEND(SDNode *N);
     358             :     SDValue visitFNEG(SDNode *N);
     359             :     SDValue visitFABS(SDNode *N);
     360             :     SDValue visitFCEIL(SDNode *N);
     361             :     SDValue visitFTRUNC(SDNode *N);
     362             :     SDValue visitFFLOOR(SDNode *N);
     363             :     SDValue visitFMINNUM(SDNode *N);
     364             :     SDValue visitFMAXNUM(SDNode *N);
     365             :     SDValue visitBRCOND(SDNode *N);
     366             :     SDValue visitBR_CC(SDNode *N);
     367             :     SDValue visitLOAD(SDNode *N);
     368             : 
     369             :     SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
     370             :     SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
     371             : 
     372             :     SDValue visitSTORE(SDNode *N);
     373             :     SDValue visitINSERT_VECTOR_ELT(SDNode *N);
     374             :     SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
     375             :     SDValue visitBUILD_VECTOR(SDNode *N);
     376             :     SDValue visitCONCAT_VECTORS(SDNode *N);
     377             :     SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
     378             :     SDValue visitVECTOR_SHUFFLE(SDNode *N);
     379             :     SDValue visitSCALAR_TO_VECTOR(SDNode *N);
     380             :     SDValue visitINSERT_SUBVECTOR(SDNode *N);
     381             :     SDValue visitMLOAD(SDNode *N);
     382             :     SDValue visitMSTORE(SDNode *N);
     383             :     SDValue visitMGATHER(SDNode *N);
     384             :     SDValue visitMSCATTER(SDNode *N);
     385             :     SDValue visitFP_TO_FP16(SDNode *N);
     386             :     SDValue visitFP16_TO_FP(SDNode *N);
     387             : 
     388             :     SDValue visitFADDForFMACombine(SDNode *N);
     389             :     SDValue visitFSUBForFMACombine(SDNode *N);
     390             :     SDValue visitFMULForFMADistributiveCombine(SDNode *N);
     391             : 
     392             :     SDValue XformToShuffleWithZero(SDNode *N);
     393             :     SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue LHS,
     394             :                            SDValue RHS);
     395             : 
     396             :     SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
     397             : 
     398             :     SDValue foldSelectOfConstants(SDNode *N);
     399             :     SDValue foldVSelectOfConstants(SDNode *N);
     400             :     SDValue foldBinOpIntoSelect(SDNode *BO);
     401             :     bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
     402             :     SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
     403             :     SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
     404             :     SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
     405             :                              SDValue N2, SDValue N3, ISD::CondCode CC,
     406             :                              bool NotExtCompare = false);
     407             :     SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
     408             :                                    SDValue N2, SDValue N3, ISD::CondCode CC);
     409             :     SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
     410             :                               const SDLoc &DL);
     411             :     SDValue unfoldMaskedMerge(SDNode *N);
     412             :     SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
     413             :                           const SDLoc &DL, bool foldBooleans);
     414             :     SDValue rebuildSetCC(SDValue N);
     415             : 
     416             :     bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
     417             :                            SDValue &CC) const;
     418             :     bool isOneUseSetCC(SDValue N) const;
     419             : 
     420             :     SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
     421             :                                          unsigned HiOp);
     422             :     SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
     423             :     SDValue CombineExtLoad(SDNode *N);
     424             :     SDValue CombineZExtLogicopShiftLoad(SDNode *N);
     425             :     SDValue combineRepeatedFPDivisors(SDNode *N);
     426             :     SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
     427             :     SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
     428             :     SDValue BuildSDIV(SDNode *N);
     429             :     SDValue BuildSDIVPow2(SDNode *N);
     430             :     SDValue BuildUDIV(SDNode *N);
     431             :     SDValue BuildLogBase2(SDValue Op, const SDLoc &DL);
     432             :     SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
     433             :     SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
     434             :     SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
     435             :     SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
     436             :     SDValue buildSqrtNROneConst(SDValue Op, SDValue Est, unsigned Iterations,
     437             :                                 SDNodeFlags Flags, bool Reciprocal);
     438             :     SDValue buildSqrtNRTwoConst(SDValue Op, SDValue Est, unsigned Iterations,
     439             :                                 SDNodeFlags Flags, bool Reciprocal);
     440             :     SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
     441             :                                bool DemandHighBits = true);
     442             :     SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
     443             :     SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
     444             :                               SDValue InnerPos, SDValue InnerNeg,
     445             :                               unsigned PosOpcode, unsigned NegOpcode,
     446             :                               const SDLoc &DL);
     447             :     SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
     448             :     SDValue MatchLoadCombine(SDNode *N);
     449             :     SDValue ReduceLoadWidth(SDNode *N);
     450             :     SDValue ReduceLoadOpStoreWidth(SDNode *N);
     451             :     SDValue splitMergedValStore(StoreSDNode *ST);
     452             :     SDValue TransformFPLoadStorePair(SDNode *N);
     453             :     SDValue convertBuildVecZextToZext(SDNode *N);
     454             :     SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
     455             :     SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
     456             :     SDValue reduceBuildVecToShuffle(SDNode *N);
     457             :     SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
     458             :                                   ArrayRef<int> VectorMask, SDValue VecIn1,
     459             :                                   SDValue VecIn2, unsigned LeftIdx);
     460             :     SDValue matchVSelectOpSizesWithSetCC(SDNode *N);
     461             : 
     462             :     /// Walk up chain skipping non-aliasing memory nodes,
     463             :     /// looking for aliasing nodes and adding them to the Aliases vector.
     464             :     void GatherAllAliases(SDNode *N, SDValue OriginalChain,
     465             :                           SmallVectorImpl<SDValue> &Aliases);
     466             : 
     467             :     /// Return true if there is any possibility that the two addresses overlap.
     468             :     bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
     469             : 
     470             :     /// Walk up chain skipping non-aliasing memory nodes, looking for a better
     471             :     /// chain (aliasing node.)
     472             :     SDValue FindBetterChain(SDNode *N, SDValue Chain);
     473             : 
     474             :     /// Try to replace a store and any possibly adjacent stores on
     475             :     /// consecutive chains with better chains. Return true only if St is
     476             :     /// replaced.
     477             :     ///
     478             :     /// Notice that other chains may still be replaced even if the function
     479             :     /// returns false.
     480             :     bool findBetterNeighborChains(StoreSDNode *St);
     481             : 
     482             :     /// Match "(X shl/srl V1) & V2" where V2 may not be present.
     483             :     bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask);
     484             : 
     485             :     /// Holds a pointer to an LSBaseSDNode as well as information on where it
     486             :     /// is located in a sequence of memory operations connected by a chain.
     487             :     struct MemOpLink {
     488             :       // Ptr to the mem node.
     489             :       LSBaseSDNode *MemNode;
     490             : 
     491             :       // Offset from the base ptr.
     492             :       int64_t OffsetFromBase;
     493             : 
     494             :       MemOpLink(LSBaseSDNode *N, int64_t Offset)
     495      849251 :           : MemNode(N), OffsetFromBase(Offset) {}
     496             :     };
     497             : 
     498             :     /// This is a helper function for visitMUL to check the profitability
     499             :     /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
     500             :     /// MulNode is the original multiply, AddNode is (add x, c1),
     501             :     /// and ConstNode is c2.
     502             :     bool isMulAddWithConstProfitable(SDNode *MulNode,
     503             :                                      SDValue &AddNode,
     504             :                                      SDValue &ConstNode);
     505             : 
     506             :     /// This is a helper function for visitAND and visitZERO_EXTEND.  Returns
     507             :     /// true if the (and (load x) c) pattern matches an extload.  ExtVT returns
     508             :     /// the type of the loaded value to be extended.
     509             :     bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
     510             :                           EVT LoadResultTy, EVT &ExtVT);
     511             : 
     512             :     /// Helper function to calculate whether the given Load can have its
     513             :     /// width reduced to ExtVT.
     514             :     bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
     515             :                            EVT &ExtVT, unsigned ShAmt = 0);
     516             : 
     517             :     /// Used by BackwardsPropagateMask to find suitable loads.
     518             :     bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
     519             :                            SmallPtrSetImpl<SDNode*> &NodeWithConsts,
     520             :                            ConstantSDNode *Mask, SDNode *&UncombinedNode);
     521             :     /// Attempt to propagate a given AND node back to load leaves so that they
     522             :     /// can be combined into narrow loads.
     523             :     bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
     524             : 
     525             :     /// Helper function for MergeConsecutiveStores which merges the
     526             :     /// component store chains.
     527             :     SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
     528             :                                 unsigned NumStores);
     529             : 
     530             :     /// This is a helper function for MergeConsecutiveStores. When the
     531             :     /// source elements of the consecutive stores are all constants or
     532             :     /// all extracted vector elements, try to merge them into one
     533             :     /// larger store introducing bitcasts if necessary.  \return True
     534             :     /// if a merged store was created.
     535             :     bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
     536             :                                          EVT MemVT, unsigned NumStores,
     537             :                                          bool IsConstantSrc, bool UseVector,
     538             :                                          bool UseTrunc);
     539             : 
     540             :     /// This is a helper function for MergeConsecutiveStores. Stores
     541             :     /// that potentially may be merged with St are placed in
     542             :     /// StoreNodes. RootNode is a chain predecessor to all store
     543             :     /// candidates.
     544             :     void getStoreMergeCandidates(StoreSDNode *St,
     545             :                                  SmallVectorImpl<MemOpLink> &StoreNodes,
     546             :                                  SDNode *&Root);
     547             : 
     548             :     /// Helper function for MergeConsecutiveStores. Checks if
     549             :     /// candidate stores have indirect dependency through their
     550             :     /// operands. RootNode is the predecessor to all stores calculated
     551             :     /// by getStoreMergeCandidates and is used to prune the dependency check.
     552             :     /// \return True if safe to merge.
     553             :     bool checkMergeStoreCandidatesForDependencies(
     554             :         SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
     555             :         SDNode *RootNode);
     556             : 
     557             :     /// Merge consecutive store operations into a wide store.
     558             :     /// This optimization uses wide integers or vectors when possible.
     559             :     /// \return number of stores that were merged into a merged store (the
     560             :     /// affected nodes are stored as a prefix in \p StoreNodes).
     561             :     bool MergeConsecutiveStores(StoreSDNode *N);
     562             : 
     563             :     /// Try to transform a truncation where C is a constant:
     564             :     ///     (trunc (and X, C)) -> (and (trunc X), (trunc C))
     565             :     ///
     566             :     /// \p N needs to be a truncation and its first operand an AND. Other
     567             :     /// requirements are checked by the function (e.g. that trunc is
     568             :     /// single-use) and if missed an empty SDValue is returned.
     569             :     SDValue distributeTruncateThroughAnd(SDNode *N);
     570             : 
     571             :     /// Helper function to determine whether the target supports operation
     572             :     /// given by \p Opcode for type \p VT, that is, whether the operation
     573             :     /// is legal or custom before legalizing operations, and whether is
     574             :     /// legal (but not custom) after legalization.
     575      235402 :     bool hasOperation(unsigned Opcode, EVT VT) {
     576      235402 :       if (LegalOperations)
     577      137952 :         return TLI.isOperationLegal(Opcode, VT);
     578       97450 :       return TLI.isOperationLegalOrCustom(Opcode, VT);
     579             :     }
     580             : 
     581             :   public:
     582             :     /// Runs the dag combiner on all nodes in the work list
     583             :     void Run(CombineLevel AtLevel);
     584             : 
     585             :     SelectionDAG &getDAG() const { return DAG; }
     586             : 
     587             :     /// Returns a type large enough to hold any valid shift amount - before type
     588             :     /// legalization these can be huge.
     589       28783 :     EVT getShiftAmountTy(EVT LHSTy) {
     590             :       assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
     591       57566 :       return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
     592             :     }
     593             : 
     594             :     /// This method returns true if we are running before type legalization or
     595             :     /// if the specified VT is legal.
     596             :     bool isTypeLegal(const EVT &VT) {
     597      366697 :       if (!LegalTypes) return true;
     598        6378 :       return TLI.isTypeLegal(VT);
     599             :     }
     600             : 
     601             :     /// Convenience wrapper around TargetLowering::getSetCCResultType
     602      139002 :     EVT getSetCCResultType(EVT VT) const {
     603      278004 :       return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
     604             :     }
     605             : 
     606             :     void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
     607             :                          SDValue OrigLoad, SDValue ExtLoad,
     608             :                          ISD::NodeType ExtType);
     609             :   };
     610             : 
     611             : /// This class is a DAGUpdateListener that removes any deleted
     612             : /// nodes from the worklist.
     613        2456 : class WorklistRemover : public SelectionDAG::DAGUpdateListener {
     614             :   DAGCombiner &DC;
     615             : 
     616             : public:
     617             :   explicit WorklistRemover(DAGCombiner &dc)
     618    61205670 :     : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
     619             : 
     620       24002 :   void NodeDeleted(SDNode *N, SDNode *E) override {
     621       24002 :     DC.removeFromWorklist(N);
     622       24002 :   }
     623             : };
     624             : 
     625             : } // end anonymous namespace
     626             : 
     627             : //===----------------------------------------------------------------------===//
     628             : //  TargetLowering::DAGCombinerInfo implementation
     629             : //===----------------------------------------------------------------------===//
     630             : 
     631       10490 : void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
     632       10490 :   ((DAGCombiner*)DC)->AddToWorklist(N);
     633       10490 : }
     634             : 
     635         843 : SDValue TargetLowering::DAGCombinerInfo::
     636             : CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
     637         843 :   return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
     638             : }
     639             : 
     640        7897 : SDValue TargetLowering::DAGCombinerInfo::
     641             : CombineTo(SDNode *N, SDValue Res, bool AddTo) {
     642       15794 :   return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
     643             : }
     644             : 
     645        3196 : SDValue TargetLowering::DAGCombinerInfo::
     646             : CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
     647        6392 :   return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
     648             : }
     649             : 
     650        1597 : void TargetLowering::DAGCombinerInfo::
     651             : CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
     652        1597 :   return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
     653             : }
     654             : 
     655             : //===----------------------------------------------------------------------===//
     656             : // Helper Functions
     657             : //===----------------------------------------------------------------------===//
     658             : 
     659      719689 : void DAGCombiner::deleteAndRecombine(SDNode *N) {
     660      719689 :   removeFromWorklist(N);
     661             : 
     662             :   // If the operands of this node are only used by the node, they will now be
     663             :   // dead. Make sure to re-visit them and recursively delete dead nodes.
     664     5104471 :   for (const SDValue &Op : N->ops())
     665             :     // For an operand generating multiple values, one of the values may
     666             :     // become dead allowing further simplification (e.g. split index
     667             :     // arithmetic from an indexed load).
     668     4265831 :     if (Op->hasOneUse() || Op->getNumValues() > 1)
     669      472399 :       AddToWorklist(Op.getNode());
     670             : 
     671      719689 :   DAG.DeleteNode(N);
     672      719689 : }
     673             : 
     674             : /// Return 1 if we can compute the negated form of the specified expression for
     675             : /// the same cost as the expression itself, or 2 if we can compute the negated
     676             : /// form more cheaply than the expression itself.
     677      184434 : static char isNegatibleForFree(SDValue Op, bool LegalOperations,
     678             :                                const TargetLowering &TLI,
     679             :                                const TargetOptions *Options,
     680             :                                unsigned Depth = 0) {
     681             :   // fneg is removable even if it has multiple uses.
     682      184434 :   if (Op.getOpcode() == ISD::FNEG) return 2;
     683             : 
     684             :   // Don't allow anything with multiple uses unless we know it is free.
     685      183619 :   EVT VT = Op.getValueType();
     686      183619 :   const SDNodeFlags Flags = Op->getFlags();
     687      183619 :   if (!Op.hasOneUse())
     688       47064 :     if (!(Op.getOpcode() == ISD::FP_EXTEND &&
     689         885 :           TLI.isFPExtFree(VT, Op.getOperand(0).getValueType())))
     690             :       return 0;
     691             : 
     692             :   // Don't recurse exponentially.
     693      136856 :   if (Depth > 6) return 0;
     694             : 
     695      133811 :   switch (Op.getOpcode()) {
     696             :   default: return false;
     697        4061 :   case ISD::ConstantFP: {
     698        4061 :     if (!LegalOperations)
     699             :       return 1;
     700             : 
     701             :     // Don't invert constant FP values after legalization unless the target says
     702             :     // the negated constant is legal.
     703         201 :     return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
     704        3084 :       TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
     705             :   }
     706        8305 :   case ISD::FADD:
     707        8305 :     if (!Options->UnsafeFPMath && !Flags.hasNoSignedZeros())
     708             :       return 0;
     709             : 
     710             :     // After operation legalization, it might not be legal to create new FSUBs.
     711        1579 :     if (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
     712             :       return 0;
     713             : 
     714             :     // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     715        3158 :     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
     716        1579 :                                     Options, Depth + 1))
     717        1560 :       return V;
     718             :     // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
     719             :     return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
     720        1560 :                               Depth + 1);
     721        2241 :   case ISD::FSUB:
     722             :     // We can't turn -(A-B) into B-A when we honor signed zeros.
     723        2241 :     if (!Options->NoSignedZerosFPMath &&
     724             :         !Flags.hasNoSignedZeros())
     725             :       return 0;
     726             : 
     727             :     // fold (fneg (fsub A, B)) -> (fsub B, A)
     728             :     return 1;
     729             : 
     730       34674 :   case ISD::FMUL:
     731             :   case ISD::FDIV:
     732             :     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
     733       69348 :     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
     734       34674 :                                     Options, Depth + 1))
     735       33721 :       return V;
     736             : 
     737             :     return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
     738       33721 :                               Depth + 1);
     739             : 
     740        2646 :   case ISD::FP_EXTEND:
     741             :   case ISD::FP_ROUND:
     742             :   case ISD::FSIN:
     743        2646 :     return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
     744        2646 :                               Depth + 1);
     745             :   }
     746             : }
     747             : 
     748             : /// If isNegatibleForFree returns true, return the newly negated expression.
     749         861 : static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
     750             :                                     bool LegalOperations, unsigned Depth = 0) {
     751         861 :   const TargetOptions &Options = DAG.getTarget().Options;
     752             :   // fneg is removable even if it has multiple uses.
     753        1295 :   if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
     754             : 
     755             :   assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
     756             : 
     757         427 :   const SDNodeFlags Flags = Op.getNode()->getFlags();
     758             : 
     759         427 :   switch (Op.getOpcode()) {
     760           0 :   default: llvm_unreachable("Unknown code");
     761             :   case ISD::ConstantFP: {
     762         148 :     APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
     763         148 :     V.changeSign();
     764         296 :     return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
     765             :   }
     766          18 :   case ISD::FADD:
     767             :     assert(Options.UnsafeFPMath || Flags.hasNoSignedZeros());
     768             : 
     769             :     // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     770          36 :     if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
     771             :                            DAG.getTargetLoweringInfo(), &Options, Depth+1))
     772          18 :       return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     773             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     774             :                                               LegalOperations, Depth+1),
     775          36 :                          Op.getOperand(1), Flags);
     776             :     // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
     777           0 :     return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     778             :                        GetNegatedExpression(Op.getOperand(1), DAG,
     779             :                                             LegalOperations, Depth+1),
     780           0 :                        Op.getOperand(0), Flags);
     781          18 :   case ISD::FSUB:
     782             :     // fold (fneg (fsub 0, B)) -> B
     783             :     if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
     784          10 :       if (N0CFP->isZero())
     785           5 :         return Op.getOperand(1);
     786             : 
     787             :     // fold (fneg (fsub A, B)) -> (fsub B, A)
     788          13 :     return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     789          13 :                        Op.getOperand(1), Op.getOperand(0), Flags);
     790             : 
     791         215 :   case ISD::FMUL:
     792             :   case ISD::FDIV:
     793             :     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
     794         430 :     if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
     795             :                            DAG.getTargetLoweringInfo(), &Options, Depth+1))
     796         100 :       return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     797             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     798             :                                               LegalOperations, Depth+1),
     799         200 :                          Op.getOperand(1), Flags);
     800             : 
     801             :     // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
     802         115 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     803             :                        Op.getOperand(0),
     804             :                        GetNegatedExpression(Op.getOperand(1), DAG,
     805         230 :                                             LegalOperations, Depth+1), Flags);
     806             : 
     807             :   case ISD::FP_EXTEND:
     808             :   case ISD::FSIN:
     809          16 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     810             :                        GetNegatedExpression(Op.getOperand(0), DAG,
     811          64 :                                             LegalOperations, Depth+1));
     812             :   case ISD::FP_ROUND:
     813          12 :       return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
     814             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     815             :                                               LegalOperations, Depth+1),
     816          48 :                          Op.getOperand(1));
     817             :   }
     818             : }
     819             : 
     820             : // APInts must be the same size for most operations, this helper
     821             : // function zero extends the shorter of the pair so that they match.
     822             : // We provide an Offset so that we can create bitwidths that won't overflow.
     823        9390 : static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
     824       18780 :   unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
     825       18780 :   LHS = LHS.zextOrSelf(Bits);
     826       18780 :   RHS = RHS.zextOrSelf(Bits);
     827        9390 : }
     828             : 
     829             : // Return true if this node is a setcc, or is a select_cc
     830             : // that selects between the target values used for true and false, making it
     831             : // equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
     832             : // the appropriate nodes based on the type of node we are checking. This
     833             : // simplifies life a bit for the callers.
     834      424337 : bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
     835             :                                     SDValue &CC) const {
     836      424337 :   if (N.getOpcode() == ISD::SETCC) {
     837       86519 :     LHS = N.getOperand(0);
     838       86519 :     RHS = N.getOperand(1);
     839       86519 :     CC  = N.getOperand(2);
     840             :     return true;
     841             :   }
     842             : 
     843         398 :   if (N.getOpcode() != ISD::SELECT_CC ||
     844      338498 :       !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
     845         564 :       !TLI.isConstFalseVal(N.getOperand(3).getNode()))
     846             :     return false;
     847             : 
     848         560 :   if (TLI.getBooleanContents(N.getValueType()) ==
     849             :       TargetLowering::UndefinedBooleanContent)
     850             :     return false;
     851             : 
     852         280 :   LHS = N.getOperand(0);
     853         280 :   RHS = N.getOperand(1);
     854         280 :   CC  = N.getOperand(4);
     855             :   return true;
     856             : }
     857             : 
     858             : /// Return true if this is a SetCC-equivalent operation with only one use.
     859             : /// If this is true, it allows the users to invert the operation for free when
     860             : /// it is profitable to do so.
     861         111 : bool DAGCombiner::isOneUseSetCC(SDValue N) const {
     862         111 :   SDValue N0, N1, N2;
     863         111 :   if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
     864             :     return true;
     865             :   return false;
     866             : }
     867             : 
     868             : static SDValue peekThroughBitcast(SDValue V) {
     869     4098437 :   while (V.getOpcode() == ISD::BITCAST)
     870       93712 :     V = V.getOperand(0);
     871             :   return V;
     872             : }
     873             : 
     874             : // Returns the SDNode if it is a constant float BuildVector
     875             : // or constant float.
     876             : static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
     877             :   if (isa<ConstantFPSDNode>(N))
     878             :     return N.getNode();
     879     1058959 :   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
     880             :     return N.getNode();
     881             :   return nullptr;
     882             : }
     883             : 
     884             : // Determines if it is a constant integer or a build vector of constant
     885             : // integers (and undefs).
     886             : // Do not permit build vector implicit truncation.
     887     5427387 : static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
     888             :   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
     889     3613498 :     return !(Const->isOpaque() && NoOpaques);
     890     1813889 :   if (N.getOpcode() != ISD::BUILD_VECTOR)
     891             :     return false;
     892      398685 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     893     1749086 :   for (const SDValue &Op : N->op_values()) {
     894     1912704 :     if (Op.isUndef())
     895        1787 :       continue;
     896             :     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
     897     1900939 :     if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
     898           0 :         (Const->isOpaque() && NoOpaques))
     899             :       return false;
     900             :   }
     901             :   return true;
     902             : }
     903             : 
     904             : // Determines if it is a constant null integer or a splatted vector of a
     905             : // constant null integer (with no undefs).
     906             : // Build vector implicit truncation is not an issue for null values.
     907      752759 : static bool isNullConstantOrNullSplatConstant(SDValue N) {
     908             :   // TODO: may want to use peekThroughBitcast() here.
     909      752759 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     910       15937 :     return Splat->isNullValue();
     911             :   return false;
     912             : }
     913             : 
     914             : // Determines if it is a constant integer of one or a splatted vector of a
     915             : // constant integer of one (with no undefs).
     916             : // Do not permit build vector implicit truncation.
     917         283 : static bool isOneConstantOrOneSplatConstant(SDValue N) {
     918             :   // TODO: may want to use peekThroughBitcast() here.
     919         283 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     920         283 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     921         534 :     return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
     922             :   return false;
     923             : }
     924             : 
     925             : // Determines if it is a constant integer of all ones or a splatted vector of a
     926             : // constant integer of all ones (with no undefs).
     927             : // Do not permit build vector implicit truncation.
     928      475856 : static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
     929      475856 :   N = peekThroughBitcast(N);
     930      475856 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     931      475856 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     932       79458 :     return Splat->isAllOnesValue() &&
     933             :            Splat->getAPIntValue().getBitWidth() == BitWidth;
     934             :   return false;
     935             : }
     936             : 
     937             : // Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
     938             : // undef's.
     939       24026 : static bool isAnyConstantBuildVector(const SDNode *N) {
     940       45662 :   return ISD::isBuildVectorOfConstantSDNodes(N) ||
     941       45662 :          ISD::isBuildVectorOfConstantFPSDNodes(N);
     942             : }
     943             : 
     944     2431379 : SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
     945             :                                     SDValue N1) {
     946     2431379 :   EVT VT = N0.getValueType();
     947     2431379 :   if (N0.getOpcode() == Opc) {
     948      996362 :     if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
     949      199033 :       if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
     950             :         // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
     951      196762 :         if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
     952      393482 :           return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
     953          21 :         return SDValue();
     954             :       }
     955        2271 :       if (N0.hasOneUse()) {
     956             :         // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
     957             :         // use
     958        4410 :         SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
     959        1470 :         if (!OpNode.getNode())
     960           0 :           return SDValue();
     961        1470 :         AddToWorklist(OpNode.getNode());
     962        2940 :         return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
     963             :       }
     964             :     }
     965             :   }
     966             : 
     967     2233147 :   if (N1.getOpcode() == Opc) {
     968       63400 :     if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
     969       26683 :       if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
     970             :         // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
     971           0 :         if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
     972           0 :           return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
     973           0 :         return SDValue();
     974             :       }
     975       53366 :       if (N1.hasOneUse()) {
     976             :         // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
     977             :         // use
     978       78252 :         SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
     979       26084 :         if (!OpNode.getNode())
     980           0 :           return SDValue();
     981       26084 :         AddToWorklist(OpNode.getNode());
     982       52168 :         return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
     983             :       }
     984             :     }
     985             :   }
     986             : 
     987     2207063 :   return SDValue();
     988             : }
     989             : 
     990      552220 : SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
     991             :                                bool AddTo) {
     992             :   assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
     993             :   ++NodesCombined;
     994             :   LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
     995             :              To[0].getNode()->dump(&DAG);
     996             :              dbgs() << " and " << NumTo - 1 << " other values\n");
     997             :   for (unsigned i = 0, e = NumTo; i != e; ++i)
     998             :     assert((!To[i].getNode() ||
     999             :             N->getValueType(i) == To[i].getValueType()) &&
    1000             :            "Cannot combine value to value of different type!");
    1001             : 
    1002             :   WorklistRemover DeadNodes(*this);
    1003      552220 :   DAG.ReplaceAllUsesWith(N, To);
    1004      552220 :   if (AddTo) {
    1005             :     // Push the new nodes and any users onto the worklist
    1006     1020388 :     for (unsigned i = 0, e = NumTo; i != e; ++i) {
    1007      391301 :       if (To[i].getNode()) {
    1008      391301 :         AddToWorklist(To[i].getNode());
    1009      391301 :         AddUsersToWorklist(To[i].getNode());
    1010             :       }
    1011             :     }
    1012             :   }
    1013             : 
    1014             :   // Finally, if the node is now dead, remove it from the graph.  The node
    1015             :   // may not be dead if the replacement process recursively simplified to
    1016             :   // something else needing this node.
    1017      552220 :   if (N->use_empty())
    1018      552178 :     deleteAndRecombine(N);
    1019     1104440 :   return SDValue(N, 0);
    1020             : }
    1021             : 
    1022       91874 : void DAGCombiner::
    1023             : CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
    1024             :   // Replace all uses.  If any nodes become isomorphic to other nodes and
    1025             :   // are deleted, make sure to remove them from our worklist.
    1026             :   WorklistRemover DeadNodes(*this);
    1027       91874 :   DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
    1028             : 
    1029             :   // Push the new node and any (possibly new) users onto the worklist.
    1030       91874 :   AddToWorklist(TLO.New.getNode());
    1031       91874 :   AddUsersToWorklist(TLO.New.getNode());
    1032             : 
    1033             :   // Finally, if the node is now dead, remove it from the graph.  The node
    1034             :   // may not be dead if the replacement process recursively simplified to
    1035             :   // something else needing this node.
    1036       91874 :   if (TLO.Old.getNode()->use_empty())
    1037       91387 :     deleteAndRecombine(TLO.Old.getNode());
    1038       91874 : }
    1039             : 
    1040             : /// Check the specified integer node value to see if it can be simplified or if
    1041             : /// things it uses can be simplified by bit propagation. If so, return true.
    1042     2629751 : bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
    1043     2629751 :   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
    1044     2629751 :   KnownBits Known;
    1045     2629751 :   if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
    1046             :     return false;
    1047             : 
    1048             :   // Revisit the node.
    1049       88565 :   AddToWorklist(Op.getNode());
    1050             : 
    1051             :   // Replace the old value with the new one.
    1052             :   ++NodesCombined;
    1053             :   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
    1054             :              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
    1055             :              dbgs() << '\n');
    1056             : 
    1057       88565 :   CommitTargetLoweringOpt(TLO);
    1058       88565 :   return true;
    1059             : }
    1060             : 
    1061             : /// Check the specified vector node value to see if it can be simplified or
    1062             : /// if things it uses can be simplified as it only uses some of the elements.
    1063             : /// If so, return true.
    1064       65481 : bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
    1065             :                                              const APInt &Demanded) {
    1066       65481 :   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
    1067             :   APInt KnownUndef, KnownZero;
    1068       65481 :   if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO))
    1069             :     return false;
    1070             : 
    1071             :   // Revisit the node.
    1072        1712 :   AddToWorklist(Op.getNode());
    1073             : 
    1074             :   // Replace the old value with the new one.
    1075             :   ++NodesCombined;
    1076             :   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
    1077             :              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
    1078             :              dbgs() << '\n');
    1079             : 
    1080        1712 :   CommitTargetLoweringOpt(TLO);
    1081        1712 :   return true;
    1082             : }
    1083             : 
    1084         186 : void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
    1085             :   SDLoc DL(Load);
    1086         372 :   EVT VT = Load->getValueType(0);
    1087         372 :   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
    1088             : 
    1089             :   LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
    1090             :              Trunc.getNode()->dump(&DAG); dbgs() << '\n');
    1091             :   WorklistRemover DeadNodes(*this);
    1092         372 :   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
    1093         372 :   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
    1094         186 :   deleteAndRecombine(Load);
    1095         186 :   AddToWorklist(Trunc.getNode());
    1096         186 : }
    1097             : 
    1098        8170 : SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
    1099        8170 :   Replace = false;
    1100             :   SDLoc DL(Op);
    1101             :   if (ISD::isUNINDEXEDLoad(Op.getNode())) {
    1102             :     LoadSDNode *LD = cast<LoadSDNode>(Op);
    1103         239 :     EVT MemVT = LD->getMemoryVT();
    1104             :     ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
    1105             :                                                       : LD->getExtensionType();
    1106         239 :     Replace = true;
    1107         239 :     return DAG.getExtLoad(ExtType, DL, PVT,
    1108             :                           LD->getChain(), LD->getBasePtr(),
    1109         478 :                           MemVT, LD->getMemOperand());
    1110             :   }
    1111             : 
    1112             :   unsigned Opc = Op.getOpcode();
    1113        7931 :   switch (Opc) {
    1114             :   default: break;
    1115           1 :   case ISD::AssertSext:
    1116           1 :     if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
    1117           2 :       return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
    1118           0 :     break;
    1119          33 :   case ISD::AssertZext:
    1120          33 :     if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
    1121          66 :       return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
    1122           0 :     break;
    1123        1520 :   case ISD::Constant: {
    1124             :     unsigned ExtOpc =
    1125        3040 :       Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    1126        3040 :     return DAG.getNode(ExtOpc, DL, PVT, Op);
    1127             :   }
    1128             :   }
    1129             : 
    1130        6377 :   if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
    1131           0 :     return SDValue();
    1132       12754 :   return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
    1133             : }
    1134             : 
    1135           1 : SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
    1136           1 :   if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
    1137           0 :     return SDValue();
    1138           1 :   EVT OldVT = Op.getValueType();
    1139             :   SDLoc DL(Op);
    1140           1 :   bool Replace = false;
    1141           1 :   SDValue NewOp = PromoteOperand(Op, PVT, Replace);
    1142           1 :   if (!NewOp.getNode())
    1143           0 :     return SDValue();
    1144           1 :   AddToWorklist(NewOp.getNode());
    1145             : 
    1146           1 :   if (Replace)
    1147           0 :     ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
    1148           1 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
    1149           2 :                      DAG.getValueType(OldVT));
    1150             : }
    1151             : 
    1152         617 : SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
    1153         617 :   EVT OldVT = Op.getValueType();
    1154             :   SDLoc DL(Op);
    1155         617 :   bool Replace = false;
    1156         617 :   SDValue NewOp = PromoteOperand(Op, PVT, Replace);
    1157         617 :   if (!NewOp.getNode())
    1158           0 :     return SDValue();
    1159         617 :   AddToWorklist(NewOp.getNode());
    1160             : 
    1161         617 :   if (Replace)
    1162           6 :     ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
    1163         617 :   return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
    1164             : }
    1165             : 
    1166             : /// Promote the specified integer binary operation if the target indicates it is
    1167             : /// beneficial. e.g. On x86, it's usually better to promote i16 operations to
    1168             : /// i32 since i16 instructions are longer.
    1169     2373945 : SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
    1170     2373945 :   if (!LegalOperations)
    1171     1352762 :     return SDValue();
    1172             : 
    1173     1021183 :   EVT VT = Op.getValueType();
    1174     1021183 :   if (VT.isVector() || !VT.isInteger())
    1175      157973 :     return SDValue();
    1176             : 
    1177             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1178             :   // promoting it.
    1179             :   unsigned Opc = Op.getOpcode();
    1180      863210 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1181      859245 :     return SDValue();
    1182             : 
    1183        3965 :   EVT PVT = VT;
    1184             :   // Consult target whether it is a good idea to promote this operation and
    1185             :   // what's the right type to promote it to.
    1186        3965 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1187             :     assert(PVT != VT && "Don't know what type to promote to!");
    1188             : 
    1189             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1190             : 
    1191        3310 :     bool Replace0 = false;
    1192        3310 :     SDValue N0 = Op.getOperand(0);
    1193        3310 :     SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
    1194             : 
    1195        3310 :     bool Replace1 = false;
    1196        3310 :     SDValue N1 = Op.getOperand(1);
    1197        3310 :     SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
    1198             :     SDLoc DL(Op);
    1199             : 
    1200             :     SDValue RV =
    1201        6620 :         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
    1202             : 
    1203             :     // We are always replacing N0/N1's use in N and only need
    1204             :     // additional replacements if there are additional uses.
    1205        3310 :     Replace0 &= !N0->hasOneUse();
    1206        3310 :     Replace1 &= (N0 != N1) && !N1->hasOneUse();
    1207             : 
    1208             :     // Combine Op here so it is preserved past replacements.
    1209             :     CombineTo(Op.getNode(), RV);
    1210             : 
    1211             :     // If operands have a use ordering, make sure we deal with
    1212             :     // predecessor first.
    1213        3312 :     if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
    1214             :       std::swap(N0, N1);
    1215             :       std::swap(NN0, NN1);
    1216             :     }
    1217             : 
    1218        3310 :     if (Replace0) {
    1219          69 :       AddToWorklist(NN0.getNode());
    1220          69 :       ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
    1221             :     }
    1222        3310 :     if (Replace1) {
    1223          92 :       AddToWorklist(NN1.getNode());
    1224          92 :       ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
    1225             :     }
    1226        3310 :     return Op;
    1227             :   }
    1228         655 :   return SDValue();
    1229             : }
    1230             : 
    1231             : /// Promote the specified integer shift operation if the target indicates it is
    1232             : /// beneficial. e.g. On x86, it's usually better to promote i16 operations to
    1233             : /// i32 since i16 instructions are longer.
    1234      277300 : SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
    1235      277300 :   if (!LegalOperations)
    1236      130209 :     return SDValue();
    1237             : 
    1238      147091 :   EVT VT = Op.getValueType();
    1239      147091 :   if (VT.isVector() || !VT.isInteger())
    1240        1954 :     return SDValue();
    1241             : 
    1242             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1243             :   // promoting it.
    1244             :   unsigned Opc = Op.getOpcode();
    1245      145137 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1246      139916 :     return SDValue();
    1247             : 
    1248        5221 :   EVT PVT = VT;
    1249             :   // Consult target whether it is a good idea to promote this operation and
    1250             :   // what's the right type to promote it to.
    1251        5221 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1252             :     assert(PVT != VT && "Don't know what type to promote to!");
    1253             : 
    1254             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1255             : 
    1256        1516 :     bool Replace = false;
    1257        1516 :     SDValue N0 = Op.getOperand(0);
    1258        1516 :     SDValue N1 = Op.getOperand(1);
    1259        1516 :     if (Opc == ISD::SRA)
    1260           0 :       N0 = SExtPromoteOperand(N0, PVT);
    1261        1516 :     else if (Opc == ISD::SRL)
    1262         584 :       N0 = ZExtPromoteOperand(N0, PVT);
    1263             :     else
    1264         932 :       N0 = PromoteOperand(N0, PVT, Replace);
    1265             : 
    1266        1516 :     if (!N0.getNode())
    1267           0 :       return SDValue();
    1268             : 
    1269             :     SDLoc DL(Op);
    1270             :     SDValue RV =
    1271        3032 :         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
    1272             : 
    1273        1516 :     AddToWorklist(N0.getNode());
    1274        1516 :     if (Replace)
    1275          19 :       ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
    1276             : 
    1277             :     // Deal with Op being deleted.
    1278        1516 :     if (Op && Op.getOpcode() != ISD::DELETED_NODE)
    1279        1516 :       return RV;
    1280             :   }
    1281        3705 :   return SDValue();
    1282             : }
    1283             : 
    1284      193711 : SDValue DAGCombiner::PromoteExtend(SDValue Op) {
    1285      193711 :   if (!LegalOperations)
    1286      125386 :     return SDValue();
    1287             : 
    1288       68325 :   EVT VT = Op.getValueType();
    1289       68325 :   if (VT.isVector() || !VT.isInteger())
    1290        1478 :     return SDValue();
    1291             : 
    1292             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1293             :   // promoting it.
    1294             :   unsigned Opc = Op.getOpcode();
    1295       66847 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1296       66442 :     return SDValue();
    1297             : 
    1298         405 :   EVT PVT = VT;
    1299             :   // Consult target whether it is a good idea to promote this operation and
    1300             :   // what's the right type to promote it to.
    1301         405 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1302             :     assert(PVT != VT && "Don't know what type to promote to!");
    1303             :     // fold (aext (aext x)) -> (aext x)
    1304             :     // fold (aext (zext x)) -> (zext x)
    1305             :     // fold (aext (sext x)) -> (sext x)
    1306             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1307        1173 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
    1308             :   }
    1309          14 :   return SDValue();
    1310             : }
    1311             : 
    1312     1739569 : bool DAGCombiner::PromoteLoad(SDValue Op) {
    1313     1739569 :   if (!LegalOperations)
    1314             :     return false;
    1315             : 
    1316             :   if (!ISD::isUNINDEXEDLoad(Op.getNode()))
    1317             :     return false;
    1318             : 
    1319      630171 :   EVT VT = Op.getValueType();
    1320      630171 :   if (VT.isVector() || !VT.isInteger())
    1321             :     return false;
    1322             : 
    1323             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1324             :   // promoting it.
    1325             :   unsigned Opc = Op.getOpcode();
    1326      420521 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1327             :     return false;
    1328             : 
    1329        1840 :   EVT PVT = VT;
    1330             :   // Consult target whether it is a good idea to promote this operation and
    1331             :   // what's the right type to promote it to.
    1332        1840 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1333             :     assert(PVT != VT && "Don't know what type to promote to!");
    1334             : 
    1335             :     SDLoc DL(Op);
    1336             :     SDNode *N = Op.getNode();
    1337             :     LoadSDNode *LD = cast<LoadSDNode>(N);
    1338           0 :     EVT MemVT = LD->getMemoryVT();
    1339             :     ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
    1340             :                                                       : LD->getExtensionType();
    1341           0 :     SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
    1342             :                                    LD->getChain(), LD->getBasePtr(),
    1343           0 :                                    MemVT, LD->getMemOperand());
    1344           0 :     SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
    1345             : 
    1346             :     LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
    1347             :                Result.getNode()->dump(&DAG); dbgs() << '\n');
    1348             :     WorklistRemover DeadNodes(*this);
    1349           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
    1350           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
    1351           0 :     deleteAndRecombine(N);
    1352           0 :     AddToWorklist(Result.getNode());
    1353             :     return true;
    1354             :   }
    1355             :   return false;
    1356             : }
    1357             : 
    1358             : /// Recursively delete a node which has no uses and any operands for
    1359             : /// which it is the only use.
    1360             : ///
    1361             : /// Note that this both deletes the nodes and removes them from the worklist.
    1362             : /// It also adds any nodes who have had a user deleted to the worklist as they
    1363             : /// may now have only one use and subject to other combines.
    1364    32046786 : bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
    1365    32046786 :   if (!N->use_empty())
    1366             :     return false;
    1367             : 
    1368             :   SmallSetVector<SDNode *, 16> Nodes;
    1369     2164793 :   Nodes.insert(N);
    1370             :   do {
    1371     9121149 :     N = Nodes.pop_back_val();
    1372     9121149 :     if (!N)
    1373           0 :       continue;
    1374             : 
    1375     9121149 :     if (N->use_empty()) {
    1376    11577394 :       for (const SDValue &ChildN : N->op_values())
    1377     7523990 :         Nodes.insert(ChildN.getNode());
    1378             : 
    1379     4053404 :       removeFromWorklist(N);
    1380     4053404 :       DAG.DeleteNode(N);
    1381             :     } else {
    1382     5067745 :       AddToWorklist(N);
    1383             :     }
    1384     9121149 :   } while (!Nodes.empty());
    1385             :   return true;
    1386             : }
    1387             : 
    1388             : //===----------------------------------------------------------------------===//
    1389             : //  Main DAG Combiner implementation
    1390             : //===----------------------------------------------------------------------===//
    1391             : 
    1392      862732 : void DAGCombiner::Run(CombineLevel AtLevel) {
    1393             :   // set the instance variables, so that the various visit routines may use it.
    1394      862732 :   Level = AtLevel;
    1395      862732 :   LegalOperations = Level >= AfterLegalizeVectorOps;
    1396      862732 :   LegalTypes = Level >= AfterLegalizeTypes;
    1397             : 
    1398             :   // Add all the dag nodes to the worklist.
    1399    26553256 :   for (SDNode &Node : DAG.allnodes())
    1400    24827792 :     AddToWorklist(&Node);
    1401             : 
    1402             :   // Create a dummy node (which is not added to allnodes), that adds a reference
    1403             :   // to the root node, preventing it from being deleted, and tracking any
    1404             :   // changes of the root.
    1405     2588196 :   HandleSDNode Dummy(DAG.getRoot());
    1406             : 
    1407             :   // While the worklist isn't empty, find a node and try to combine it.
    1408    31389833 :   while (!WorklistMap.empty()) {
    1409             :     SDNode *N;
    1410             :     // The Worklist holds the SDNodes in order, but it may contain null entries.
    1411             :     do {
    1412    32419380 :       N = Worklist.pop_back_val();
    1413    32419380 :     } while (!N);
    1414             : 
    1415    30527101 :     bool GoodWorklistEntry = WorklistMap.erase(N);
    1416             :     (void)GoodWorklistEntry;
    1417             :     assert(GoodWorklistEntry &&
    1418             :            "Found a worklist entry without a corresponding map entry!");
    1419             : 
    1420             :     // If N has no uses, it is dead.  Make sure to revisit all N's operands once
    1421             :     // N is deleted from the DAG, since they too may now be dead or may have a
    1422             :     // reduced number of uses, allowing other xforms.
    1423    30527101 :     if (recursivelyDeleteUnusedNodes(N))
    1424    29653227 :       continue;
    1425             : 
    1426             :     WorklistRemover DeadNodes(*this);
    1427             : 
    1428             :     // If this combine is running after legalizing the DAG, re-legalize any
    1429             :     // nodes pulled off the worklist.
    1430    29881734 :     if (Level == AfterLegalizeDAG) {
    1431             :       SmallSetVector<SDNode *, 16> UpdatedNodes;
    1432    11673900 :       bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
    1433             : 
    1434    11721506 :       for (SDNode *LN : UpdatedNodes) {
    1435       23803 :         AddToWorklist(LN);
    1436             :         AddUsersToWorklist(LN);
    1437             :       }
    1438    11673900 :       if (!NIsValid)
    1439             :         continue;
    1440             :     }
    1441             : 
    1442             :     LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
    1443             : 
    1444             :     // Add any operands of the new node which have not yet been combined to the
    1445             :     // worklist as well. Because the worklist uniques things already, this
    1446             :     // won't repeatedly process the same operand.
    1447    29871437 :     CombinedNodes.insert(N);
    1448   116203924 :     for (const SDValue &ChildN : N->op_values())
    1449    56461050 :       if (!CombinedNodes.count(ChildN.getNode()))
    1450    48305845 :         AddToWorklist(ChildN.getNode());
    1451             : 
    1452    29871437 :     SDValue RV = combine(N);
    1453             : 
    1454    29871437 :     if (!RV.getNode())
    1455    27690582 :       continue;
    1456             : 
    1457             :     ++NodesCombined;
    1458             : 
    1459             :     // If we get back the same node we passed in, rather than a new node or
    1460             :     // zero, we know that the node must have defined multiple values and
    1461             :     // CombineTo was used.  Since CombineTo takes care of the worklist
    1462             :     // mechanics for us, we have no work to do in this case.
    1463     2180855 :     if (RV.getNode() == N)
    1464      661614 :       continue;
    1465             : 
    1466             :     assert(N->getOpcode() != ISD::DELETED_NODE &&
    1467             :            RV.getOpcode() != ISD::DELETED_NODE &&
    1468             :            "Node was deleted but visit returned new node!");
    1469             : 
    1470             :     LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG));
    1471             : 
    1472     1519241 :     if (N->getNumValues() == RV.getNode()->getNumValues())
    1473     1361442 :       DAG.ReplaceAllUsesWith(N, RV.getNode());
    1474             :     else {
    1475             :       assert(N->getValueType(0) == RV.getValueType() &&
    1476             :              N->getNumValues() == 1 && "Type mismatch");
    1477      157799 :       DAG.ReplaceAllUsesWith(N, &RV);
    1478             :     }
    1479             : 
    1480             :     // Push the new node and any users onto the worklist
    1481     1519241 :     AddToWorklist(RV.getNode());
    1482     1519241 :     AddUsersToWorklist(RV.getNode());
    1483             : 
    1484             :     // Finally, if the node is now dead, remove it from the graph.  The node
    1485             :     // may not be dead if the replacement process recursively simplified to
    1486             :     // something else needing this node. This will also take care of adding any
    1487             :     // operands which have lost a user to the worklist.
    1488     1519241 :     recursivelyDeleteUnusedNodes(N);
    1489             :   }
    1490             : 
    1491             :   // If the root changed (e.g. it was a dead load, update the root).
    1492      862732 :   DAG.setRoot(Dummy.getValue());
    1493      862732 :   DAG.RemoveDeadNodes();
    1494      862732 : }
    1495             : 
    1496    29872494 : SDValue DAGCombiner::visit(SDNode *N) {
    1497    59744988 :   switch (N->getOpcode()) {
    1498             :   default: break;
    1499     1810479 :   case ISD::TokenFactor:        return visitTokenFactor(N);
    1500       68773 :   case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
    1501     1960367 :   case ISD::ADD:                return visitADD(N);
    1502      419552 :   case ISD::SUB:                return visitSUB(N);
    1503         412 :   case ISD::ADDC:               return visitADDC(N);
    1504       83236 :   case ISD::UADDO:              return visitUADDO(N);
    1505          33 :   case ISD::SUBC:               return visitSUBC(N);
    1506        1832 :   case ISD::USUBO:              return visitUSUBO(N);
    1507        2395 :   case ISD::ADDE:               return visitADDE(N);
    1508      121058 :   case ISD::ADDCARRY:           return visitADDCARRY(N);
    1509          43 :   case ISD::SUBE:               return visitSUBE(N);
    1510         812 :   case ISD::SUBCARRY:           return visitSUBCARRY(N);
    1511       52707 :   case ISD::MUL:                return visitMUL(N);
    1512        3393 :   case ISD::SDIV:               return visitSDIV(N);
    1513        2224 :   case ISD::UDIV:               return visitUDIV(N);
    1514        2966 :   case ISD::SREM:
    1515        2966 :   case ISD::UREM:               return visitREM(N);
    1516        4645 :   case ISD::MULHU:              return visitMULHU(N);
    1517        1153 :   case ISD::MULHS:              return visitMULHS(N);
    1518         638 :   case ISD::SMUL_LOHI:          return visitSMUL_LOHI(N);
    1519        5156 :   case ISD::UMUL_LOHI:          return visitUMUL_LOHI(N);
    1520          96 :   case ISD::SMULO:              return visitSMULO(N);
    1521         141 :   case ISD::UMULO:              return visitUMULO(N);
    1522       24194 :   case ISD::SMIN:
    1523             :   case ISD::SMAX:
    1524             :   case ISD::UMIN:
    1525       24194 :   case ISD::UMAX:               return visitIMINMAX(N);
    1526      233261 :   case ISD::AND:                return visitAND(N);
    1527      123532 :   case ISD::OR:                 return visitOR(N);
    1528       75412 :   case ISD::XOR:                return visitXOR(N);
    1529      191913 :   case ISD::SHL:                return visitSHL(N);
    1530       25700 :   case ISD::SRA:                return visitSRA(N);
    1531      136433 :   case ISD::SRL:                return visitSRL(N);
    1532        2106 :   case ISD::ROTR:
    1533        2106 :   case ISD::ROTL:               return visitRotate(N);
    1534        1146 :   case ISD::ABS:                return visitABS(N);
    1535        1408 :   case ISD::BSWAP:              return visitBSWAP(N);
    1536         543 :   case ISD::BITREVERSE:         return visitBITREVERSE(N);
    1537        1198 :   case ISD::CTLZ:               return visitCTLZ(N);
    1538         527 :   case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
    1539         467 :   case ISD::CTTZ:               return visitCTTZ(N);
    1540         557 :   case ISD::CTTZ_ZERO_UNDEF:    return visitCTTZ_ZERO_UNDEF(N);
    1541        1957 :   case ISD::CTPOP:              return visitCTPOP(N);
    1542       50924 :   case ISD::SELECT:             return visitSELECT(N);
    1543       38535 :   case ISD::VSELECT:            return visitVSELECT(N);
    1544       15730 :   case ISD::SELECT_CC:          return visitSELECT_CC(N);
    1545      178454 :   case ISD::SETCC:              return visitSETCC(N);
    1546         268 :   case ISD::SETCCCARRY:         return visitSETCCCARRY(N);
    1547       38030 :   case ISD::SIGN_EXTEND:        return visitSIGN_EXTEND(N);
    1548      154812 :   case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
    1549       42906 :   case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
    1550       68314 :   case ISD::AssertSext:
    1551       68314 :   case ISD::AssertZext:         return visitAssertExt(N);
    1552       43574 :   case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
    1553        2770 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
    1554        5441 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
    1555      194064 :   case ISD::TRUNCATE:           return visitTRUNCATE(N);
    1556      587130 :   case ISD::BITCAST:            return visitBITCAST(N);
    1557             :   case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
    1558       36691 :   case ISD::FADD:               return visitFADD(N);
    1559        9017 :   case ISD::FSUB:               return visitFSUB(N);
    1560       23749 :   case ISD::FMUL:               return visitFMUL(N);
    1561        8338 :   case ISD::FMA:                return visitFMA(N);
    1562        4329 :   case ISD::FDIV:               return visitFDIV(N);
    1563         270 :   case ISD::FREM:               return visitFREM(N);
    1564        1954 :   case ISD::FSQRT:              return visitFSQRT(N);
    1565         909 :   case ISD::FCOPYSIGN:          return visitFCOPYSIGN(N);
    1566       10447 :   case ISD::SINT_TO_FP:         return visitSINT_TO_FP(N);
    1567        4516 :   case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
    1568        4888 :   case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
    1569        2948 :   case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
    1570        4456 :   case ISD::FP_ROUND:           return visitFP_ROUND(N);
    1571           0 :   case ISD::FP_ROUND_INREG:     return visitFP_ROUND_INREG(N);
    1572        8383 :   case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
    1573        5869 :   case ISD::FNEG:               return visitFNEG(N);
    1574        2484 :   case ISD::FABS:               return visitFABS(N);
    1575        1114 :   case ISD::FFLOOR:             return visitFFLOOR(N);
    1576        3105 :   case ISD::FMINNUM:            return visitFMINNUM(N);
    1577        2805 :   case ISD::FMAXNUM:            return visitFMAXNUM(N);
    1578         994 :   case ISD::FCEIL:              return visitFCEIL(N);
    1579        1312 :   case ISD::FTRUNC:             return visitFTRUNC(N);
    1580      161717 :   case ISD::BRCOND:             return visitBRCOND(N);
    1581        6072 :   case ISD::BR_CC:              return visitBR_CC(N);
    1582     1891169 :   case ISD::LOAD:               return visitLOAD(N);
    1583     3016699 :   case ISD::STORE:              return visitSTORE(N);
    1584       49914 :   case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
    1585      264783 :   case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
    1586      357279 :   case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
    1587       30619 :   case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
    1588       67505 :   case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
    1589       68151 :   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
    1590       15559 :   case ISD::SCALAR_TO_VECTOR:   return visitSCALAR_TO_VECTOR(N);
    1591       18829 :   case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
    1592         948 :   case ISD::MGATHER:            return visitMGATHER(N);
    1593        1345 :   case ISD::MLOAD:              return visitMLOAD(N);
    1594         197 :   case ISD::MSCATTER:           return visitMSCATTER(N);
    1595         758 :   case ISD::MSTORE:             return visitMSTORE(N);
    1596             :   case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
    1597        6139 :   case ISD::FP16_TO_FP:         return visitFP16_TO_FP(N);
    1598             :   }
    1599    16969310 :   return SDValue();
    1600             : }
    1601             : 
    1602    29872494 : SDValue DAGCombiner::combine(SDNode *N) {
    1603    29872494 :   SDValue RV = visit(N);
    1604             : 
    1605             :   // If nothing happened, try a target-specific DAG combine.
    1606    29872494 :   if (!RV.getNode()) {
    1607             :     assert(N->getOpcode() != ISD::DELETED_NODE &&
    1608             :            "Node was deleted but visit returned NULL!");
    1609             : 
    1610    81552310 :     if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
    1611    25711902 :         TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
    1612             : 
    1613             :       // Expose the DAG combiner to the target combiner impls.
    1614             :       TargetLowering::DAGCombinerInfo
    1615    10636513 :         DagCombineInfo(DAG, Level, false, this);
    1616             : 
    1617    10636513 :       RV = TLI.PerformDAGCombine(N, DagCombineInfo);
    1618             :     }
    1619             :   }
    1620             : 
    1621             :   // If nothing happened still, try promoting the operation.
    1622    29872494 :   if (!RV.getNode()) {
    1623    55392978 :     switch (N->getOpcode()) {
    1624             :     default: break;
    1625             :     case ISD::ADD:
    1626             :     case ISD::SUB:
    1627             :     case ISD::MUL:
    1628             :     case ISD::AND:
    1629             :     case ISD::OR:
    1630             :     case ISD::XOR:
    1631     2373945 :       RV = PromoteIntBinOp(SDValue(N, 0));
    1632     2373945 :       break;
    1633             :     case ISD::SHL:
    1634             :     case ISD::SRA:
    1635             :     case ISD::SRL:
    1636      277300 :       RV = PromoteIntShiftOp(SDValue(N, 0));
    1637      277300 :       break;
    1638             :     case ISD::SIGN_EXTEND:
    1639             :     case ISD::ZERO_EXTEND:
    1640             :     case ISD::ANY_EXTEND:
    1641      193711 :       RV = PromoteExtend(SDValue(N, 0));
    1642      193711 :       break;
    1643             :     case ISD::LOAD:
    1644     1739569 :       if (PromoteLoad(SDValue(N, 0)))
    1645             :         RV = SDValue(N, 0);
    1646             :       break;
    1647             :     }
    1648             :   }
    1649             : 
    1650             :   // If N is a commutative binary node, try eliminate it if the commuted
    1651             :   // version is already present in the DAG.
    1652    59699951 :   if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
    1653     2136185 :       N->getNumValues() == 1) {
    1654     2045243 :     SDValue N0 = N->getOperand(0);
    1655     2045243 :     SDValue N1 = N->getOperand(1);
    1656             : 
    1657             :     // Constant operands are canonicalized to RHS.
    1658             :     if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
    1659      678805 :       SDValue Ops[] = {N1, N0};
    1660     2036415 :       SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
    1661      678805 :                                             N->getFlags());
    1662      678805 :       if (CSENode)
    1663          26 :         return SDValue(CSENode, 0);
    1664             :     }
    1665             :   }
    1666             : 
    1667    29872468 :   return RV;
    1668             : }
    1669             : 
    1670             : /// Given a node, return its input chain if it has one, otherwise return a null
    1671             : /// sd operand.
    1672     2287238 : static SDValue getInputChainForNode(SDNode *N) {
    1673     4574476 :   if (unsigned NumOps = N->getNumOperands()) {
    1674     2285944 :     if (N->getOperand(0).getValueType() == MVT::Other)
    1675     2257811 :       return N->getOperand(0);
    1676       56266 :     if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
    1677       56266 :       return N->getOperand(NumOps-1);
    1678           0 :     for (unsigned i = 1; i < NumOps-1; ++i)
    1679           0 :       if (N->getOperand(i).getValueType() == MVT::Other)
    1680           0 :         return N->getOperand(i);
    1681             :   }
    1682        1294 :   return SDValue();
    1683             : }
    1684             : 
    1685     1810479 : SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
    1686             :   // If N has two operands, where one has an input chain equal to the other,
    1687             :   // the 'other' chain is redundant.
    1688     1810479 :   if (N->getNumOperands() == 2) {
    1689     2310476 :     if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
    1690       23238 :       return N->getOperand(0);
    1691     2188245 :     if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
    1692       75755 :       return N->getOperand(1);
    1693             :   }
    1694             : 
    1695             :   SmallVector<SDNode *, 8> TFs;     // List of token factors to visit.
    1696             :   SmallVector<SDValue, 8> Ops;      // Ops for replacing token factor.
    1697             :   SmallPtrSet<SDNode*, 16> SeenOps;
    1698     1711486 :   bool Changed = false;             // If we should replace this token factor.
    1699             : 
    1700             :   // Start out with this token factor.
    1701     1711486 :   TFs.push_back(N);
    1702             : 
    1703             :   // Iterate through token factors.  The TFs grows when new token factors are
    1704             :   // encountered.
    1705    10249679 :   for (unsigned i = 0; i < TFs.size(); ++i) {
    1706     2275569 :     SDNode *TF = TFs[i];
    1707             : 
    1708             :     // Check each of the operands.
    1709     9438270 :     for (const SDValue &Op : TF->op_values()) {
    1710    14325402 :       switch (Op.getOpcode()) {
    1711        5867 :       case ISD::EntryToken:
    1712             :         // Entry tokens don't need to be added to the list. They are
    1713             :         // redundant.
    1714        5867 :         Changed = true;
    1715        5867 :         break;
    1716             : 
    1717     1154795 :       case ISD::TokenFactor:
    1718     2873673 :         if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
    1719             :           // Queue up for processing.
    1720      564083 :           TFs.push_back(Op.getNode());
    1721             :           // Clean up in case the token factor is removed.
    1722      564083 :           AddToWorklist(Op.getNode());
    1723      564083 :           Changed = true;
    1724      564083 :           break;
    1725             :         }
    1726             :         LLVM_FALLTHROUGH;
    1727             : 
    1728             :       default:
    1729             :         // Only add if it isn't already in the list.
    1730     6592751 :         if (SeenOps.insert(Op.getNode()).second)
    1731     6491407 :           Ops.push_back(Op);
    1732             :         else
    1733      101344 :           Changed = true;
    1734             :         break;
    1735             :       }
    1736             :     }
    1737             :   }
    1738             : 
    1739             :   // Remove Nodes that are chained to another node in the list. Do so
    1740             :   // by walking up chains breath-first stopping when we've seen
    1741             :   // another operand. In general we must climb to the EntryNode, but we can exit
    1742             :   // early if we find all remaining work is associated with just one operand as
    1743             :   // no further pruning is possible.
    1744             : 
    1745             :   // List of nodes to search through and original Ops from which they originate.
    1746             :   SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
    1747             :   SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
    1748             :   SmallPtrSet<SDNode *, 16> SeenChains;
    1749     1711486 :   bool DidPruneOps = false;
    1750             : 
    1751     1711486 :   unsigned NumLeftToConsider = 0;
    1752    14694300 :   for (const SDValue &Op : Ops) {
    1753     6491407 :     Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
    1754     6491407 :     OpWorkCount.push_back(1);
    1755             :   }
    1756             : 
    1757    14082439 :   auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
    1758             :     // If this is an Op, we can remove the op from the list. Remark any
    1759             :     // search associated with it as from the current OpNumber.
    1760    14082439 :     if (SeenOps.count(Op) != 0) {
    1761      725964 :       Changed = true;
    1762      725964 :       DidPruneOps = true;
    1763             :       unsigned OrigOpNumber = 0;
    1764   103528112 :       while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
    1765    25337555 :         OrigOpNumber++;
    1766             :       assert((OrigOpNumber != Ops.size()) &&
    1767             :              "expected to find TokenFactor Operand");
    1768             :       // Re-mark worklist from OrigOpNumber to OpNumber
    1769    51112476 :       for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
    1770    20873320 :         if (Worklist[i].second == OrigOpNumber) {
    1771      433719 :           Worklist[i].second = OpNumber;
    1772             :         }
    1773             :       }
    1774    10091800 :       OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
    1775      725964 :       OpWorkCount[OrigOpNumber] = 0;
    1776      725964 :       NumLeftToConsider--;
    1777             :     }
    1778             :     // Add if it's a new chain
    1779    14082439 :     if (SeenChains.insert(Op).second) {
    1780    15827816 :       OpWorkCount[OpNumber]++;
    1781     7913908 :       Worklist.push_back(std::make_pair(Op, OpNumber));
    1782             :     }
    1783    15793925 :   };
    1784             : 
    1785    38635139 :   for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
    1786             :     // We need at least be consider at least 2 Ops to prune.
    1787    13131611 :     if (NumLeftToConsider <= 1)
    1788             :       break;
    1789    11737389 :     auto CurNode = Worklist[i].first;
    1790    11737389 :     auto CurOpNumber = Worklist[i].second;
    1791             :     assert((OpWorkCount[CurOpNumber] > 0) &&
    1792             :            "Node should not appear in worklist");
    1793    23474778 :     switch (CurNode->getOpcode()) {
    1794      322932 :     case ISD::EntryToken:
    1795             :       // Hitting EntryToken is the only way for the search to terminate without
    1796             :       // hitting
    1797             :       // another operand's search. Prevent us from marking this operand
    1798             :       // considered.
    1799      322932 :       NumLeftToConsider++;
    1800      322932 :       break;
    1801             :     case ISD::TokenFactor:
    1802     6906137 :       for (const SDValue &Op : CurNode->op_values())
    1803     5144303 :         AddToWorklist(i, Op.getNode(), CurOpNumber);
    1804             :       break;
    1805      547890 :     case ISD::CopyFromReg:
    1806             :     case ISD::CopyToReg:
    1807      547890 :       AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
    1808      547890 :       break;
    1809             :     default:
    1810             :       if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
    1811     8390246 :         AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
    1812             :       break;
    1813             :     }
    1814    23474778 :     OpWorkCount[CurOpNumber]--;
    1815    11737389 :     if (OpWorkCount[CurOpNumber] == 0)
    1816     4537973 :       NumLeftToConsider--;
    1817             :   }
    1818             : 
    1819             :   // If we've changed things around then replace token factor.
    1820     1711486 :   if (Changed) {
    1821             :     SDValue Result;
    1822      462826 :     if (Ops.empty()) {
    1823             :       // The entry token is the only possible outcome.
    1824         301 :       Result = DAG.getEntryNode();
    1825             :     } else {
    1826      462525 :       if (DidPruneOps) {
    1827             :         SmallVector<SDValue, 8> PrunedOps;
    1828             :         //
    1829     3370465 :         for (const SDValue &Op : Ops) {
    1830     1541016 :           if (SeenChains.count(Op.getNode()) == 0)
    1831      956515 :             PrunedOps.push_back(Op);
    1832             :         }
    1833     1153732 :         Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
    1834             :       } else {
    1835      696368 :         Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
    1836             :       }
    1837             :     }
    1838      462826 :     return Result;
    1839             :   }
    1840     1248660 :   return SDValue();
    1841             : }
    1842             : 
    1843             : /// MERGE_VALUES can always be eliminated.
    1844       68773 : SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
    1845             :   WorklistRemover DeadNodes(*this);
    1846             :   // Replacing results may cause a different MERGE_VALUES to suddenly
    1847             :   // be CSE'd with N, and carry its uses with it. Iterate until no
    1848             :   // uses remain, to ensure that the node can be safely deleted.
    1849             :   // First add the users of this node to the work list so that they
    1850             :   // can be tried again once they have new operands.
    1851             :   AddUsersToWorklist(N);
    1852             :   do {
    1853      422712 :     for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
    1854      285166 :       DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
    1855       68773 :   } while (!N->use_empty());
    1856       68773 :   deleteAndRecombine(N);
    1857      137546 :   return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    1858             : }
    1859             : 
    1860             : /// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
    1861             : /// ConstantSDNode pointer else nullptr.
    1862             : static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
    1863             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
    1864       97822 :   return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
    1865             : }
    1866             : 
    1867     3304805 : SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
    1868     3304805 :   auto BinOpcode = BO->getOpcode();
    1869             :   assert((BinOpcode == ISD::ADD || BinOpcode == ISD::SUB ||
    1870             :           BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV ||
    1871             :           BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM ||
    1872             :           BinOpcode == ISD::UREM || BinOpcode == ISD::AND ||
    1873             :           BinOpcode == ISD::OR || BinOpcode == ISD::XOR ||
    1874             :           BinOpcode == ISD::SHL || BinOpcode == ISD::SRL ||
    1875             :           BinOpcode == ISD::SRA || BinOpcode == ISD::FADD ||
    1876             :           BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL ||
    1877             :           BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) &&
    1878             :          "Unexpected binary operator");
    1879             : 
    1880             :   // Bail out if any constants are opaque because we can't constant fold those.
    1881     3304805 :   SDValue C1 = BO->getOperand(1);
    1882     3304805 :   if (!isConstantOrConstantVector(C1, true) &&
    1883             :       !isConstantFPBuildVectorOrConstantFP(C1))
    1884      917812 :     return SDValue();
    1885             : 
    1886             :   // Don't do this unless the old select is going away. We want to eliminate the
    1887             :   // binary operator, not replace a binop with a select.
    1888             :   // TODO: Handle ISD::SELECT_CC.
    1889     2386993 :   SDValue Sel = BO->getOperand(0);
    1890     2413182 :   if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
    1891     2361104 :     return SDValue();
    1892             : 
    1893       25889 :   SDValue CT = Sel.getOperand(1);
    1894       25889 :   if (!isConstantOrConstantVector(CT, true) &&
    1895             :       !isConstantFPBuildVectorOrConstantFP(CT))
    1896         491 :     return SDValue();
    1897             : 
    1898       25398 :   SDValue CF = Sel.getOperand(2);
    1899       25398 :   if (!isConstantOrConstantVector(CF, true) &&
    1900             :       !isConstantFPBuildVectorOrConstantFP(CF))
    1901          50 :     return SDValue();
    1902             : 
    1903             :   // We have a select-of-constants followed by a binary operator with a
    1904             :   // constant. Eliminate the binop by pulling the constant math into the select.
    1905             :   // Example: add (select Cond, CT, CF), C1 --> select Cond, CT + C1, CF + C1
    1906       25348 :   EVT VT = Sel.getValueType();
    1907             :   SDLoc DL(Sel);
    1908       50696 :   SDValue NewCT = DAG.getNode(BinOpcode, DL, VT, CT, C1);
    1909       25348 :   if (!NewCT.isUndef() &&
    1910       25348 :       !isConstantOrConstantVector(NewCT, true) &&
    1911             :       !isConstantFPBuildVectorOrConstantFP(NewCT))
    1912           2 :     return SDValue();
    1913             : 
    1914       50692 :   SDValue NewCF = DAG.getNode(BinOpcode, DL, VT, CF, C1);
    1915       25346 :   if (!NewCF.isUndef() &&
    1916       25346 :       !isConstantOrConstantVector(NewCF, true) &&
    1917             :       !isConstantFPBuildVectorOrConstantFP(NewCF))
    1918           0 :     return SDValue();
    1919             : 
    1920       50692 :   return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
    1921             : }
    1922             : 
    1923     1960367 : SDValue DAGCombiner::visitADD(SDNode *N) {
    1924     1960367 :   SDValue N0 = N->getOperand(0);
    1925     1960367 :   SDValue N1 = N->getOperand(1);
    1926     1960367 :   EVT VT = N0.getValueType();
    1927             :   SDLoc DL(N);
    1928             : 
    1929             :   // fold vector ops
    1930     1960367 :   if (VT.isVector()) {
    1931      113557 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    1932          17 :       return FoldedVOp;
    1933             : 
    1934             :     // fold (add x, 0) -> x, vector edition
    1935      113540 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    1936          75 :       return N0;
    1937      113465 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    1938         124 :       return N1;
    1939             :   }
    1940             : 
    1941             :   // fold (add x, undef) -> undef
    1942     1960151 :   if (N0.isUndef())
    1943           3 :     return N0;
    1944             : 
    1945     1960148 :   if (N1.isUndef())
    1946          11 :     return N1;
    1947             : 
    1948     1960137 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
    1949             :     // canonicalize constant to RHS
    1950        1857 :     if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
    1951        2496 :       return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
    1952             :     // fold (add c1, c2) -> c1+c2
    1953         609 :     return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
    1954         609 :                                       N1.getNode());
    1955             :   }
    1956             : 
    1957             :   // fold (add x, 0) -> x
    1958     1958280 :   if (isNullConstant(N1))
    1959          84 :     return N0;
    1960             : 
    1961     1958196 :   if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
    1962             :     // fold ((c1-A)+c2) -> (c1+c2)-A
    1963     1486292 :     if (N0.getOpcode() == ISD::SUB &&
    1964         410 :         isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
    1965             :       // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
    1966          32 :       return DAG.getNode(ISD::SUB, DL, VT,
    1967             :                          DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
    1968          32 :                          N0.getOperand(1));
    1969             :     }
    1970             : 
    1971             :     // add (sext i1 X), 1 -> zext (not i1 X)
    1972             :     // We don't transform this pattern:
    1973             :     //   add (zext i1 X), -1 -> sext (not i1 X)
    1974             :     // because most (?) targets generate better code for the zext form.
    1975     1486291 :     if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
    1976         205 :         isOneConstantOrOneSplatConstant(N1)) {
    1977          23 :       SDValue X = N0.getOperand(0);
    1978          23 :       if ((!LegalOperations ||
    1979           1 :            (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
    1980          45 :             TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
    1981          22 :           X.getScalarValueSizeInBits() == 1) {
    1982          34 :         SDValue Not = DAG.getNOT(DL, X, X.getValueType());
    1983          34 :         return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
    1984             :       }
    1985             :     }
    1986             : 
    1987             :     // Undo the add -> or combine to merge constant offsets from a frame index.
    1988             :     if (N0.getOpcode() == ISD::OR &&
    1989             :         isa<FrameIndexSDNode>(N0.getOperand(0)) &&
    1990     1488674 :         isa<ConstantSDNode>(N0.getOperand(1)) &&
    1991        5682 :         DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
    1992        5682 :       SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
    1993        5682 :       return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
    1994             :     }
    1995             :   }
    1996             : 
    1997     1955306 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    1998           8 :     return NewSel;
    1999             : 
    2000             :   // reassociate add
    2001     1955298 :   if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1))
    2002      222042 :     return RADD;
    2003             : 
    2004             :   // fold ((0-A) + B) -> B-A
    2005     1735617 :   if (N0.getOpcode() == ISD::SUB &&
    2006        2361 :       isNullConstantOrNullSplatConstant(N0.getOperand(0)))
    2007          30 :     return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
    2008             : 
    2009             :   // fold (A + (0-B)) -> A-B
    2010     1734014 :   if (N1.getOpcode() == ISD::SUB &&
    2011         773 :       isNullConstantOrNullSplatConstant(N1.getOperand(0)))
    2012         316 :     return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
    2013             : 
    2014             :   // fold (A+(B-A)) -> B
    2015     1733083 :   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
    2016           3 :     return N1.getOperand(0);
    2017             : 
    2018             :   // fold ((B-A)+A) -> B
    2019     1733080 :   if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
    2020          27 :     return N0.getOperand(0);
    2021             : 
    2022             :   // fold (A+(B-(A+C))) to (B-C)
    2023     1734277 :   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
    2024             :       N0 == N1.getOperand(1).getOperand(0))
    2025           2 :     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
    2026           2 :                        N1.getOperand(1).getOperand(1));
    2027             : 
    2028             :   // fold (A+(B-(C+A))) to (B-C)
    2029     1734271 :   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
    2030             :       N0 == N1.getOperand(1).getOperand(1))
    2031           3 :     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
    2032           3 :                        N1.getOperand(1).getOperand(0));
    2033             : 
    2034             :   // fold (A+((B-A)+or-C)) to (B+or-C)
    2035     1736515 :   if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
    2036     1737122 :       N1.getOperand(0).getOpcode() == ISD::SUB &&
    2037             :       N0 == N1.getOperand(0).getOperand(1))
    2038           5 :     return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
    2039           5 :                        N1.getOperand(1));
    2040             : 
    2041             :   // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
    2042     1735359 :   if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
    2043          12 :     SDValue N00 = N0.getOperand(0);
    2044          12 :     SDValue N01 = N0.getOperand(1);
    2045          12 :     SDValue N10 = N1.getOperand(0);
    2046          12 :     SDValue N11 = N1.getOperand(1);
    2047             : 
    2048          12 :     if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
    2049           5 :       return DAG.getNode(ISD::SUB, DL, VT,
    2050          10 :                          DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
    2051          20 :                          DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
    2052             :   }
    2053             : 
    2054     1733038 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    2055        2914 :     return SDValue(N, 0);
    2056             : 
    2057             :   // fold (a+b) -> (a|b) iff a and b share no bits.
    2058     3444524 :   if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
    2059     1714400 :       DAG.haveNoCommonBitsSet(N0, N1))
    2060       52770 :     return DAG.getNode(ISD::OR, DL, VT, N0, N1);
    2061             : 
    2062             :   // fold (add (xor a, -1), 1) -> (sub 0, a)
    2063     1703746 :   if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1))
    2064           4 :     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
    2065           4 :                        N0.getOperand(0));
    2066             : 
    2067     1703735 :   if (SDValue Combined = visitADDLike(N0, N1, N))
    2068         813 :     return Combined;
    2069             : 
    2070     1702922 :   if (SDValue Combined = visitADDLike(N1, N0, N))
    2071          62 :     return Combined;
    2072             : 
    2073     1702860 :   return SDValue();
    2074             : }
    2075             : 
    2076     3074615 : static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
    2077             :   bool Masked = false;
    2078             : 
    2079             :   // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
    2080             :   while (true) {
    2081     3085521 :     if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
    2082        5193 :       V = V.getOperand(0);
    2083        5193 :       continue;
    2084             :     }
    2085             : 
    2086     3078166 :     if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
    2087             :       Masked = true;
    2088         520 :       V = V.getOperand(0);
    2089         520 :       continue;
    2090             :     }
    2091             : 
    2092             :     break;
    2093             :   }
    2094             : 
    2095             :   // If this is not a carry, return.
    2096     3074615 :   if (V.getResNo() != 1)
    2097     3062122 :     return SDValue();
    2098             : 
    2099       12221 :   if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
    2100       24084 :       V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
    2101       11591 :     return SDValue();
    2102             : 
    2103             :   // If the result is masked, then no matter what kind of bool it is we can
    2104             :   // return. If it isn't, then we need to make sure the bool type is either 0 or
    2105             :   // 1 and not other values.
    2106        1364 :   if (Masked ||
    2107         924 :       TLI.getBooleanContents(V.getValueType()) ==
    2108             :           TargetLoweringBase::ZeroOrOneBooleanContent)
    2109         902 :     return V;
    2110             : 
    2111           0 :   return SDValue();
    2112             : }
    2113             : 
    2114     3406657 : SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
    2115     6813314 :   EVT VT = N0.getValueType();
    2116             :   SDLoc DL(LocReference);
    2117             : 
    2118             :   // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
    2119     3700197 :   if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
    2120         290 :       isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
    2121          45 :     return DAG.getNode(ISD::SUB, DL, VT, N0,
    2122             :                        DAG.getNode(ISD::SHL, DL, VT,
    2123          45 :                                    N1.getOperand(0).getOperand(1),
    2124          45 :                                    N1.getOperand(1)));
    2125             : 
    2126     3406612 :   if (N1.getOpcode() == ISD::AND) {
    2127        2796 :     SDValue AndOp0 = N1.getOperand(0);
    2128        2796 :     unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
    2129             :     unsigned DestBits = VT.getScalarSizeInBits();
    2130             : 
    2131             :     // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
    2132             :     // and similar xforms where the inner op is either ~0 or 0.
    2133        2863 :     if (NumSignBits == DestBits &&
    2134         134 :         isOneConstantOrOneSplatConstant(N1->getOperand(1)))
    2135         120 :       return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
    2136             :   }
    2137             : 
    2138             :   // add (sext i1), X -> sub X, (zext i1)
    2139             :   if (N0.getOpcode() == ISD::SIGN_EXTEND &&
    2140     3408188 :       N0.getOperand(0).getValueType() == MVT::i1 &&
    2141         153 :       !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
    2142         220 :     SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
    2143         220 :     return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
    2144             :   }
    2145             : 
    2146             :   // add X, (sextinreg Y i1) -> sub X, (and Y 1)
    2147     3406442 :   if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2148             :     VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    2149             :     if (TN->getVT() == MVT::i1) {
    2150         625 :       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
    2151        1250 :                                  DAG.getConstant(1, DL, VT));
    2152        1250 :       return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
    2153             :     }
    2154             :   }
    2155             : 
    2156             :   // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
    2157     3405860 :   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
    2158          20 :       N1.getResNo() == 0)
    2159          19 :     return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
    2160          38 :                        N0, N1.getOperand(0), N1.getOperand(2));
    2161             : 
    2162             :   // (add X, Carry) -> (addcarry X, 0, Carry)
    2163     3405798 :   if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
    2164     2671850 :     if (SDValue Carry = getAsCarry(TLI, N1))
    2165          16 :       return DAG.getNode(ISD::ADDCARRY, DL,
    2166          16 :                          DAG.getVTList(VT, Carry.getValueType()), N0,
    2167          48 :                          DAG.getConstant(0, DL, VT), Carry);
    2168             : 
    2169     3405782 :   return SDValue();
    2170             : }
    2171             : 
    2172         412 : SDValue DAGCombiner::visitADDC(SDNode *N) {
    2173         412 :   SDValue N0 = N->getOperand(0);
    2174         412 :   SDValue N1 = N->getOperand(1);
    2175         412 :   EVT VT = N0.getValueType();
    2176             :   SDLoc DL(N);
    2177             : 
    2178             :   // If the flag result is dead, turn this into an ADD.
    2179         412 :   if (!N->hasAnyUseOfValue(1))
    2180           1 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2181           4 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2182             : 
    2183             :   // canonicalize constant to RHS.
    2184             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2185             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2186         411 :   if (N0C && !N1C)
    2187           0 :     return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
    2188             : 
    2189             :   // fold (addc x, 0) -> x + no carry out
    2190         411 :   if (isNullConstant(N1))
    2191           0 :     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
    2192           0 :                                         DL, MVT::Glue));
    2193             : 
    2194             :   // If it cannot overflow, transform into an add.
    2195         411 :   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
    2196           0 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2197           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2198             : 
    2199         411 :   return SDValue();
    2200             : }
    2201             : 
    2202           7 : static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT,
    2203             :                            SelectionDAG &DAG, const TargetLowering &TLI) {
    2204           7 :   SDValue Cst;
    2205           7 :   switch (TLI.getBooleanContents(VT)) {
    2206           7 :   case TargetLowering::ZeroOrOneBooleanContent:
    2207             :   case TargetLowering::UndefinedBooleanContent:
    2208           7 :     Cst = DAG.getConstant(1, DL, VT);
    2209           7 :     break;
    2210           0 :   case TargetLowering::ZeroOrNegativeOneBooleanContent:
    2211           0 :     Cst = DAG.getConstant(-1, DL, VT);
    2212           0 :     break;
    2213             :   }
    2214             : 
    2215           7 :   return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
    2216             : }
    2217             : 
    2218           8 : static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) {
    2219           8 :   if (V.getOpcode() != ISD::XOR) return false;
    2220             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V.getOperand(1));
    2221             :   if (!Const) return false;
    2222             : 
    2223           3 :   switch(TLI.getBooleanContents(VT)) {
    2224           3 :     case TargetLowering::ZeroOrOneBooleanContent:
    2225           3 :       return Const->isOne();
    2226           0 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    2227           0 :       return Const->isAllOnesValue();
    2228           0 :     case TargetLowering::UndefinedBooleanContent:
    2229           0 :       return (Const->getAPIntValue() & 0x01) == 1;
    2230             :   }
    2231           0 :   llvm_unreachable("Unsupported boolean content");
    2232             : }
    2233             : 
    2234       83236 : SDValue DAGCombiner::visitUADDO(SDNode *N) {
    2235       83236 :   SDValue N0 = N->getOperand(0);
    2236       83236 :   SDValue N1 = N->getOperand(1);
    2237       83236 :   EVT VT = N0.getValueType();
    2238       83236 :   if (VT.isVector())
    2239           0 :     return SDValue();
    2240             : 
    2241      166472 :   EVT CarryVT = N->getValueType(1);
    2242             :   SDLoc DL(N);
    2243             : 
    2244             :   // If the flag result is dead, turn this into an ADD.
    2245       83236 :   if (!N->hasAnyUseOfValue(1))
    2246         384 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2247        1152 :                      DAG.getUNDEF(CarryVT));
    2248             : 
    2249             :   // canonicalize constant to RHS.
    2250             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2251             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2252       82852 :   if (N0C && !N1C)
    2253          12 :     return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
    2254             : 
    2255             :   // fold (uaddo x, 0) -> x + no carry out
    2256       82846 :   if (isNullConstant(N1))
    2257         436 :     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
    2258             : 
    2259             :   // If it cannot overflow, transform into an add.
    2260       82410 :   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
    2261          79 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2262         237 :                      DAG.getConstant(0, DL, CarryVT));
    2263             : 
    2264             :   // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
    2265       82331 :   if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1)) {
    2266           4 :     SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
    2267             :                               DAG.getConstant(0, DL, VT),
    2268           8 :                               N0.getOperand(0));
    2269             :     return CombineTo(N, Sub,
    2270           4 :                      flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
    2271             :   }
    2272             : 
    2273       82327 :   if (SDValue Combined = visitUADDOLike(N0, N1, N))
    2274         225 :     return Combined;
    2275             : 
    2276       82102 :   if (SDValue Combined = visitUADDOLike(N1, N0, N))
    2277           9 :     return Combined;
    2278             : 
    2279       82093 :   return SDValue();
    2280             : }
    2281             : 
    2282      164429 : SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
    2283      328858 :   auto VT = N0.getValueType();
    2284             : 
    2285             :   // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
    2286             :   // If Y + 1 cannot overflow.
    2287      165035 :   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
    2288         345 :     SDValue Y = N1.getOperand(0);
    2289        1035 :     SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
    2290         345 :     if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
    2291         468 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
    2292         468 :                          N1.getOperand(2));
    2293             :   }
    2294             : 
    2295             :   // (uaddo X, Carry) -> (addcarry X, 0, Carry)
    2296      164195 :   if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
    2297      163863 :     if (SDValue Carry = getAsCarry(TLI, N1))
    2298           0 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
    2299           0 :                          DAG.getConstant(0, SDLoc(N), VT), Carry);
    2300             : 
    2301      164195 :   return SDValue();
    2302             : }
    2303             : 
    2304        2395 : SDValue DAGCombiner::visitADDE(SDNode *N) {
    2305        2395 :   SDValue N0 = N->getOperand(0);
    2306        2395 :   SDValue N1 = N->getOperand(1);
    2307        2395 :   SDValue CarryIn = N->getOperand(2);
    2308             : 
    2309             :   // canonicalize constant to RHS
    2310             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2311             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2312        2395 :   if (N0C && !N1C)
    2313           8 :     return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
    2314           8 :                        N1, N0, CarryIn);
    2315             : 
    2316             :   // fold (adde x, y, false) -> (addc x, y)
    2317        2391 :   if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    2318           0 :     return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
    2319             : 
    2320        2391 :   return SDValue();
    2321             : }
    2322             : 
    2323      121058 : SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
    2324      121058 :   SDValue N0 = N->getOperand(0);
    2325      121058 :   SDValue N1 = N->getOperand(1);
    2326      121058 :   SDValue CarryIn = N->getOperand(2);
    2327             :   SDLoc DL(N);
    2328             : 
    2329             :   // canonicalize constant to RHS
    2330             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2331             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2332      121058 :   if (N0C && !N1C)
    2333         716 :     return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
    2334             : 
    2335             :   // fold (addcarry x, y, false) -> (uaddo x, y)
    2336      120700 :   if (isNullConstant(CarryIn)) {
    2337         510 :     if (!LegalOperations ||
    2338          76 :         TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
    2339        1020 :       return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
    2340             :   }
    2341             : 
    2342      120190 :   EVT CarryVT = CarryIn.getValueType();
    2343             : 
    2344             :   // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
    2345      120190 :   if (isNullConstant(N0) && isNullConstant(N1)) {
    2346         504 :     EVT VT = N0.getValueType();
    2347         504 :     SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
    2348         504 :     AddToWorklist(CarryExt.getNode());
    2349         504 :     return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
    2350             :                                     DAG.getConstant(1, DL, VT)),
    2351        1512 :                      DAG.getConstant(0, DL, CarryVT));
    2352             :   }
    2353             : 
    2354             :   // fold (addcarry (xor a, -1), 0, !b) -> (subcarry 0, a, b) and flip carry.
    2355      119702 :   if (isBitwiseNot(N0) && isNullConstant(N1) &&
    2356           8 :       isBooleanFlip(CarryIn, CarryVT, TLI)) {
    2357           3 :     SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(),
    2358             :                               DAG.getConstant(0, DL, N0.getValueType()),
    2359           6 :                               N0.getOperand(0), CarryIn.getOperand(0));
    2360             :     return CombineTo(N, Sub,
    2361           3 :                      flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
    2362             :   }
    2363             : 
    2364      119683 :   if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
    2365         233 :     return Combined;
    2366             : 
    2367      119450 :   if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
    2368           0 :     return Combined;
    2369             : 
    2370      119450 :   return SDValue();
    2371             : }
    2372             : 
    2373      239133 : SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
    2374             :                                        SDNode *N) {
    2375             :   // Iff the flag result is dead:
    2376             :   // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
    2377      237696 :   if ((N0.getOpcode() == ISD::ADD ||
    2378       16446 :        (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
    2379      254346 :       isNullConstant(N1) && !N->hasAnyUseOfValue(1))
    2380         462 :     return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
    2381         462 :                        N0.getOperand(0), N0.getOperand(1), CarryIn);
    2382             : 
    2383             :   /**
    2384             :    * When one of the addcarry argument is itself a carry, we may be facing
    2385             :    * a diamond carry propagation. In which case we try to transform the DAG
    2386             :    * to ensure linear carry propagation if that is possible.
    2387             :    *
    2388             :    * We are trying to get:
    2389             :    *   (addcarry X, 0, (addcarry A, B, Z):Carry)
    2390             :    */
    2391      238902 :   if (auto Y = getAsCarry(TLI, N1)) {
    2392             :     /**
    2393             :      *            (uaddo A, B)
    2394             :      *             /       \
    2395             :      *          Carry      Sum
    2396             :      *            |          \
    2397             :      *            | (addcarry *, 0, Z)
    2398             :      *            |       /
    2399             :      *             \   Carry
    2400             :      *              |   /
    2401             :      * (addcarry X, *, *)
    2402             :      */
    2403         619 :     if (Y.getOpcode() == ISD::UADDO &&
    2404        1238 :         CarryIn.getResNo() == 1 &&
    2405          12 :         CarryIn.getOpcode() == ISD::ADDCARRY &&
    2406         898 :         isNullConstant(CarryIn.getOperand(1)) &&
    2407             :         CarryIn.getOperand(0) == Y.getValue(0)) {
    2408           4 :       auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
    2409             :                               Y.getOperand(0), Y.getOperand(1),
    2410           4 :                               CarryIn.getOperand(2));
    2411           2 :       AddToWorklist(NewY.getNode());
    2412           4 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
    2413           2 :                          DAG.getConstant(0, SDLoc(N), N0.getValueType()),
    2414           4 :                          NewY.getValue(1));
    2415             :     }
    2416             :   }
    2417             : 
    2418      238900 :   return SDValue();
    2419             : }
    2420             : 
    2421             : // Since it may not be valid to emit a fold to zero for vector initializers
    2422             : // check if we can before folding.
    2423          34 : static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
    2424             :                              SelectionDAG &DAG, bool LegalOperations,
    2425             :                              bool LegalTypes) {
    2426          34 :   if (!VT.isVector())
    2427          21 :     return DAG.getConstant(0, DL, VT);
    2428          13 :   if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
    2429          13 :     return DAG.getConstant(0, DL, VT);
    2430           0 :   return SDValue();
    2431             : }
    2432             : 
    2433      419552 : SDValue DAGCombiner::visitSUB(SDNode *N) {
    2434      419552 :   SDValue N0 = N->getOperand(0);
    2435      419552 :   SDValue N1 = N->getOperand(1);
    2436      419552 :   EVT VT = N0.getValueType();
    2437             :   SDLoc DL(N);
    2438             : 
    2439             :   // fold vector ops
    2440      419552 :   if (VT.isVector()) {
    2441      395167 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2442           4 :       return FoldedVOp;
    2443             : 
    2444             :     // fold (sub x, 0) -> x, vector edition
    2445      395163 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    2446           7 :       return N0;
    2447             :   }
    2448             : 
    2449             :   // fold (sub x, x) -> 0
    2450             :   // FIXME: Refactor this and xor and other similar operations together.
    2451             :   if (N0 == N1)
    2452          21 :     return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
    2453      426499 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    2454        6979 :       DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
    2455             :     // fold (sub c1, c2) -> c1-c2
    2456           2 :     return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
    2457           2 :                                       N1.getNode());
    2458             :   }
    2459             : 
    2460      419518 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2461           2 :     return NewSel;
    2462             : 
    2463             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    2464             : 
    2465             :   // fold (sub x, c) -> (add x, -c)
    2466             :   if (N1C) {
    2467        1591 :     return DAG.getNode(ISD::ADD, DL, VT, N0,
    2468        6364 :                        DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
    2469             :   }
    2470             : 
    2471      417925 :   if (isNullConstantOrNullSplatConstant(N0)) {
    2472             :     unsigned BitWidth = VT.getScalarSizeInBits();
    2473             :     // Right-shifting everything out but the sign bit followed by negation is
    2474             :     // the same as flipping arithmetic/logical shift type without the negation:
    2475             :     // -(X >>u 31) -> (X >>s 31)
    2476             :     // -(X >>s 31) -> (X >>u 31)
    2477        7020 :     if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
    2478         116 :       ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
    2479         232 :       if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
    2480           6 :         auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
    2481           6 :         if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
    2482          12 :           return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
    2483             :       }
    2484             :     }
    2485             : 
    2486             :     // 0 - X --> 0 if the sub is NUW.
    2487        3504 :     if (N->getFlags().hasNoUnsignedWrap())
    2488           2 :       return N0;
    2489             : 
    2490       14008 :     if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
    2491             :       // N1 is either 0 or the minimum signed value. If the sub is NSW, then
    2492             :       // N1 must be 0 because negating the minimum signed value is undefined.
    2493           6 :       if (N->getFlags().hasNoSignedWrap())
    2494           2 :         return N0;
    2495             : 
    2496             :       // 0 - X --> X if X is 0 or the minimum signed value.
    2497           4 :       return N1;
    2498             :     }
    2499             :   }
    2500             : 
    2501             :   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
    2502      417911 :   if (isAllOnesConstantOrAllOnesSplatConstant(N0))
    2503          86 :     return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
    2504             : 
    2505             :   // fold A-(A-B) -> B
    2506      417868 :   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
    2507         237 :     return N1.getOperand(1);
    2508             : 
    2509             :   // fold (A+B)-A -> B
    2510      417631 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
    2511           5 :     return N0.getOperand(1);
    2512             : 
    2513             :   // fold (A+B)-B -> A
    2514      417626 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
    2515           3 :     return N0.getOperand(0);
    2516             : 
    2517             :   // fold C2-(A+C1) -> (C2-C1)-A
    2518      417623 :   if (N1.getOpcode() == ISD::ADD) {
    2519         484 :     SDValue N11 = N1.getOperand(1);
    2520         649 :     if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
    2521         165 :         isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
    2522         282 :       SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
    2523         282 :       return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
    2524             :     }
    2525             :   }
    2526             : 
    2527             :   // fold ((A+(B+or-C))-B) -> A+or-C
    2528        1126 :   if (N0.getOpcode() == ISD::ADD &&
    2529        2232 :       (N0.getOperand(1).getOpcode() == ISD::SUB ||
    2530      417482 :        N0.getOperand(1).getOpcode() == ISD::ADD) &&
    2531             :       N0.getOperand(1).getOperand(0) == N1)
    2532           7 :     return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
    2533           7 :                        N0.getOperand(1).getOperand(1));
    2534             : 
    2535             :   // fold ((A+(C+B))-B) -> A+C
    2536      419713 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
    2537             :       N0.getOperand(1).getOperand(1) == N1)
    2538           1 :     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
    2539           1 :                        N0.getOperand(1).getOperand(0));
    2540             : 
    2541             :   // fold ((A-(B-C))-C) -> A-B
    2542      421482 :   if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
    2543             :       N0.getOperand(1).getOperand(1) == N1)
    2544           3 :     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
    2545           3 :                        N0.getOperand(1).getOperand(0));
    2546             : 
    2547             :   // If either operand of a sub is undef, the result is undef
    2548      417471 :   if (N0.isUndef())
    2549           0 :     return N0;
    2550      417471 :   if (N1.isUndef())
    2551           0 :     return N1;
    2552             : 
    2553             :   // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
    2554      417471 :   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
    2555       14506 :     if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
    2556           2 :       SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
    2557           2 :       SDValue S0 = N1.getOperand(0);
    2558             :       if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
    2559             :         unsigned OpSizeInBits = VT.getScalarSizeInBits();
    2560           2 :         if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
    2561           4 :           if (C->getAPIntValue() == (OpSizeInBits - 1))
    2562           6 :             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
    2563             :       }
    2564             :     }
    2565             :   }
    2566             : 
    2567             :   // If the relocation model supports it, consider symbol offsets.
    2568             :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
    2569          11 :     if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
    2570             :       // fold (sub Sym, c) -> Sym-c
    2571             :       if (N1C && GA->getOpcode() == ISD::GlobalAddress)
    2572             :         return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
    2573             :                                     GA->getOffset() -
    2574             :                                         (uint64_t)N1C->getSExtValue());
    2575             :       // fold (sub Sym+c1, Sym+c2) -> c1-c2
    2576             :       if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
    2577           0 :         if (GA->getGlobal() == GB->getGlobal())
    2578           0 :           return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
    2579           0 :                                  DL, VT);
    2580             :     }
    2581             : 
    2582             :   // sub X, (sextinreg Y i1) -> add X, (and Y 1)
    2583      417469 :   if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2584             :     VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    2585             :     if (TN->getVT() == MVT::i1) {
    2586          58 :       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
    2587         116 :                                  DAG.getConstant(1, DL, VT));
    2588         116 :       return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
    2589             :     }
    2590             :   }
    2591             : 
    2592      417411 :   return SDValue();
    2593             : }
    2594             : 
    2595          33 : SDValue DAGCombiner::visitSUBC(SDNode *N) {
    2596          33 :   SDValue N0 = N->getOperand(0);
    2597          33 :   SDValue N1 = N->getOperand(1);
    2598          33 :   EVT VT = N0.getValueType();
    2599             :   SDLoc DL(N);
    2600             : 
    2601             :   // If the flag result is dead, turn this into an SUB.
    2602          33 :   if (!N->hasAnyUseOfValue(1))
    2603           0 :     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
    2604           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2605             : 
    2606             :   // fold (subc x, x) -> 0 + no borrow
    2607             :   if (N0 == N1)
    2608           0 :     return CombineTo(N, DAG.getConstant(0, DL, VT),
    2609           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2610             : 
    2611             :   // fold (subc x, 0) -> x + no borrow
    2612          33 :   if (isNullConstant(N1))
    2613           0 :     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2614             : 
    2615             :   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
    2616          33 :   if (isAllOnesConstant(N0))
    2617           0 :     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
    2618           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2619             : 
    2620          33 :   return SDValue();
    2621             : }
    2622             : 
    2623        1832 : SDValue DAGCombiner::visitUSUBO(SDNode *N) {
    2624        1832 :   SDValue N0 = N->getOperand(0);
    2625        1832 :   SDValue N1 = N->getOperand(1);
    2626        1832 :   EVT VT = N0.getValueType();
    2627        1832 :   if (VT.isVector())
    2628           0 :     return SDValue();
    2629             : 
    2630        3664 :   EVT CarryVT = N->getValueType(1);
    2631             :   SDLoc DL(N);
    2632             : 
    2633             :   // If the flag result is dead, turn this into an SUB.
    2634        1832 :   if (!N->hasAnyUseOfValue(1))
    2635          52 :     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
    2636         156 :                      DAG.getUNDEF(CarryVT));
    2637             : 
    2638             :   // fold (usubo x, x) -> 0 + no borrow
    2639             :   if (N0 == N1)
    2640           7 :     return CombineTo(N, DAG.getConstant(0, DL, VT),
    2641          14 :                      DAG.getConstant(0, DL, CarryVT));
    2642             : 
    2643             :   // fold (usubo x, 0) -> x + no borrow
    2644        1773 :   if (isNullConstant(N1))
    2645          13 :     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
    2646             : 
    2647             :   // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
    2648        1760 :   if (isAllOnesConstant(N0))
    2649           3 :     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
    2650           9 :                      DAG.getConstant(0, DL, CarryVT));
    2651             : 
    2652        1757 :   return SDValue();
    2653             : }
    2654             : 
    2655          43 : SDValue DAGCombiner::visitSUBE(SDNode *N) {
    2656          43 :   SDValue N0 = N->getOperand(0);
    2657          43 :   SDValue N1 = N->getOperand(1);
    2658          43 :   SDValue CarryIn = N->getOperand(2);
    2659             : 
    2660             :   // fold (sube x, y, false) -> (subc x, y)
    2661          43 :   if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    2662           0 :     return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
    2663             : 
    2664          43 :   return SDValue();
    2665             : }
    2666             : 
    2667         812 : SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
    2668         812 :   SDValue N0 = N->getOperand(0);
    2669         812 :   SDValue N1 = N->getOperand(1);
    2670         812 :   SDValue CarryIn = N->getOperand(2);
    2671             : 
    2672             :   // fold (subcarry x, y, false) -> (usubo x, y)
    2673         812 :   if (isNullConstant(CarryIn)) {
    2674         119 :     if (!LegalOperations ||
    2675         109 :         TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
    2676         357 :       return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
    2677             :   }
    2678             : 
    2679         693 :   return SDValue();
    2680             : }
    2681             : 
    2682       52707 : SDValue DAGCombiner::visitMUL(SDNode *N) {
    2683       52707 :   SDValue N0 = N->getOperand(0);
    2684       52707 :   SDValue N1 = N->getOperand(1);
    2685      105414 :   EVT VT = N0.getValueType();
    2686             : 
    2687             :   // fold (mul x, undef) -> 0
    2688      105413 :   if (N0.isUndef() || N1.isUndef())
    2689          84 :     return DAG.getConstant(0, SDLoc(N), VT);
    2690             : 
    2691             :   bool N0IsConst = false;
    2692             :   bool N1IsConst = false;
    2693             :   bool N1IsOpaqueConst = false;
    2694             :   bool N0IsOpaqueConst = false;
    2695             :   APInt ConstValue0, ConstValue1;
    2696             :   // fold vector ops
    2697       52679 :   if (VT.isVector()) {
    2698        8646 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2699          13 :       return FoldedVOp;
    2700             : 
    2701        8633 :     N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
    2702        8633 :     N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
    2703             :     assert((!N0IsConst ||
    2704             :             ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&
    2705             :            "Splat APInt should be element width");
    2706             :     assert((!N1IsConst ||
    2707             :             ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&
    2708             :            "Splat APInt should be element width");
    2709             :   } else {
    2710             :     N0IsConst = isa<ConstantSDNode>(N0);
    2711             :     if (N0IsConst) {
    2712          18 :       ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
    2713             :       N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
    2714             :     }
    2715             :     N1IsConst = isa<ConstantSDNode>(N1);
    2716             :     if (N1IsConst) {
    2717       69186 :       ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
    2718             :       N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
    2719             :     }
    2720             :   }
    2721             : 
    2722             :   // fold (mul c1, c2) -> c1*c2
    2723       52666 :   if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
    2724          12 :     return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
    2725          12 :                                       N0.getNode(), N1.getNode());
    2726             : 
    2727             :   // canonicalize constant to RHS (vector doesn't have to splat)
    2728       52689 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    2729          29 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    2730          87 :     return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
    2731             :   // fold (mul x, 0) -> 0
    2732       88265 :   if (N1IsConst && ConstValue1.isNullValue())
    2733          38 :     return N1;
    2734             :   // fold (mul x, 1) -> x
    2735       88189 :   if (N1IsConst && ConstValue1.isOneValue())
    2736         793 :     return N0;
    2737             : 
    2738       51800 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2739           3 :     return NewSel;
    2740             : 
    2741             :   // fold (mul x, -1) -> 0-x
    2742       86597 :   if (N1IsConst && ConstValue1.isAllOnesValue()) {
    2743             :     SDLoc DL(N);
    2744          44 :     return DAG.getNode(ISD::SUB, DL, VT,
    2745          44 :                        DAG.getConstant(0, DL, VT), N0);
    2746             :   }
    2747             :   // fold (mul x, (1 << c)) -> x << c
    2748       87076 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    2749      113989 :       DAG.isKnownToBeAPowerOfTwo(N1) &&
    2750         155 :       (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
    2751             :     SDLoc DL(N);
    2752       26913 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    2753       26913 :     AddToWorklist(LogBase2.getNode());
    2754             : 
    2755       53826 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    2756       26913 :     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
    2757       26913 :     AddToWorklist(Trunc.getNode());
    2758       53826 :     return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
    2759             :   }
    2760             :   // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
    2761       82453 :   if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
    2762         132 :     unsigned Log2Val = (-ConstValue1).logBase2();
    2763             :     SDLoc DL(N);
    2764             :     // FIXME: If the input is something that is easily negated (e.g. a
    2765             :     // single-use add), we should put the negate there.
    2766          66 :     return DAG.getNode(ISD::SUB, DL, VT,
    2767          66 :                        DAG.getConstant(0, DL, VT),
    2768             :                        DAG.getNode(ISD::SHL, DL, VT, N0,
    2769             :                             DAG.getConstant(Log2Val, DL,
    2770         198 :                                       getShiftAmountTy(N0.getValueType()))));
    2771             :   }
    2772             : 
    2773             :   // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
    2774       24797 :   if (N0.getOpcode() == ISD::SHL &&
    2775       24780 :       isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
    2776          12 :       isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
    2777          24 :     SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
    2778           6 :     if (isConstantOrConstantVector(C3))
    2779          24 :       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
    2780             :   }
    2781             : 
    2782             :   // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
    2783             :   // use.
    2784             :   {
    2785             :     SDValue Sh(nullptr, 0), Y(nullptr, 0);
    2786             : 
    2787             :     // Check for both (mul (shl X, C), Y)  and  (mul Y, (shl X, C)).
    2788       24785 :     if (N0.getOpcode() == ISD::SHL &&
    2789       24785 :         isConstantOrConstantVector(N0.getOperand(1)) &&
    2790          15 :         N0.getNode()->hasOneUse()) {
    2791          12 :       Sh = N0; Y = N1;
    2792       24765 :     } else if (N1.getOpcode() == ISD::SHL &&
    2793       24765 :                isConstantOrConstantVector(N1.getOperand(1)) &&
    2794           6 :                N1.getNode()->hasOneUse()) {
    2795           3 :       Sh = N1; Y = N0;
    2796             :     }
    2797             : 
    2798       24768 :     if (Sh.getNode()) {
    2799          45 :       SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
    2800          45 :       return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
    2801             :     }
    2802             :   }
    2803             : 
    2804             :   // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
    2805       33136 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
    2806        8834 :       N0.getOpcode() == ISD::ADD &&
    2807       25846 :       DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
    2808         191 :       isMulAddWithConstProfitable(N, N0, N1))
    2809         176 :       return DAG.getNode(ISD::ADD, SDLoc(N), VT,
    2810         176 :                          DAG.getNode(ISD::MUL, SDLoc(N0), VT,
    2811          88 :                                      N0.getOperand(0), N1),
    2812          88 :                          DAG.getNode(ISD::MUL, SDLoc(N1), VT,
    2813         528 :                                      N0.getOperand(1), N1));
    2814             : 
    2815             :   // reassociate mul
    2816       49330 :   if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
    2817          34 :     return RMUL;
    2818             : 
    2819       24631 :   return SDValue();
    2820             : }
    2821             : 
    2822             : /// Return true if divmod libcall is available.
    2823        1371 : static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
    2824             :                                      const TargetLowering &TLI) {
    2825             :   RTLIB::Libcall LC;
    2826        1371 :   EVT NodeType = Node->getValueType(0);
    2827        1371 :   if (!NodeType.isSimple())
    2828             :     return false;
    2829        1371 :   switch (NodeType.getSimpleVT().SimpleTy) {
    2830             :   default: return false; // No libcall for vector types.
    2831           0 :   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
    2832           4 :   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
    2833        1172 :   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
    2834         188 :   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
    2835           7 :   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
    2836             :   }
    2837             : 
    2838        1371 :   return TLI.getLibcallName(LC) != nullptr;
    2839             : }
    2840             : 
    2841             : /// Issue divrem if both quotient and remainder are needed.
    2842        5776 : SDValue DAGCombiner::useDivRem(SDNode *Node) {
    2843        5776 :   if (Node->use_empty())
    2844           0 :     return SDValue(); // This is a dead node, leave it alone.
    2845             : 
    2846        5776 :   unsigned Opcode = Node->getOpcode();
    2847        5776 :   bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
    2848        5776 :   unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    2849             : 
    2850             :   // DivMod lib calls can still work on non-legal types if using lib-calls.
    2851       11552 :   EVT VT = Node->getValueType(0);
    2852        5776 :   if (VT.isVector() || !VT.isInteger())
    2853         613 :     return SDValue();
    2854             : 
    2855        5983 :   if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
    2856         837 :     return SDValue();
    2857             : 
    2858             :   // If DIVREM is going to get expanded into a libcall,
    2859             :   // but there is no libcall available, then don't combine.
    2860        5697 :   if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
    2861        1371 :       !isDivRemLibcallAvailable(Node, isSigned, TLI))
    2862        1249 :     return SDValue();
    2863             : 
    2864             :   // If div is legal, it's better to do the normal expansion
    2865             :   unsigned OtherOpcode = 0;
    2866        3077 :   if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
    2867        1595 :     OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
    2868        1595 :     if (TLI.isOperationLegalOrCustom(Opcode, VT))
    2869         209 :       return SDValue();
    2870             :   } else {
    2871        1482 :     OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
    2872        1482 :     if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
    2873         176 :       return SDValue();
    2874             :   }
    2875             : 
    2876        2692 :   SDValue Op0 = Node->getOperand(0);
    2877        2692 :   SDValue Op1 = Node->getOperand(1);
    2878             :   SDValue combined;
    2879        2692 :   for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
    2880        7021 :          UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
    2881             :     SDNode *User = *UI;
    2882       10211 :     if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
    2883        1541 :         User->use_empty())
    2884        2799 :       continue;
    2885             :     // Convert the other matching node(s), too;
    2886             :     // otherwise, the DIVREM may get target-legalized into something
    2887             :     // target-specific that we won't be able to recognize.
    2888             :     unsigned UserOpc = User->getOpcode();
    2889        2305 :     if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
    2890        2307 :         User->getOperand(0) == Op0 &&
    2891             :         User->getOperand(1) == Op1) {
    2892         175 :       if (!combined) {
    2893         175 :         if (UserOpc == OtherOpcode) {
    2894         175 :           SDVTList VTs = DAG.getVTList(VT, VT);
    2895         525 :           combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
    2896           0 :         } else if (UserOpc == DivRemOpc) {
    2897             :           combined = SDValue(User, 0);
    2898             :         } else {
    2899             :           assert(UserOpc == Opcode);
    2900           0 :           continue;
    2901             :         }
    2902             :       }
    2903         175 :       if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
    2904             :         CombineTo(User, combined);
    2905          32 :       else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
    2906             :         CombineTo(User, combined.getValue(1));
    2907             :     }
    2908             :   }
    2909        2692 :   return combined;
    2910             : }
    2911             : 
    2912        8416 : static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
    2913        8416 :   SDValue N0 = N->getOperand(0);
    2914        8416 :   SDValue N1 = N->getOperand(1);
    2915       16832 :   EVT VT = N->getValueType(0);
    2916             :   SDLoc DL(N);
    2917             : 
    2918       25248 :   if (DAG.isUndef(N->getOpcode(), {N0, N1}))
    2919           1 :     return DAG.getUNDEF(VT);
    2920             : 
    2921             :   // undef / X -> 0
    2922             :   // undef % X -> 0
    2923        8415 :   if (N0.isUndef())
    2924           0 :     return DAG.getConstant(0, DL, VT);
    2925             : 
    2926        8415 :   return SDValue();
    2927             : }
    2928             : 
    2929        3393 : SDValue DAGCombiner::visitSDIV(SDNode *N) {
    2930        3393 :   SDValue N0 = N->getOperand(0);
    2931        3393 :   SDValue N1 = N->getOperand(1);
    2932        6786 :   EVT VT = N->getValueType(0);
    2933             : 
    2934             :   // fold vector ops
    2935        3393 :   if (VT.isVector())
    2936         417 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2937           1 :       return FoldedVOp;
    2938             : 
    2939             :   SDLoc DL(N);
    2940             : 
    2941             :   // fold (sdiv c1, c2) -> c1/c2
    2942        3392 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    2943        3392 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    2944        3392 :   if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
    2945           0 :     return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
    2946             :   // fold (sdiv X, 1) -> X
    2947        5124 :   if (N1C && N1C->isOne())
    2948         148 :     return N0;
    2949             :   // fold (sdiv X, -1) -> 0-X
    2950        6412 :   if (N1C && N1C->isAllOnesValue())
    2951          24 :     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
    2952             : 
    2953        3232 :   if (SDValue V = simplifyDivRem(N, DAG))
    2954           0 :     return V;
    2955             : 
    2956        3232 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2957           2 :     return NewSel;
    2958             : 
    2959             :   // If we know the sign bits of both operands are zero, strength reduce to a
    2960             :   // udiv instead.  Handles (X&15) /s 4 -> X&15 >> 2
    2961        3230 :   if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    2962          44 :     return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
    2963             : 
    2964             :   // Helper for determining whether a value is a power-2 constant scalar or a
    2965             :   // vector of such elements.
    2966             :   SmallBitVector KnownNegatives(
    2967        8068 :       (N1C || !VT.isVector()) ? 1 : VT.getVectorNumElements(), false);
    2968        3208 :   unsigned EltIndex = 0;
    2969        1376 :   auto IsPowerOfTwo = [&KnownNegatives, &EltIndex](ConstantSDNode *C) {
    2970        1345 :     unsigned Idx = EltIndex++;
    2971        4035 :     if (C->isNullValue() || C->isOpaque())
    2972             :       return false;
    2973             :     // The instruction sequence to be generated contains shifting C by (op size
    2974             :     // in bits - # of trailing zeros in C), which results in an undef value when
    2975             :     // C == 1. (e.g. if the op size in bits is 32, it will be (sra x , 32) if C
    2976             :     // == 1)
    2977        1336 :     if (C->getAPIntValue().isOneValue())
    2978             :       return false;
    2979             : 
    2980        1234 :     if (C->getAPIntValue().isPowerOf2())
    2981             :       return true;
    2982        1479 :     if ((-C->getAPIntValue()).isPowerOf2()) {
    2983          31 :       KnownNegatives.set(Idx);
    2984          31 :       return true;
    2985             :     }
    2986             :     return false;
    2987             :   };
    2988             : 
    2989             :   // fold (sdiv X, pow2) -> simple ops after legalize
    2990             :   // FIXME: We check for the exact bit here because the generic lowering gives
    2991             :   // better results in that case. The target-specific lowering should learn how
    2992             :   // to handle exact sdivs efficiently.
    2993        9305 :   if (!N->getFlags().hasExact() &&
    2994        8536 :       ISD::matchUnaryPredicate(N1C ? SDValue(N1C, 0) : N1, IsPowerOfTwo)) {
    2995             :     // Target-specific implementation of sdiv x, pow2.
    2996         769 :     if (SDValue Res = BuildSDIVPow2(N))
    2997          37 :       return Res;
    2998             : 
    2999             :     // Create constants that are functions of the shift amount value.
    3000         732 :     EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
    3001        1464 :     SDValue Bits = DAG.getConstant(VT.getScalarSizeInBits(), DL, ShiftAmtTy);
    3002        1464 :     SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
    3003         732 :     C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
    3004        1464 :     SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
    3005         732 :     if (!isConstantOrConstantVector(Inexact))
    3006           0 :       return SDValue();
    3007             :     // Splat the sign bit into the register
    3008         732 :     SDValue Sign = DAG.getNode(
    3009             :         ISD::SRA, DL, VT, N0,
    3010         732 :         DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, ShiftAmtTy));
    3011         732 :     AddToWorklist(Sign.getNode());
    3012             : 
    3013             :     // Add (N0 < 0) ? abs2 - 1 : 0;
    3014        1464 :     SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
    3015        1464 :     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
    3016         732 :     AddToWorklist(Srl.getNode());
    3017         732 :     AddToWorklist(Add.getNode()); // Divide by pow2
    3018        1464 :     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
    3019             : 
    3020             :     // If dividing by a positive value, we're done. Otherwise, the result must
    3021             :     // be negated.
    3022         732 :     if (KnownNegatives.none())
    3023         708 :       return Sra;
    3024             : 
    3025          24 :     AddToWorklist(Sra.getNode());
    3026             :     SDValue Sub =
    3027          48 :         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Sra);
    3028             :     // If all shift amount elements are negative, we're done.
    3029          24 :     if (KnownNegatives.all())
    3030          24 :       return Sub;
    3031             : 
    3032             :     // Shift amount has both positive and negative elements.
    3033             :     assert(VT.isVector() && !N0C &&
    3034             :            "Expecting a non-splat vector shift amount");
    3035             : 
    3036             :     SmallVector<SDValue, 64> VSelectMask;
    3037           0 :     for (int i = 0, e = VT.getVectorNumElements(); i < e; ++i)
    3038           0 :       VSelectMask.push_back(
    3039           0 :           DAG.getConstant(KnownNegatives[i] ? -1 : 0, DL, MVT::i1));
    3040             : 
    3041             :     SDValue Mask =
    3042           0 :         DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i1,
    3043             :                                             VT.getVectorElementCount()),
    3044           0 :                            DL, VSelectMask);
    3045           0 :     return DAG.getNode(ISD::VSELECT, DL, VT, Mask, Sub, Sra);
    3046             :   }
    3047             : 
    3048             :   // If integer divide is expensive and we satisfy the requirements, emit an
    3049             :   // alternate sequence.  Targets may check function attributes for size/speed
    3050             :   // trade-offs.
    3051        2439 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3052        3229 :   if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
    3053         747 :     if (SDValue Op = BuildSDIV(N))
    3054         631 :       return Op;
    3055             : 
    3056             :   // sdiv, srem -> sdivrem
    3057             :   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
    3058             :   // true.  Otherwise, we break the simplification logic in visitREM().
    3059        1967 :   if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    3060        1692 :     if (SDValue DivRem = useDivRem(N))
    3061          14 :         return DivRem;
    3062             : 
    3063        1794 :   return SDValue();
    3064             : }
    3065             : 
    3066        2224 : SDValue DAGCombiner::visitUDIV(SDNode *N) {
    3067        2224 :   SDValue N0 = N->getOperand(0);
    3068        2224 :   SDValue N1 = N->getOperand(1);
    3069        4448 :   EVT VT = N->getValueType(0);
    3070             : 
    3071             :   // fold vector ops
    3072        2224 :   if (VT.isVector())
    3073         247 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3074           0 :       return FoldedVOp;
    3075             : 
    3076             :   SDLoc DL(N);
    3077             : 
    3078             :   // fold (udiv c1, c2) -> c1/c2
    3079        2224 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3080        2224 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3081        2224 :   if (N0C && N1C)
    3082           0 :     if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
    3083           0 :                                                     N0C, N1C))
    3084           0 :       return Folded;
    3085             : 
    3086        2224 :   if (SDValue V = simplifyDivRem(N, DAG))
    3087           0 :     return V;
    3088             : 
    3089        2224 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3090           2 :     return NewSel;
    3091             : 
    3092             :   // fold (udiv x, (1 << c)) -> x >>u c
    3093        2889 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    3094         667 :       DAG.isKnownToBeAPowerOfTwo(N1)) {
    3095          58 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    3096          58 :     AddToWorklist(LogBase2.getNode());
    3097             : 
    3098          58 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    3099          58 :     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
    3100          58 :     AddToWorklist(Trunc.getNode());
    3101         116 :     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    3102             :   }
    3103             : 
    3104             :   // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
    3105        2164 :   if (N1.getOpcode() == ISD::SHL) {
    3106           9 :     SDValue N10 = N1.getOperand(0);
    3107          18 :     if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
    3108           9 :         DAG.isKnownToBeAPowerOfTwo(N10)) {
    3109           9 :       SDValue LogBase2 = BuildLogBase2(N10, DL);
    3110           9 :       AddToWorklist(LogBase2.getNode());
    3111             : 
    3112          18 :       EVT ADDVT = N1.getOperand(1).getValueType();
    3113           9 :       SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
    3114           9 :       AddToWorklist(Trunc.getNode());
    3115          18 :       SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
    3116           9 :       AddToWorklist(Add.getNode());
    3117          18 :       return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
    3118             :     }
    3119             :   }
    3120             : 
    3121             :   // fold (udiv x, c) -> alternate
    3122        2155 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3123        2764 :   if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
    3124         568 :     if (SDValue Op = BuildUDIV(N))
    3125         455 :       return Op;
    3126             : 
    3127             :   // sdiv, srem -> sdivrem
    3128             :   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
    3129             :   // true.  Otherwise, we break the simplification logic in visitREM().
    3130        1854 :   if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    3131        1587 :     if (SDValue DivRem = useDivRem(N))
    3132          18 :         return DivRem;
    3133             : 
    3134        1682 :   return SDValue();
    3135             : }
    3136             : 
    3137             : // handles ISD::SREM and ISD::UREM
    3138        2966 : SDValue DAGCombiner::visitREM(SDNode *N) {
    3139        2966 :   unsigned Opcode = N->getOpcode();
    3140        2966 :   SDValue N0 = N->getOperand(0);
    3141        2966 :   SDValue N1 = N->getOperand(1);
    3142        5932 :   EVT VT = N->getValueType(0);
    3143             :   bool isSigned = (Opcode == ISD::SREM);
    3144             :   SDLoc DL(N);
    3145             : 
    3146             :   // fold (rem c1, c2) -> c1%c2
    3147        2966 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3148        2966 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3149        2966 :   if (N0C && N1C)
    3150           6 :     if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
    3151           6 :       return Folded;
    3152             : 
    3153        2960 :   if (SDValue V = simplifyDivRem(N, DAG))
    3154           1 :     return V;
    3155             : 
    3156        2959 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3157           4 :     return NewSel;
    3158             : 
    3159        2955 :   if (isSigned) {
    3160             :     // If we know the sign bits of both operands are zero, strength reduce to a
    3161             :     // urem instead.  Handles (X & 0x0FFFFFFF) %s 16 -> X&15
    3162        1417 :     if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    3163          28 :       return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
    3164             :   } else {
    3165        1538 :     SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
    3166        1538 :     if (DAG.isKnownToBeAPowerOfTwo(N1)) {
    3167             :       // fold (urem x, pow2) -> (and x, pow2-1)
    3168         102 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
    3169          51 :       AddToWorklist(Add.getNode());
    3170         102 :       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    3171             :     }
    3172        1493 :     if (N1.getOpcode() == ISD::SHL &&
    3173          12 :         DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
    3174             :       // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
    3175          12 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
    3176           6 :       AddToWorklist(Add.getNode());
    3177          12 :       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    3178             :     }
    3179             :   }
    3180             : 
    3181        2884 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3182             : 
    3183             :   // If X/C can be simplified by the division-by-constant logic, lower
    3184             :   // X%C to the equivalent of X-X/C*C.
    3185             :   // To avoid mangling nodes, this simplification requires that the combine()
    3186             :   // call for the speculative DIV must not cause a DIVREM conversion.  We guard
    3187             :   // against this by skipping the simplification if isIntDivCheap().  When
    3188             :   // div is not cheap, combine will not return a DIVREM.  Regardless,
    3189             :   // checking cheapness here makes sense since the simplification results in
    3190             :   // fatter code.
    3191        3451 :   if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap(VT, Attr)) {
    3192         496 :     unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
    3193         992 :     SDValue Div = DAG.getNode(DivOpcode, DL, VT, N0, N1);
    3194         496 :     AddToWorklist(Div.getNode());
    3195         496 :     SDValue OptimizedDiv = combine(Div.getNode());
    3196         780 :     if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode() &&
    3197         886 :         OptimizedDiv.getOpcode() != ISD::UDIVREM &&
    3198             :         OptimizedDiv.getOpcode() != ISD::SDIVREM) {
    3199         774 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
    3200         774 :       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
    3201         387 :       AddToWorklist(Mul.getNode());
    3202         387 :       return Sub;
    3203             :     }
    3204             :   }
    3205             : 
    3206             :   // sdiv, srem -> sdivrem
    3207        2497 :   if (SDValue DivRem = useDivRem(N))
    3208         143 :     return DivRem.getValue(1);
    3209             : 
    3210        2354 :   return SDValue();
    3211             : }
    3212             : 
    3213        1153 : SDValue DAGCombiner::visitMULHS(SDNode *N) {
    3214        1153 :   SDValue N0 = N->getOperand(0);
    3215        1153 :   SDValue N1 = N->getOperand(1);
    3216        2306 :   EVT VT = N->getValueType(0);
    3217             :   SDLoc DL(N);
    3218             : 
    3219        1153 :   if (VT.isVector()) {
    3220             :     // fold (mulhs x, 0) -> 0
    3221         562 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    3222           0 :       return N1;
    3223         562 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    3224           2 :       return N0;
    3225             :   }
    3226             : 
    3227             :   // fold (mulhs x, 0) -> 0
    3228        1151 :   if (isNullConstant(N1))
    3229           0 :     return N1;
    3230             :   // fold (mulhs x, 1) -> (sra x, size(x)-1)
    3231        1151 :   if (isOneConstant(N1))
    3232           0 :     return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
    3233           0 :                        DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
    3234           0 :                                        getShiftAmountTy(N0.getValueType())));
    3235             : 
    3236             :   // fold (mulhs x, undef) -> 0
    3237        3453 :   if (N0.isUndef() || N1.isUndef())
    3238           0 :     return DAG.getConstant(0, DL, VT);
    3239             : 
    3240             :   // If the type twice as wide is legal, transform the mulhs to a wider multiply
    3241             :   // plus a shift.
    3242        2302 :   if (VT.isSimple() && !VT.isVector()) {
    3243         591 :     MVT Simple = VT.getSimpleVT();
    3244         591 :     unsigned SimpleSize = Simple.getSizeInBits();
    3245         591 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3246         591 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3247          98 :       N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
    3248          98 :       N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
    3249          98 :       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
    3250          98 :       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
    3251             :             DAG.getConstant(SimpleSize, DL,
    3252          98 :                             getShiftAmountTy(N1.getValueType())));
    3253          98 :       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    3254             :     }
    3255             :   }
    3256             : 
    3257        1102 :   return SDValue();
    3258             : }
    3259             : 
    3260        4645 : SDValue DAGCombiner::visitMULHU(SDNode *N) {
    3261        4645 :   SDValue N0 = N->getOperand(0);
    3262        4645 :   SDValue N1 = N->getOperand(1);
    3263        9290 :   EVT VT = N->getValueType(0);
    3264             :   SDLoc DL(N);
    3265             : 
    3266        4645 :   if (VT.isVector()) {
    3267             :     // fold (mulhu x, 0) -> 0
    3268         529 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    3269           0 :       return N1;
    3270         529 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    3271           0 :       return N0;
    3272             :   }
    3273             : 
    3274             :   // fold (mulhu x, 0) -> 0
    3275        4645 :   if (isNullConstant(N1))
    3276           0 :     return N1;
    3277             :   // fold (mulhu x, 1) -> 0
    3278        4645 :   if (isOneConstant(N1))
    3279           0 :     return DAG.getConstant(0, DL, N0.getValueType());
    3280             :   // fold (mulhu x, undef) -> 0
    3281        9290 :   if (N0.isUndef() || N1.isUndef())
    3282           0 :     return DAG.getConstant(0, DL, VT);
    3283             : 
    3284             :   // If the type twice as wide is legal, transform the mulhu to a wider multiply
    3285             :   // plus a shift.
    3286        9290 :   if (VT.isSimple() && !VT.isVector()) {
    3287        4116 :     MVT Simple = VT.getSimpleVT();
    3288        4116 :     unsigned SimpleSize = Simple.getSizeInBits();
    3289        4116 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3290        4116 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3291         266 :       N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
    3292         266 :       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
    3293         266 :       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
    3294         266 :       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
    3295             :             DAG.getConstant(SimpleSize, DL,
    3296         266 :                             getShiftAmountTy(N1.getValueType())));
    3297         266 :       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    3298             :     }
    3299             :   }
    3300             : 
    3301        4512 :   return SDValue();
    3302             : }
    3303             : 
    3304             : /// Perform optimizations common to nodes that compute two values. LoOp and HiOp
    3305             : /// give the opcodes for the two computations that are being performed. Return
    3306             : /// true if a simplification was made.
    3307        5794 : SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
    3308             :                                                 unsigned HiOp) {
    3309             :   // If the high half is not needed, just compute the low half.
    3310        5794 :   bool HiExists = N->hasAnyUseOfValue(1);
    3311        5822 :   if (!HiExists &&
    3312          41 :       (!LegalOperations ||
    3313          26 :        TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
    3314          60 :     SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    3315             :     return CombineTo(N, Res, Res);
    3316             :   }
    3317             : 
    3318             :   // If the low half is not needed, just compute the high half.
    3319        5779 :   bool LoExists = N->hasAnyUseOfValue(0);
    3320        6792 :   if (!LoExists &&
    3321        1013 :       (!LegalOperations ||
    3322         548 :        TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
    3323        1860 :     SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    3324             :     return CombineTo(N, Res, Res);
    3325             :   }
    3326             : 
    3327             :   // If both halves are used, return as it is.
    3328        5314 :   if (LoExists && HiExists)
    3329        4753 :     return SDValue();
    3330             : 
    3331             :   // If the two computed results can be simplified separately, separate them.
    3332         561 :   if (LoExists) {
    3333          52 :     SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    3334          13 :     AddToWorklist(Lo.getNode());
    3335          13 :     SDValue LoOpt = combine(Lo.getNode());
    3336          16 :     if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
    3337           3 :         (!LegalOperations ||
    3338           3 :          TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
    3339           3 :       return CombineTo(N, LoOpt, LoOpt);
    3340             :   }
    3341             : 
    3342         558 :   if (HiExists) {
    3343        2192 :     SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    3344         548 :     AddToWorklist(Hi.getNode());
    3345         548 :     SDValue HiOpt = combine(Hi.getNode());
    3346         548 :     if (HiOpt.getNode() && HiOpt != Hi &&
    3347           0 :         (!LegalOperations ||
    3348           0 :          TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
    3349           0 :       return CombineTo(N, HiOpt, HiOpt);
    3350             :   }
    3351             : 
    3352         558 :   return SDValue();
    3353             : }
    3354             : 
    3355         638 : SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
    3356         638 :   if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
    3357         160 :     return Res;
    3358             : 
    3359         956 :   EVT VT = N->getValueType(0);
    3360             :   SDLoc DL(N);
    3361             : 
    3362             :   // If the type is twice as wide is legal, transform the mulhu to a wider
    3363             :   // multiply plus a shift.
    3364         956 :   if (VT.isSimple() && !VT.isVector()) {
    3365         478 :     MVT Simple = VT.getSimpleVT();
    3366         478 :     unsigned SimpleSize = Simple.getSizeInBits();
    3367         478 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3368         478 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3369           0 :       SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
    3370           0 :       SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
    3371           0 :       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
    3372             :       // Compute the high part as N1.
    3373           0 :       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
    3374             :             DAG.getConstant(SimpleSize, DL,
    3375           0 :                             getShiftAmountTy(Lo.getValueType())));
    3376           0 :       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
    3377             :       // Compute the low part as N0.
    3378           0 :       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
    3379             :       return CombineTo(N, Lo, Hi);
    3380             :     }
    3381             :   }
    3382             : 
    3383         478 :   return SDValue();
    3384             : }
    3385             : 
    3386        5156 : SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
    3387        5156 :   if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
    3388         323 :     return Res;
    3389             : 
    3390        9666 :   EVT VT = N->getValueType(0);
    3391             :   SDLoc DL(N);
    3392             : 
    3393             :   // If the type is twice as wide is legal, transform the mulhu to a wider
    3394             :   // multiply plus a shift.
    3395        9666 :   if (VT.isSimple() && !VT.isVector()) {
    3396        4833 :     MVT Simple = VT.getSimpleVT();
    3397        4833 :     unsigned SimpleSize = Simple.getSizeInBits();
    3398        4833 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3399        4833 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3400           0 :       SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
    3401           0 :       SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
    3402           0 :       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
    3403             :       // Compute the high part as N1.
    3404           0 :       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
    3405             :             DAG.getConstant(SimpleSize, DL,
    3406           0 :                             getShiftAmountTy(Lo.getValueType())));
    3407           0 :       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
    3408             :       // Compute the low part as N0.
    3409           0 :       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
    3410             :       return CombineTo(N, Lo, Hi);
    3411             :     }
    3412             :   }
    3413             : 
    3414        4833 :   return SDValue();
    3415             : }
    3416             : 
    3417          96 : SDValue DAGCombiner::visitSMULO(SDNode *N) {
    3418             :   // (smulo x, 2) -> (saddo x, x)
    3419          96 :   if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    3420          14 :     if (C2->getAPIntValue() == 2)
    3421           6 :       return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
    3422           6 :                          N->getOperand(0), N->getOperand(0));
    3423             : 
    3424          93 :   return SDValue();
    3425             : }
    3426             : 
    3427         141 : SDValue DAGCombiner::visitUMULO(SDNode *N) {
    3428             :   // (umulo x, 2) -> (uaddo x, x)
    3429         141 :   if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    3430         104 :     if (C2->getAPIntValue() == 2)
    3431           8 :       return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
    3432           8 :                          N->getOperand(0), N->getOperand(0));
    3433             : 
    3434         137 :   return SDValue();
    3435             : }
    3436             : 
    3437       24194 : SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
    3438       24194 :   SDValue N0 = N->getOperand(0);
    3439       24194 :   SDValue N1 = N->getOperand(1);
    3440       24194 :   EVT VT = N0.getValueType();
    3441             : 
    3442             :   // fold vector ops
    3443       24194 :   if (VT.isVector())
    3444       19563 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3445         392 :       return FoldedVOp;
    3446             : 
    3447             :   // fold operation with constant operands.
    3448             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    3449             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    3450       23802 :   if (N0C && N1C)
    3451           0 :     return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
    3452             : 
    3453             :   // canonicalize constant to RHS
    3454       23812 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    3455          10 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    3456          40 :     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
    3457             : 
    3458             :   // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
    3459             :   // Only do this if the current op isn't legal and the flipped is.
    3460       23792 :   unsigned Opcode = N->getOpcode();
    3461       23792 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    3462        5398 :   if (!TLI.isOperationLegal(Opcode, VT) &&
    3463        5446 :       (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
    3464          48 :       (N1.isUndef() || DAG.SignBitIsZero(N1))) {
    3465             :     unsigned AltOpcode;
    3466          14 :     switch (Opcode) {
    3467             :     case ISD::SMIN: AltOpcode = ISD::UMIN; break;
    3468           1 :     case ISD::SMAX: AltOpcode = ISD::UMAX; break;
    3469          10 :     case ISD::UMIN: AltOpcode = ISD::SMIN; break;
    3470           2 :     case ISD::UMAX: AltOpcode = ISD::SMAX; break;
    3471           0 :     default: llvm_unreachable("Unknown MINMAX opcode");
    3472             :     }
    3473             :     if (TLI.isOperationLegal(AltOpcode, VT))
    3474          12 :       return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
    3475             :   }
    3476             : 
    3477       23788 :   return SDValue();
    3478             : }
    3479             : 
    3480             : /// If this is a binary operator with two operands of the same opcode, try to
    3481             : /// simplify it.
    3482       50790 : SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
    3483       50790 :   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
    3484       50790 :   EVT VT = N0.getValueType();
    3485             :   assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
    3486             : 
    3487             :   // Bail early if none of these transforms apply.
    3488       50790 :   if (N0.getNumOperands() == 0) return SDValue();
    3489             : 
    3490             :   // For each of OP in AND/OR/XOR:
    3491             :   // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
    3492             :   // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
    3493             :   // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
    3494             :   // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
    3495             :   // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
    3496             :   //
    3497             :   // do not sink logical op inside of a vector extend, since it may combine
    3498             :   // into a vsetcc.
    3499      101554 :   EVT Op0VT = N0.getOperand(0).getValueType();
    3500       50620 :   if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
    3501       50561 :        N0.getOpcode() == ISD::SIGN_EXTEND ||
    3502       50561 :        N0.getOpcode() == ISD::BSWAP ||
    3503             :        // Avoid infinite looping with PromoteIntBinOp.
    3504         965 :        (N0.getOpcode() == ISD::ANY_EXTEND &&
    3505       53440 :         (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
    3506         990 :        (N0.getOpcode() == ISD::TRUNCATE &&
    3507        1203 :         (!TLI.isZExtFree(VT, Op0VT) ||
    3508         213 :          !TLI.isTruncateFree(Op0VT, VT)) &&
    3509        1764 :         TLI.isTypeLegal(Op0VT))) &&
    3510           0 :       !VT.isVector() &&
    3511       52393 :       Op0VT == N1.getOperand(0).getValueType() &&
    3512         960 :       (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
    3513        1594 :     SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
    3514             :                                  N0.getOperand(0).getValueType(),
    3515        3188 :                                  N0.getOperand(0), N1.getOperand(0));
    3516         797 :     AddToWorklist(ORNode.getNode());
    3517        2391 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
    3518             :   }
    3519             : 
    3520             :   // For each of OP in SHL/SRL/SRA/AND...
    3521             :   //   fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
    3522             :   //   fold (or  (OP x, z), (OP y, z)) -> (OP (or  x, y), z)
    3523             :   //   fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
    3524       49748 :   if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
    3525       99609 :        N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
    3526             :       N0.getOperand(1) == N1.getOperand(1)) {
    3527         208 :     SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
    3528             :                                  N0.getOperand(0).getValueType(),
    3529         416 :                                  N0.getOperand(0), N1.getOperand(0));
    3530         104 :     AddToWorklist(ORNode.getNode());
    3531         208 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
    3532         208 :                        ORNode, N0.getOperand(1));
    3533             :   }
    3534             : 
    3535             :   // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
    3536             :   // Only perform this optimization up until type legalization, before
    3537             :   // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
    3538             :   // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
    3539             :   // we don't want to undo this promotion.
    3540             :   // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
    3541             :   // on scalars.
    3542       25914 :   if ((N0.getOpcode() == ISD::BITCAST ||
    3543       73842 :        N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
    3544       23966 :        Level <= AfterLegalizeTypes) {
    3545         696 :     SDValue In0 = N0.getOperand(0);
    3546         696 :     SDValue In1 = N1.getOperand(0);
    3547         696 :     EVT In0Ty = In0.getValueType();
    3548         696 :     EVT In1Ty = In1.getValueType();
    3549             :     SDLoc DL(N);
    3550             :     // If both incoming values are integers, and the original types are the
    3551             :     // same.
    3552         726 :     if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
    3553         351 :       SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
    3554         234 :       SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
    3555         117 :       AddToWorklist(Op.getNode());
    3556         117 :       return BC;
    3557             :     }
    3558             :   }
    3559             : 
    3560             :   // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
    3561             :   // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
    3562             :   // If both shuffles use the same mask, and both shuffle within a single
    3563             :   // vector, then it is worthwhile to move the swizzle after the operation.
    3564             :   // The type-legalizer generates this pattern when loading illegal
    3565             :   // vector types from memory. In many cases this allows additional shuffle
    3566             :   // optimizations.
    3567             :   // There are other cases where moving the shuffle after the xor/and/or
    3568             :   // is profitable even if shuffles don't perform a swizzle.
    3569             :   // If both shuffles use the same mask, and both shuffles have the same first
    3570             :   // or second operand, then it might still be profitable to move the shuffle
    3571             :   // after the xor/and/or operation.
    3572       49759 :   if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
    3573             :     ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
    3574             :     ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
    3575             : 
    3576             :     assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
    3577             :            "Inputs to shuffles are not the same type");
    3578             : 
    3579             :     // Check that both shuffles use the same mask. The masks are known to be of
    3580             :     // the same length because the result vector type is the same.
    3581             :     // Check also that shuffles have only one use to avoid introducing extra
    3582             :     // instructions.
    3583         161 :     if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
    3584         392 :         SVN0->getMask().equals(SVN1->getMask())) {
    3585         143 :       SDValue ShOp = N0->getOperand(1);
    3586             : 
    3587             :       // Don't try to fold this node if it requires introducing a
    3588             :       // build vector of all zeros that might be illegal at this stage.
    3589         181 :       if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
    3590          26 :         if (!LegalTypes)
    3591          72 :           ShOp = DAG.getConstant(0, SDLoc(N), VT);
    3592             :         else
    3593           2 :           ShOp = SDValue();
    3594             :       }
    3595             : 
    3596             :       // (AND (shuf (A, C), shuf (B, C))) -> shuf (AND (A, B), C)
    3597             :       // (OR  (shuf (A, C), shuf (B, C))) -> shuf (OR  (A, B), C)
    3598             :       // (XOR (shuf (A, C), shuf (B, C))) -> shuf (XOR (A, B), V_0)
    3599          74 :       if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
    3600         148 :         SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
    3601         222 :                                       N0->getOperand(0), N1->getOperand(0));
    3602          74 :         AddToWorklist(NewNode.getNode());
    3603         148 :         return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
    3604          74 :                                     SVN0->getMask());
    3605             :       }
    3606             : 
    3607             :       // Don't try to fold this node if it requires introducing a
    3608             :       // build vector of all zeros that might be illegal at this stage.
    3609          69 :       ShOp = N0->getOperand(0);
    3610          83 :       if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
    3611          14 :         if (!LegalTypes)
    3612          36 :           ShOp = DAG.getConstant(0, SDLoc(N), VT);
    3613             :         else
    3614           2 :           ShOp = SDValue();
    3615             :       }
    3616             : 
    3617             :       // (AND (shuf (C, A), shuf (C, B))) -> shuf (C, AND (A, B))
    3618             :       // (OR  (shuf (C, A), shuf (C, B))) -> shuf (C, OR  (A, B))
    3619             :       // (XOR (shuf (C, A), shuf (C, B))) -> shuf (V_0, XOR (A, B))
    3620         105 :       if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
    3621          72 :         SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
    3622         108 :                                       N0->getOperand(1), N1->getOperand(1));
    3623          36 :         AddToWorklist(NewNode.getNode());
    3624          72 :         return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
    3625          36 :                                     SVN0->getMask());
    3626             :       }
    3627             :     }
    3628             :   }
    3629             : 
    3630       49649 :   return SDValue();
    3631             : }
    3632             : 
    3633             : /// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
    3634      318631 : SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
    3635             :                                        const SDLoc &DL) {
    3636      318631 :   SDValue LL, LR, RL, RR, N0CC, N1CC;
    3637      362503 :   if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
    3638       43872 :       !isSetCCEquivalent(N1, RL, RR, N1CC))
    3639      316450 :     return SDValue();
    3640             : 
    3641             :   assert(N0.getValueType() == N1.getValueType() &&
    3642             :          "Unexpected operand types for bitwise logic op");
    3643             :   assert(LL.getValueType() == LR.getValueType() &&
    3644             :          RL.getValueType() == RR.getValueType() &&
    3645             :          "Unexpected operand types for setcc");
    3646             : 
    3647             :   // If we're here post-legalization or the logic op type is not i1, the logic
    3648             :   // op type must match a setcc result type. Also, all folds require new
    3649             :   // operations on the left and right operands, so those types must match.
    3650        2181 :   EVT VT = N0.getValueType();
    3651        4362 :   EVT OpVT = LL.getValueType();
    3652        3310 :   if (LegalOperations || VT.getScalarType() != MVT::i1)
    3653        1052 :     if (VT != getSetCCResultType(OpVT))
    3654           0 :       return SDValue();
    3655        4370 :   if (OpVT != RL.getValueType())
    3656          93 :     return SDValue();
    3657             : 
    3658        2088 :   ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
    3659        2088 :   ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
    3660        2088 :   bool IsInteger = OpVT.isInteger();
    3661         438 :   if (LR == RR && CC0 == CC1 && IsInteger) {
    3662         220 :     bool IsZero = isNullConstantOrNullSplatConstant(LR);
    3663         220 :     bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
    3664             : 
    3665             :     // All bits clear?
    3666         220 :     bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
    3667             :     // All sign bits clear?
    3668         220 :     bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
    3669             :     // Any bits set?
    3670         220 :     bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
    3671             :     // Any sign bits set?
    3672         220 :     bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
    3673             : 
    3674             :     // (and (seteq X,  0), (seteq Y,  0)) --> (seteq (or X, Y),  0)
    3675             :     // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
    3676             :     // (or  (setne X,  0), (setne Y,  0)) --> (setne (or X, Y),  0)
    3677             :     // (or  (setlt X,  0), (setlt Y,  0)) --> (setlt (or X, Y),  0)
    3678         220 :     if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
    3679         204 :       SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
    3680          68 :       AddToWorklist(Or.getNode());
    3681          68 :       return DAG.getSetCC(DL, VT, Or, LR, CC1);
    3682             :     }
    3683             : 
    3684             :     // All bits set?
    3685         152 :     bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
    3686             :     // All sign bits set?
    3687         152 :     bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
    3688             :     // Any bits clear?
    3689         152 :     bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
    3690             :     // Any sign bits clear?
    3691         152 :     bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
    3692             : 
    3693             :     // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
    3694             :     // (and (setlt X,  0), (setlt Y,  0)) --> (setlt (and X, Y),  0)
    3695             :     // (or  (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
    3696             :     // (or  (setgt X, -1), (setgt Y  -1)) --> (setgt (and X, Y), -1)
    3697         152 :     if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
    3698          66 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
    3699          22 :       AddToWorklist(And.getNode());
    3700          22 :       return DAG.getSetCC(DL, VT, And, LR, CC1);
    3701             :     }
    3702             :   }
    3703             : 
    3704             :   // TODO: What is the 'or' equivalent of this fold?
    3705             :   // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
    3706         262 :   if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
    3707        2075 :       IsInteger && CC0 == ISD::SETNE &&
    3708          62 :       ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
    3709          34 :        (isAllOnesConstant(LR) && isNullConstant(RR)))) {
    3710           3 :     SDValue One = DAG.getConstant(1, DL, OpVT);
    3711           3 :     SDValue Two = DAG.getConstant(2, DL, OpVT);
    3712           9 :     SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
    3713           3 :     AddToWorklist(Add.getNode());
    3714           3 :     return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
    3715             :   }
    3716             : 
    3717             :   // Try more general transforms if the predicates match and the only user of
    3718             :   // the compares is the 'and' or 'or'.
    3719        1564 :   if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
    3720        2215 :       N0.hasOneUse() && N1.hasOneUse()) {
    3721             :     // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
    3722             :     // or  (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
    3723         220 :     if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
    3724         231 :       SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
    3725         231 :       SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
    3726         154 :       SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
    3727          77 :       SDValue Zero = DAG.getConstant(0, DL, OpVT);
    3728          77 :       return DAG.getSetCC(DL, VT, Or, Zero, CC1);
    3729             :     }
    3730             :   }
    3731             : 
    3732             :   // Canonicalize equivalent operands to LL == RL.
    3733             :   if (LL == RR && LR == RL) {
    3734           0 :     CC1 = ISD::getSetCCSwappedOperands(CC1);
    3735             :     std::swap(RL, RR);
    3736             :   }
    3737             : 
    3738             :   // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
    3739             :   // (or  (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
    3740             :   if (LL == RL && LR == RR) {
    3741         101 :     ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
    3742          31 :                                 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
    3743         135 :     if (NewCC != ISD::SETCC_INVALID &&
    3744          98 :         (!LegalOperations ||
    3745          33 :          (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
    3746           0 :           TLI.isOperationLegal(ISD::SETCC, OpVT))))
    3747          32 :       return DAG.getSetCC(DL, VT, LL, LR, NewCC);
    3748             :   }
    3749             : 
    3750        1886 :   return SDValue();
    3751             : }
    3752             : 
    3753             : /// This contains all DAGCombine rules which reduce two values combined by
    3754             : /// an And operation to a single value. This makes them reusable in the context
    3755             : /// of visitSELECT(). Rules involving constants are not included as
    3756             : /// visitSELECT() already handles those cases.
    3757      197442 : SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
    3758      394884 :   EVT VT = N1.getValueType();
    3759             :   SDLoc DL(N);
    3760             : 
    3761             :   // fold (and x, undef) -> 0
    3762      394883 :   if (N0.isUndef() || N1.isUndef())
    3763           2 :     return DAG.getConstant(0, DL, VT);
    3764             : 
    3765      197440 :   if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
    3766          88 :     return V;
    3767             : 
    3768      202475 :   if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
    3769          78 :       VT.getSizeInBits() <= 64) {
    3770             :     if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    3771             :       if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
    3772             :         // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
    3773             :         // immediate for an add, but it is legal if its top c2 bits are set,
    3774             :         // transform the ADD so the immediate doesn't need to be materialized
    3775             :         // in a register.
    3776          54 :         APInt ADDC = ADDI->getAPIntValue();
    3777          54 :         APInt SRLC = SRLI->getAPIntValue();
    3778         108 :         if (ADDC.getMinSignedBits() <= 64 &&
    3779         107 :             SRLC.ult(VT.getSizeInBits()) &&
    3780         106 :             !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
    3781             :           APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
    3782          12 :                                              SRLC.getZExtValue());
    3783          24 :           if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
    3784             :             ADDC |= Mask;
    3785          24 :             if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
    3786             :               SDLoc DL0(N0);
    3787             :               SDValue NewAdd =
    3788          12 :                 DAG.getNode(ISD::ADD, DL0, VT,
    3789          24 :                             N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
    3790             :               CombineTo(N0.getNode(), NewAdd);
    3791             :               // Return N so it doesn't get rechecked!
    3792          12 :               return SDValue(N, 0);
    3793             :             }
    3794             :           }
    3795             :         }
    3796             :       }
    3797             :     }
    3798             :   }
    3799             : 
    3800             :   // Reduce bit extract of low half of an integer to the narrower type.
    3801             :   // (and (srl i64:x, K), KMask) ->
    3802             :   //   (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
    3803      219110 :   if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    3804             :     if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
    3805             :       if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    3806       13613 :         unsigned Size = VT.getSizeInBits();
    3807       13613 :         const APInt &AndMask = CAnd->getAPIntValue();
    3808       27226 :         unsigned ShiftBits = CShift->getZExtValue();
    3809             : 
    3810             :         // Bail out, this node will probably disappear anyway.
    3811       13613 :         if (ShiftBits == 0)
    3812           2 :           return SDValue();
    3813             : 
    3814             :         unsigned MaskBits = AndMask.countTrailingOnes();
    3815       13611 :         EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
    3816             : 
    3817       25790 :         if (AndMask.isMask() &&
    3818             :             // Required bits must not span the two halves of the integer and
    3819             :             // must fit in the half size type.
    3820       19395 :             (ShiftBits + MaskBits <= Size / 2) &&
    3821        7325 :             TLI.isNarrowingProfitable(VT, HalfVT) &&
    3822         206 :             TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
    3823         194 :             TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
    3824       13805 :             TLI.isTruncateFree(VT, HalfVT) &&
    3825          97 :             TLI.isZExtFree(HalfVT, VT)) {
    3826             :           // The isNarrowingProfitable is to avoid regressions on PPC and
    3827             :           // AArch64 which match a few 64-bit bit insert / bit extract patterns
    3828             :           // on downstream users of this. Those patterns could probably be
    3829             :           // extended to handle extensions mixed in.
    3830             : 
    3831             :           SDValue SL(N0);
    3832             :           assert(MaskBits <= Size);
    3833             : 
    3834             :           // Extracting the highest bit of the low half.
    3835         190 :           EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
    3836          95 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
    3837         190 :                                       N0.getOperand(0));
    3838             : 
    3839         380 :           SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
    3840         285 :           SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
    3841         285 :           SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
    3842         285 :           SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
    3843         285 :           return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
    3844             :         }
    3845             :       }
    3846             :     }
    3847             :   }
    3848             : 
    3849      197243 :   return SDValue();
    3850             : }
    3851             : 
    3852        1446 : bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
    3853             :                                    EVT LoadResultTy, EVT &ExtVT) {
    3854        2892 :   if (!AndC->getAPIntValue().isMask())
    3855             :     return false;
    3856             : 
    3857             :   unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
    3858             : 
    3859        1446 :   ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    3860        1446 :   EVT LoadedVT = LoadN->getMemoryVT();
    3861             : 
    3862          16 :   if (ExtVT == LoadedVT &&
    3863          17 :       (!LegalOperations ||
    3864           1 :        TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
    3865             :     // ZEXTLOAD will match without needing to change the size of the value being
    3866             :     // loaded.
    3867             :     return true;
    3868             :   }
    3869             : 
    3870             :   // Do not change the width of a volatile load.
    3871        1430 :   if (LoadN->isVolatile())
    3872             :     return false;
    3873             : 
    3874             :   // Do not generate loads of non-round integer types since these can
    3875             :   // be expensive (and would be wrong if the type is not byte sized).
    3876        1423 :   if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
    3877             :     return false;
    3878             : 
    3879        1019 :   if (LegalOperations &&
    3880         457 :       !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
    3881             :     return false;
    3882             : 
    3883         147 :   if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
    3884             :     return false;
    3885             : 
    3886             :   return true;
    3887             : }
    3888             : 
    3889       34144 : bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
    3890             :                                     EVT &ExtVT, unsigned ShAmt) {
    3891             :   // Don't transform one with multiple uses, this would require adding a new
    3892             :   // load.
    3893       34144 :   if (!SDValue(LoadN, 0).hasOneUse())
    3894             :     return false;
    3895             : 
    3896       21744 :   if (LegalOperations &&
    3897       18274 :       !TLI.isLoadExtLegal(ExtType, LoadN->getValueType(0), ExtVT))
    3898             :     return false;
    3899             : 
    3900             :   // Do not generate loads of non-round integer types since these can
    3901             :   // be expensive (and would be wrong if the type is not byte sized).
    3902             :   if (!ExtVT.isRound())
    3903             :     return false;
    3904             : 
    3905             :   // Don't change the width of a volatile load.
    3906        3938 :   if (LoadN->isVolatile())
    3907             :     return false;
    3908             : 
    3909             :   // Verify that we are actually reducing a load width here.
    3910        3776 :   if (LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits())
    3911             :     return false;
    3912             : 
    3913             :   // For the transform to be legal, the load must produce only two values
    3914             :   // (the value loaded and the chain).  Don't transform a pre-increment
    3915             :   // load, for example, which produces an extra value.  Otherwise the
    3916             :   // transformation is not equivalent, and the downstream logic to replace
    3917             :   // uses gets things wrong.
    3918        3609 :   if (LoadN->getNumValues() > 2)
    3919             :     return false;
    3920             : 
    3921             :  // Only allow byte offsets.
    3922        3609 :   if (ShAmt % 8)
    3923             :     return false;
    3924             : 
    3925             :   // Ensure that this isn't going to produce an unsupported unaligned access.
    3926        5394 :   if (ShAmt && !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
    3927             :                                        ExtVT, LoadN->getAddressSpace(),
    3928             :                                        ShAmt / 8))
    3929             :     return false;
    3930             : 
    3931             : 
    3932             :   // If the load that we're shrinking is an extload and we're not just
    3933             :   // discarding the extension we can't simply shrink the load. Bail.
    3934             :   // TODO: It would be possible to merge the extensions in some cases.
    3935        4261 :   if (LoadN->getExtensionType() != ISD::NON_EXTLOAD &&
    3936        4261 :       LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt)
    3937             :     return false;
    3938             : 
    3939        3555 :   if (!TLI.shouldReduceLoadWidth(LoadN, ExtType, ExtVT))
    3940             :     return false;
    3941             : 
    3942             :   // It's not possible to generate a constant of extended or untyped type.
    3943        1528 :   EVT PtrType = LoadN->getOperand(1).getValueType();
    3944        1528 :   if (PtrType == MVT::Untyped || PtrType.isExtended())
    3945             :     return false;
    3946             : 
    3947             :   return true;
    3948             : }
    3949             : 
    3950      109487 : bool DAGCombiner::SearchForAndLoads(SDNode *N,
    3951             :                                     SmallPtrSetImpl<LoadSDNode*> &Loads,
    3952             :                                     SmallPtrSetImpl<SDNode*> &NodesWithConsts,
    3953             :                                     ConstantSDNode *Mask,
    3954             :                                     SDNode *&NodeToMask) {
    3955             :   // Recursively search for the operands, looking for loads which can be
    3956             :   // narrowed.
    3957      515570 :   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
    3958      367832 :     SDValue Op = N->getOperand(i);
    3959             : 
    3960      367832 :     if (Op.getValueType().isVector())
    3961             :       return false;
    3962             : 
    3963             :     // Some constants may need fixing up later if they are too large.
    3964       73809 :     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
    3965      222202 :       if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
    3966      222977 :           (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
    3967          94 :         NodesWithConsts.insert(N);
    3968       73809 :       continue;
    3969             :     }
    3970             : 
    3971      110107 :     if (!Op.hasOneUse())
    3972             :       return false;
    3973             : 
    3974       80427 :     switch(Op.getOpcode()) {
    3975             :     case ISD::LOAD: {
    3976             :       auto *Load = cast<LoadSDNode>(Op);
    3977        1445 :       EVT ExtVT;
    3978        3197 :       if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
    3979         159 :           isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) {
    3980             : 
    3981             :         // ZEXTLOAD is already small enough.
    3982         179 :         if (Load->getExtensionType() == ISD::ZEXTLOAD &&
    3983          13 :             ExtVT.bitsGE(Load->getMemoryVT()))
    3984         166 :           continue;
    3985             : 
    3986             :         // Use LE to convert equal sized loads to zext.
    3987         148 :         if (ExtVT.bitsLE(Load->getMemoryVT()))
    3988         148 :           Loads.insert(Load);
    3989             : 
    3990         148 :         continue;
    3991             :       }
    3992        1288 :       return false;
    3993             :     }
    3994        1256 :     case ISD::ZERO_EXTEND:
    3995             :     case ISD::AssertZext: {
    3996        1256 :       unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
    3997        1256 :       EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    3998             :       EVT VT = Op.getOpcode() == ISD::AssertZext ?
    3999             :         cast<VTSDNode>(Op.getOperand(1))->getVT() :
    4000        3073 :         Op.getOperand(0).getValueType();
    4001             : 
    4002             :       // We can accept extending nodes if the mask is wider or an equal
    4003             :       // width to the original type.
    4004        1256 :       if (ExtVT.bitsGE(VT))
    4005         766 :         continue;
    4006         490 :       break;
    4007             :     }
    4008        4632 :     case ISD::OR:
    4009             :     case ISD::XOR:
    4010             :     case ISD::AND:
    4011        4632 :       if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
    4012             :                              NodeToMask))
    4013             :         return false;
    4014        1050 :       continue;
    4015             :     }
    4016             : 
    4017             :     // Allow one node which will masked along with any loads found.
    4018       73584 :     if (NodeToMask)
    4019             :       return false;
    4020             :  
    4021             :     // Also ensure that the node to be masked only produces one data result. 
    4022       73171 :     NodeToMask = Op.getNode();
    4023      146342 :     if (NodeToMask->getNumValues() > 1) {
    4024             :       bool HasValue = false;
    4025       21206 :       for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
    4026        9103 :         MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
    4027        9103 :         if (VT != MVT::Glue && VT != MVT::Other) {
    4028        4965 :           if (HasValue) {
    4029         655 :             NodeToMask = nullptr;
    4030             :             return false;
    4031             :           }
    4032             :           HasValue = true;
    4033             :         }
    4034             :       }
    4035             :       assert(HasValue && "Node to be masked has no data result?");
    4036             :     }
    4037             :   }
    4038             :   return true;
    4039             : }
    4040             : 
    4041      177631 : bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
    4042      177631 :   auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
    4043             :   if (!Mask)
    4044             :     return false;
    4045             : 
    4046      270630 :   if (!Mask->getAPIntValue().isMask())
    4047             :     return false;
    4048             : 
    4049             :   // No need to do anything if the and directly uses a load.
    4050      116580 :   if (isa<LoadSDNode>(N->getOperand(0)))
    4051             :     return false;
    4052             : 
    4053             :   SmallPtrSet<LoadSDNode*, 8> Loads;
    4054             :   SmallPtrSet<SDNode*, 2> NodesWithConsts;
    4055      104855 :   SDNode *FixupNode = nullptr;
    4056      104855 :   if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
    4057       72819 :     if (Loads.size() == 0)
    4058             :       return false;
    4059             : 
    4060             :     LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump());
    4061          87 :     SDValue MaskOp = N->getOperand(1);
    4062             : 
    4063             :     // If it exists, fixup the single node we allow in the tree that needs
    4064             :     // masking.
    4065          87 :     if (FixupNode) {
    4066             :       LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump());
    4067          25 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
    4068             :                                 FixupNode->getValueType(0),
    4069          75 :                                 SDValue(FixupNode, 0), MaskOp);
    4070          50 :       DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
    4071          50 :       DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0),
    4072             :                              MaskOp);
    4073             :     }
    4074             : 
    4075             :     // Narrow any constants that need it.
    4076          87 :     for (auto *LogicN : NodesWithConsts) {
    4077          14 :       SDValue Op0 = LogicN->getOperand(0);
    4078          14 :       SDValue Op1 = LogicN->getOperand(1);
    4079             : 
    4080             :       if (isa<ConstantSDNode>(Op0))
    4081             :           std::swap(Op0, Op1);
    4082             : 
    4083          14 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
    4084          28 :                                 Op1, MaskOp);
    4085             : 
    4086          14 :       DAG.UpdateNodeOperands(LogicN, Op0, And);
    4087             :     }
    4088             : 
    4089             :     // Create narrow loads.
    4090          87 :     for (auto *Load : Loads) {
    4091             :       LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump());
    4092         143 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
    4093         429 :                                 SDValue(Load, 0), MaskOp);
    4094         143 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
    4095         143 :       DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp);
    4096         143 :       SDValue NewLoad = ReduceLoadWidth(And.getNode());
    4097             :       assert(NewLoad &&
    4098             :              "Shouldn't be masking the load if it can't be narrowed");
    4099             :       CombineTo(Load, NewLoad, NewLoad.getValue(1));
    4100             :     }
    4101          87 :     DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
    4102          87 :     return true;
    4103             :   }
    4104             :   return false;
    4105             : }
    4106             : 
    4107      233261 : SDValue DAGCombiner::visitAND(SDNode *N) {
    4108      233261 :   SDValue N0 = N->getOperand(0);
    4109      233261 :   SDValue N1 = N->getOperand(1);
    4110      466522 :   EVT VT = N1.getValueType();
    4111             : 
    4112             :   // x & x --> x
    4113             :   if (N0 == N1)
    4114          21 :     return N0;
    4115             : 
    4116             :   // fold vector ops
    4117      233240 :   if (VT.isVector()) {
    4118       41829 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    4119           4 :       return FoldedVOp;
    4120             : 
    4121             :     // fold (and x, 0) -> 0, vector edition
    4122       41825 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    4123             :       // do not return N0, because undef node may exist in N0
    4124         180 :       return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
    4125         180 :                              SDLoc(N), N0.getValueType());
    4126       41765 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    4127             :       // do not return N1, because undef node may exist in N1
    4128          15 :       return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
    4129          15 :                              SDLoc(N), N1.getValueType());
    4130             : 
    4131             :     // fold (and x, -1) -> x, vector edition
    4132       41760 :     if (ISD::isBuildVectorAllOnes(N0.getNode()))
    4133          50 :       return N1;
    4134       41710 :     if (ISD::isBuildVectorAllOnes(N1.getNode()))
    4135           7 :       return N0;
    4136             :   }
    4137             : 
    4138             :   // fold (and c1, c2) -> c1&c2
    4139      233114 :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    4140      233114 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    4141      233140 :   if (N0C && N1C && !N1C->isOpaque())
    4142          78 :     return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
    4143             :   // canonicalize constant to RHS
    4144      233628 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    4145         540 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    4146        1596 :     return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
    4147             :   // fold (and x, -1) -> x
    4148      232556 :   if (isAllOnesConstant(N1))
    4149          10 :     return N0;
    4150             :   // if (and x, c) is known to be zero, return 0
    4151             :   unsigned BitWidth = VT.getScalarSizeInBits();
    4152      646453 :   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
    4153      646395 :                                    APInt::getAllOnesValue(BitWidth)))
    4154         174 :     return DAG.getConstant(0, SDLoc(N), VT);
    4155             : 
    4156      232488 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    4157          27 :     return NewSel;
    4158             : 
    4159             :   // reassociate and
    4160      464922 :   if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
    4161        1998 :     return RAND;
    4162             : 
    4163             :   // Try to convert a constant mask AND into a shuffle clear mask.
    4164      230463 :   if (VT.isVector())
    4165       41117 :     if (SDValue Shuffle = XformToShuffleWithZero(N))
    4166        1014 :       return Shuffle;
    4167             : 
    4168             :   // fold (and (or x, C), D) -> D if (C & D) == D
    4169        5837 :   auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    4170       11674 :     return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
    4171        5837 :   };
    4172      699245 :   if (N0.getOpcode() == ISD::OR &&
    4173      467510 :       ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
    4174        2286 :     return N1;
    4175             :   // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
    4176      403363 :   if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    4177       36609 :     SDValue N0Op0 = N0.getOperand(0);
    4178       73218 :     APInt Mask = ~N1C->getAPIntValue();
    4179       73218 :     Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
    4180       36609 :     if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
    4181       55356 :       SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
    4182       83034 :                                  N0.getValueType(), N0Op0);
    4183             : 
    4184             :       // Replace uses of the AND with uses of the Zero extend node.
    4185             :       CombineTo(N, Zext);
    4186             : 
    4187             :       // We actually want to replace all uses of the any_extend with the
    4188             :       // zero_extend, to avoid duplicating things.  This will later cause this
    4189             :       // AND to be folded.
    4190       27678 :       CombineTo(N0.getNode(), Zext);
    4191       27678 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4192             :     }
    4193             :   }
    4194             :   // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
    4195             :   // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
    4196             :   // already be zero by virtue of the width of the base type of the load.
    4197             :   //
    4198             :   // the 'X' node here can either be nothing or an extract_vector_elt to catch
    4199             :   // more cases.
    4200      208497 :   if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
    4201       25986 :        N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
    4202       19086 :        N0.getOperand(0).getOpcode() == ISD::LOAD &&
    4203      398970 :        N0.getOperand(0).getResNo() == 0) ||
    4204      222238 :       (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
    4205       58152 :     LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
    4206             :                                          N0 : N0.getOperand(0) );
    4207             : 
    4208             :     // Get the constant (if applicable) the zero'th operand is being ANDed with.
    4209             :     // This can be a pure constant or a vector splat, in which case we treat the
    4210             :     // vector as a scalar and use the splat value.
    4211             :     APInt Constant = APInt::getNullValue(1);
    4212             :     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
    4213       46852 :       Constant = C->getAPIntValue();
    4214             :     } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
    4215             :       APInt SplatValue, SplatUndef;
    4216             :       unsigned SplatBitSize;
    4217             :       bool HasAnyUndefs;
    4218             :       bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
    4219        1065 :                                              SplatBitSize, HasAnyUndefs);
    4220        1065 :       if (IsSplat) {
    4221             :         // Undef bits can contribute to a possible optimisation if set, so
    4222             :         // set them.
    4223             :         SplatValue |= SplatUndef;
    4224             : 
    4225             :         // The splat value may be something like "0x00FFFFFF", which means 0 for
    4226             :         // the first vector value and FF for the rest, repeating. We need a mask
    4227             :         // that will apply equally to all members of the vector, so AND all the
    4228             :         // lanes of the constant together.
    4229        2128 :         EVT VT = Vector->getValueType(0);
    4230             :         unsigned BitWidth = VT.getScalarSizeInBits();
    4231             : 
    4232             :         // If the splat value has been compressed to a bitlength lower
    4233             :         // than the size of the vector lane, we need to re-expand it to
    4234             :         // the lane size.
    4235        1064 :         if (BitWidth > SplatBitSize)
    4236           3 :           for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
    4237           2 :                SplatBitSize < BitWidth;
    4238           1 :                SplatBitSize = SplatBitSize * 2)
    4239           2 :             SplatValue |= SplatValue.shl(SplatBitSize);
    4240             : 
    4241             :         // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
    4242             :         // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
    4243        1064 :         if (SplatBitSize % BitWidth == 0) {
    4244        2126 :           Constant = APInt::getAllOnesValue(BitWidth);
    4245        2171 :           for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
    4246        3324 :             Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
    4247             :         }
    4248             :       }
    4249             :     }
    4250             : 
    4251             :     // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
    4252             :     // actually legal and isn't going to get expanded, else this is a false
    4253             :     // optimisation.
    4254       29076 :     bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
    4255             :                                                     Load->getValueType(0),
    4256             :                                                     Load->getMemoryVT());
    4257             : 
    4258             :     // Resize the constant to the same size as the original memory access before
    4259             :     // extension. If it is still the AllOnesValue then this AND is completely
    4260             :     // unneeded.
    4261       87228 :     Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
    4262             : 
    4263             :     bool B;
    4264       29076 :     switch (Load->getExtensionType()) {
    4265             :     default: B = false; break;
    4266             :     case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
    4267             :     case ISD::ZEXTLOAD:
    4268             :     case ISD::NON_EXTLOAD: B = true; break;
    4269             :     }
    4270             : 
    4271       30892 :     if (B && Constant.isAllOnesValue()) {
    4272             :       // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
    4273             :       // preserve semantics once we get rid of the AND.
    4274             :       SDValue NewLoad(Load, 0);
    4275             : 
    4276             :       // Fold the AND away. NewLoad may get replaced immediately.
    4277        1967 :       CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
    4278             : 
    4279        1967 :       if (Load->getExtensionType() == ISD::EXTLOAD) {
    4280        2330 :         NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
    4281        1165 :                               Load->getValueType(0), SDLoc(Load),
    4282             :                               Load->getChain(), Load->getBasePtr(),
    4283             :                               Load->getOffset(), Load->getMemoryVT(),
    4284        2330 :                               Load->getMemOperand());
    4285             :         // Replace uses of the EXTLOAD with the new ZEXTLOAD.
    4286        1165 :         if (Load->getNumValues() == 3) {
    4287             :           // PRE/POST_INC loads have 3 values.
    4288             :           SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
    4289           0 :                            NewLoad.getValue(2) };
    4290           0 :           CombineTo(Load, To, 3, true);
    4291             :         } else {
    4292        1165 :           CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
    4293             :         }
    4294             :       }
    4295             : 
    4296        1967 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    4297             :     }
    4298             :   }
    4299             : 
    4300             :   // fold (and (load x), 255) -> (zextload x, i8)
    4301             :   // fold (and (extload x, i16), 255) -> (zextload x, i8)
    4302             :   // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
    4303      342383 :   if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
    4304        8910 :                                 (N0.getOpcode() == ISD::ANY_EXTEND &&
    4305        8910 :                                  N0.getOperand(0).getOpcode() == ISD::LOAD))) {
    4306       19520 :     if (SDValue Res = ReduceLoadWidth(N)) {
    4307          44 :       LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
    4308          44 :         ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
    4309             : 
    4310          44 :       AddToWorklist(N);
    4311             :       CombineTo(LN0, Res, Res.getValue(1));
    4312          44 :       return SDValue(N, 0);
    4313             :     }
    4314             :   }
    4315             : 
    4316      197474 :   if (Level >= AfterLegalizeTypes) {
    4317             :     // Attempt to propagate the AND back up to the leaves which, if they're
    4318             :     // loads, can be combined to narrow loads and the AND node can be removed.
    4319             :     // Perform after legalization so that extend nodes will already be
    4320             :     // combined into the loads.
    4321      177631 :     if (BackwardsPropagateMask(N, DAG)) {
    4322          87 :       return SDValue(N, 0);
    4323             :     }
    4324             :   }
    4325             : 
    4326      197387 :   if (SDValue Combined = visitANDLike(N0, N1, N))
    4327         193 :     return Combined;
    4328             : 
    4329             :   // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
    4330      591582 :   if (N0.getOpcode() == N1.getOpcode())
    4331       20562 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    4332         291 :       return Tmp;
    4333             : 
    4334             :   // Masking the negated extension of a boolean is just the zero-extended
    4335             :   // boolean:
    4336             :   // and (sub 0, zext(bool X)), 1 --> zext(bool X)
    4337             :   // and (sub 0, sext(bool X)), 1 --> zext(bool X)
    4338             :   //
    4339             :   // Note: the SimplifyDemandedBits fold below can make an information-losing
    4340             :   // transform, and then we have no way to find this better fold.
    4341      411586 :   if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
    4342          27 :     if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
    4343          12 :       SDValue SubRHS = N0.getOperand(1);
    4344           9 :       if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
    4345           3 :           SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
    4346           3 :         return SubRHS;
    4347           6 :       if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
    4348           3 :           SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
    4349           9 :         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
    4350             :     }
    4351             :   }
    4352             : 
    4353             :   // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
    4354             :   // fold (and (sra)) -> (and (srl)) when possible.
    4355      196897 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    4356       53874 :     return SDValue(N, 0);
    4357             : 
    4358             :   // fold (zext_inreg (extload x)) -> (zextload x)
    4359      143023 :   if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
    4360             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    4361        3385 :     EVT MemVT = LN0->getMemoryVT();
    4362             :     // If we zero all the possible extended bits, then we can turn this into
    4363             :     // a zextload if we are running before legalize or the operation is legal.
    4364        3385 :     unsigned BitWidth = N1.getScalarValueSizeInBits();
    4365       10155 :     if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
    4366        6701 :                            BitWidth - MemVT.getScalarSizeInBits())) &&
    4367        6635 :         ((!LegalOperations && !LN0->isVolatile()) ||
    4368        2533 :          TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
    4369        2882 :       SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
    4370             :                                        LN0->getChain(), LN0->getBasePtr(),
    4371        2882 :                                        MemVT, LN0->getMemOperand());
    4372        1441 :       AddToWorklist(N);
    4373        1441 :       CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    4374        1441 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4375             :     }
    4376             :   }
    4377             :   // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
    4378      141869 :   if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
    4379         287 :       N0.hasOneUse()) {
    4380             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    4381         250 :     EVT MemVT = LN0->getMemoryVT();
    4382             :     // If we zero all the possible extended bits, then we can turn this into
    4383             :     // a zextload if we are running before legalize or the operation is legal.
    4384         250 :     unsigned BitWidth = N1.getScalarValueSizeInBits();
    4385         750 :     if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
    4386         487 :                            BitWidth - MemVT.getScalarSizeInBits())) &&
    4387         474 :         ((!LegalOperations && !LN0->isVolatile()) ||
    4388         227 :          TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
    4389         442 :       SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
    4390             :                                        LN0->getChain(), LN0->getBasePtr(),
    4391         442 :                                        MemVT, LN0->getMemOperand());
    4392         221 :       AddToWorklist(N);
    4393         221 :       CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    4394         221 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4395             :     }
    4396             :   }
    4397             :   // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
    4398      251657 :   if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
    4399         243 :     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
    4400         243 :                                            N0.getOperand(1), false))
    4401           1 :       return BSwap;
    4402             :   }
    4403             : 
    4404      141360 :   return SDValue();
    4405             : }
    4406             : 
    4407             : /// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
    4408      120973 : SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
    4409             :                                         bool DemandHighBits) {
    4410      120973 :   if (!LegalOperations)
    4411       51728 :     return SDValue();
    4412             : 
    4413      138490 :   EVT VT = N->getValueType(0);
    4414             :   if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
    4415       14798 :     return SDValue();
    4416       54447 :   if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    4417       13863 :     return SDValue();
    4418             : 
    4419             :   // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
    4420             :   bool LookPassAnd0 = false;
    4421             :   bool LookPassAnd1 = false;
    4422       46720 :   if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
    4423             :       std::swap(N0, N1);
    4424       44562 :   if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
    4425             :       std::swap(N0, N1);
    4426       40584 :   if (N0.getOpcode() == ISD::AND) {
    4427             :     if (!N0.getNode()->hasOneUse())
    4428         172 :       return SDValue();
    4429             :     ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4430             :     // Also handle 0xffff since the LHS is guaranteed to have zeros there.
    4431             :     // This is needed for X86.
    4432        5098 :     if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
    4433             :                   N01C->getZExtValue() != 0xFFFF))
    4434        1744 :       return SDValue();
    4435         962 :     N0 = N0.getOperand(0);
    4436             :     LookPassAnd0 = true;
    4437             :   }
    4438             : 
    4439       38668 :   if (N1.getOpcode() == ISD::AND) {
    4440             :     if (!N1.getNode()->hasOneUse())
    4441          19 :       return SDValue();
    4442             :     ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    4443        2576 :     if (!N11C || N11C->getZExtValue() != 0xFF)
    4444        1177 :       return SDValue();
    4445         125 :     N1 = N1.getOperand(0);
    4446             :     LookPassAnd1 = true;
    4447             :   }
    4448             : 
    4449       38032 :   if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
    4450             :     std::swap(N0, N1);
    4451       40213 :   if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
    4452       36619 :     return SDValue();
    4453             :   if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
    4454           8 :     return SDValue();
    4455             : 
    4456             :   ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4457             :   ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    4458         845 :   if (!N01C || !N11C)
    4459         236 :     return SDValue();
    4460        1254 :   if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
    4461         589 :     return SDValue();
    4462             : 
    4463             :   // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
    4464          20 :   SDValue N00 = N0->getOperand(0);
    4465          37 :   if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
    4466             :     if (!N00.getNode()->hasOneUse())
    4467           0 :       return SDValue();
    4468             :     ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
    4469          12 :     if (!N001C || N001C->getZExtValue() != 0xFF)
    4470           2 :       return SDValue();
    4471           4 :     N00 = N00.getOperand(0);
    4472             :     LookPassAnd0 = true;
    4473             :   }
    4474             : 
    4475          18 :   SDValue N10 = N1->getOperand(0);
    4476          30 :   if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
    4477             :     if (!N10.getNode()->hasOneUse())
    4478           0 :       return SDValue();
    4479             :     ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
    4480             :     // Also allow 0xFFFF since the bits will be shifted out. This is needed
    4481             :     // for X86.
    4482          10 :     if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
    4483             :                    N101C->getZExtValue() != 0xFFFF))
    4484           0 :       return SDValue();
    4485           5 :     N10 = N10.getOperand(0);
    4486             :     LookPassAnd1 = true;
    4487             :   }
    4488             : 
    4489             :   if (N00 != N10)
    4490           0 :     return SDValue();
    4491             : 
    4492             :   // Make sure everything beyond the low halfword gets set to zero since the SRL
    4493             :   // 16 will clear the top bits.
    4494          18 :   unsigned OpSizeInBits = VT.getSizeInBits();
    4495          18 :   if (DemandHighBits && OpSizeInBits > 16) {
    4496             :     // If the left-shift isn't masked out then the only way this is a bswap is
    4497             :     // if all bits beyond the low 8 are 0. In that case the entire pattern
    4498             :     // reduces to a left shift anyway: leave it for other parts of the combiner.
    4499           9 :     if (!LookPassAnd0)
    4500           2 :       return SDValue();
    4501             : 
    4502             :     // However, if the right shift isn't masked out then it might be because
    4503             :     // it's not needed. See if we can spot that too.
    4504          16 :     if (!LookPassAnd1 &&
    4505           4 :         !DAG.MaskedValueIsZero(
    4506          16 :             N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
    4507           0 :       return SDValue();
    4508             :   }
    4509             : 
    4510          48 :   SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
    4511          16 :   if (OpSizeInBits > 16) {
    4512             :     SDLoc DL(N);
    4513          32 :     Res = DAG.getNode(ISD::SRL, DL, VT, Res,
    4514          16 :                       DAG.getConstant(OpSizeInBits - 16, DL,
    4515          48 :                                       getShiftAmountTy(VT)));
    4516             :   }
    4517          16 :   return Res;
    4518             : }
    4519             : 
    4520             : /// Return true if the specified node is an element that makes up a 32-bit
    4521             : /// packed halfword byteswap.
    4522             : /// ((x & 0x000000ff) << 8) |
    4523             : /// ((x & 0x0000ff00) >> 8) |
    4524             : /// ((x & 0x00ff0000) << 8) |
    4525             : /// ((x & 0xff000000) >> 8)
    4526        3363 : static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
    4527             :   if (!N.getNode()->hasOneUse())
    4528             :     return false;
    4529             : 
    4530             :   unsigned Opc = N.getOpcode();
    4531        3291 :   if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
    4532             :     return false;
    4533             : 
    4534        2867 :   SDValue N0 = N.getOperand(0);
    4535             :   unsigned Opc0 = N0.getOpcode();
    4536        2867 :   if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
    4537             :     return false;
    4538             : 
    4539             :   ConstantSDNode *N1C = nullptr;
    4540             :   // SHL or SRL: look upstream for AND mask operand
    4541        2525 :   if (Opc == ISD::AND)
    4542             :     N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4543        2495 :   else if (Opc0 == ISD::AND)
    4544             :     N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4545        2524 :   if (!N1C)
    4546             :     return false;
    4547             : 
    4548             :   unsigned MaskByteOffset;
    4549        5048 :   switch (N1C->getZExtValue()) {
    4550             :   default:
    4551             :     return false;
    4552             :   case 0xFF:       MaskByteOffset = 0; break;
    4553           4 :   case 0xFF00:     MaskByteOffset = 1; break;
    4554           6 :   case 0xFFFF:
    4555             :     // In case demanded bits didn't clear the bits that will be shifted out.
    4556             :     // This is needed for X86.
    4557           6 :     if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
    4558             :       MaskByteOffset = 1;
    4559             :       break;
    4560             :     }
    4561             :     return false;
    4562          10 :   case 0xFF0000:   MaskByteOffset = 2; break;
    4563           8 :   case 0xFF000000: MaskByteOffset = 3; break;
    4564             :   }
    4565             : 
    4566             :   // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
    4567          38 :   if (Opc == ISD::AND) {
    4568          24 :     if (MaskByteOffset == 0 || MaskByteOffset == 2) {
    4569             :       // (x >> 8) & 0xff
    4570             :       // (x >> 8) & 0xff0000
    4571          12 :       if (Opc0 != ISD::SRL)
    4572             :         return false;
    4573             :       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4574          24 :       if (!C || C->getZExtValue() != 8)
    4575             :         return false;
    4576             :     } else {
    4577             :       // (x << 8) & 0xff00
    4578             :       // (x << 8) & 0xff000000
    4579          12 :       if (Opc0 != ISD::SHL)
    4580             :         return false;
    4581             :       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4582          24 :       if (!C || C->getZExtValue() != 8)
    4583             :         return false;
    4584             :     }
    4585          14 :   } else if (Opc == ISD::SHL) {
    4586             :     // (x & 0xff) << 8
    4587             :     // (x & 0xff0000) << 8
    4588           8 :     if (MaskByteOffset != 0 && MaskByteOffset != 2)
    4589             :       return false;
    4590             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4591          16 :     if (!C || C->getZExtValue() != 8)
    4592             :       return false;
    4593             :   } else { // Opc == ISD::SRL
    4594             :     // (x & 0xff00) >> 8
    4595             :     // (x & 0xff000000) >> 8
    4596           6 :     if (MaskByteOffset != 1 && MaskByteOffset != 3)
    4597             :       return false;
    4598             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4599          12 :     if (!C || C->getZExtValue() != 8)
    4600             :       return false;
    4601             :   }
    4602             : 
    4603          76 :   if (Parts[MaskByteOffset])
    4604             :     return false;
    4605             : 
    4606          38 :   Parts[MaskByteOffset] = N0.getOperand(0).getNode();
    4607             :   return true;
    4608             : }
    4609             : 
    4610             : /// Match a 32-bit packed halfword bswap. That is
    4611             : /// ((x & 0x000000ff) << 8) |
    4612             : /// ((x & 0x0000ff00) >> 8) |
    4613             : /// ((x & 0x00ff0000) << 8) |
    4614             : /// ((x & 0xff000000) >> 8)
    4615             : /// => (rotl (bswap x), 16)
    4616      120680 : SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
    4617      120680 :   if (!LegalOperations)
    4618       51569 :     return SDValue();
    4619             : 
    4620      138222 :   EVT VT = N->getValueType(0);
    4621             :   if (VT != MVT::i32)
    4622       26837 :     return SDValue();
    4623       42274 :   if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    4624        9837 :     return SDValue();
    4625             : 
    4626             :   // Look for either
    4627             :   // (or (or (and), (and)), (or (and), (and)))
    4628             :   // (or (or (or (and), (and)), (and)), (and))
    4629       32437 :   if (N0.getOpcode() != ISD::OR)
    4630       29104 :     return SDValue();
    4631        3333 :   SDValue N00 = N0.getOperand(0);
    4632        3333 :   SDValue N01 = N0.getOperand(1);
    4633        3333 :   SDNode *Parts[4] = {};
    4634             : 
    4635          48 :   if (N1.getOpcode() == ISD::OR &&
    4636        3370 :       N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
    4637             :     // (or (or (and), (and)), (or (and), (and)))
    4638          37 :     if (!isBSwapHWordElement(N00, Parts))
    4639          31 :       return SDValue();
    4640             : 
    4641           6 :     if (!isBSwapHWordElement(N01, Parts))
    4642           0 :       return SDValue();
    4643           6 :     SDValue N10 = N1.getOperand(0);
    4644           6 :     if (!isBSwapHWordElement(N10, Parts))
    4645           0 :       return SDValue();
    4646           6 :     SDValue N11 = N1.getOperand(1);
    4647           6 :     if (!isBSwapHWordElement(N11, Parts))
    4648           2 :       return SDValue();
    4649             :   } else {
    4650             :     // (or (or (or (and), (and)), (and)), (and))
    4651        3296 :     if (!isBSwapHWordElement(N1, Parts))
    4652        3292 :       return SDValue();
    4653           4 :     if (!isBSwapHWordElement(N01, Parts))
    4654           0 :       return SDValue();
    4655           4 :     if (N00.getOpcode() != ISD::OR)
    4656           0 :       return SDValue();
    4657           4 :     SDValue N000 = N00.getOperand(0);
    4658           4 :     if (!isBSwapHWordElement(N000, Parts))
    4659           0 :       return SDValue();
    4660           4 :     SDValue N001 = N00.getOperand(1);
    4661           4 :     if (!isBSwapHWordElement(N001, Parts))
    4662           0 :       return SDValue();
    4663             :   }
    4664             : 
    4665             :   // Make sure the parts are all coming from the same node.
    4666           8 :   if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
    4667           0 :     return SDValue();
    4668             : 
    4669             :   SDLoc DL(N);
    4670           8 :   SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
    4671          16 :                               SDValue(Parts[0], 0));
    4672             : 
    4673             :   // Result of the bswap should be rotated by 16. If it's not legal, then
    4674             :   // do  (x << 16) | (x >> 16).
    4675           8 :   SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
    4676           8 :   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
    4677           8 :     return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
    4678             :   if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
    4679           8 :     return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
    4680           0 :   return DAG.getNode(ISD::OR, DL, VT,
    4681           0 :                      DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
    4682           0 :                      DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
    4683             : }
    4684             : 
    4685             : /// This contains all DAGCombine rules which reduce two values combined by
    4686             : /// an Or operation to a single value \see visitANDLike().
    4687      121202 : SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
    4688      242404 :   EVT VT = N1.getValueType();
    4689             :   SDLoc DL(N);
    4690             : 
    4691             :   // fold (or x, undef) -> -1
    4692      224979 :   if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
    4693          11 :     return DAG.getAllOnesConstant(DL, VT);
    4694             : 
    4695      121191 :   if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
    4696         114 :     return V;
    4697             : 
    4698             :   // (or (and X, C1), (and Y, C2))  -> (and (or X, Y), C3) if possible.
    4699      141166 :   if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
    4700             :       // Don't increase # computations.
    4701             :       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
    4702             :     // We can only do this xform if we know that bits from X that are set in C2
    4703             :     // but not in C1 are already zero.  Likewise for Y.
    4704             :     if (const ConstantSDNode *N0O1C =
    4705        6933 :         getAsNonOpaqueConstant(N0.getOperand(1))) {
    4706             :       if (const ConstantSDNode *N1O1C =
    4707        3153 :           getAsNonOpaqueConstant(N1.getOperand(1))) {
    4708             :         // We can only do this xform if we know that bits from X that are set in
    4709             :         // C2 but not in C1 are already zero.  Likewise for Y.
    4710        3073 :         const APInt &LHSMask = N0O1C->getAPIntValue();
    4711        3073 :         const APInt &RHSMask = N1O1C->getAPIntValue();
    4712             : 
    4713       15597 :         if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
    4714       12756 :             DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
    4715          42 :           SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
    4716          42 :                                   N0.getOperand(0), N1.getOperand(0));
    4717          21 :           return DAG.getNode(ISD::AND, DL, VT, X,
    4718          63 :                              DAG.getConstant(LHSMask | RHSMask, DL, VT));
    4719             :         }
    4720             :       }
    4721             :     }
    4722             :   }
    4723             : 
    4724             :   // (or (and X, M), (and X, N)) -> (and X, (or M, N))
    4725       20068 :   if (N0.getOpcode() == ISD::AND &&
    4726             :       N1.getOpcode() == ISD::AND &&
    4727      121056 :       N0.getOperand(0) == N1.getOperand(0) &&
    4728             :       // Don't increase # computations.
    4729             :       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
    4730         578 :     SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
    4731         578 :                             N0.getOperand(1), N1.getOperand(1));
    4732         578 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
    4733             :   }
    4734             : 
    4735      120767 :   return SDValue();
    4736             : }
    4737             : 
    4738      123532 : SDValue DAGCombiner::visitOR(SDNode *N) {
    4739      123532 :   SDValue N0 = N->getOperand(0);
    4740      123532 :   SDValue N1 = N->getOperand(1);
    4741      123532 :   EVT VT = N1.getValueType();
    4742             : 
    4743             :   // x | x --> x
    4744             :   if (N0 == N1)
    4745          11 :     return N0;
    4746             : 
    4747             :   // fold vector ops
    4748      123521 :   if (VT.isVector()) {
    4749       18299 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    4750           4 :       return FoldedVOp;
    4751             : 
    4752             :     // fold (or x, 0) -> x, vector edition
    4753       18295 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    4754          18 :       return N1;
    4755       18277 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    4756         127 :       return N0;
    4757             : 
    4758             :     // fold (or x, -1) -> -1, vector edition
    4759       18150 :     if (ISD::isBuildVectorAllOnes(N0.getNode()))
    4760             :       // do not return N0, because undef node may exist in N0
    4761           6 :       return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
    4762       18148 :     if (ISD::isBuildVectorAllOnes(N1.getNode()))
    4763             :       // do not return N1, because undef node may exist in N1
    4764           3 :       return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
    4765             : 
    4766             :     // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
    4767             :     // Do this only if the resulting shuffle is legal.
    4768         135 :     if (isa<ShuffleVectorSDNode>(N0) &&
    4769       18147 :         isa<ShuffleVectorSDNode>(N1) &&
    4770             :         // Avoid folding a node with illegal type.
    4771         124 :         TLI.isTypeLegal(VT)) {
    4772         123 :       bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
    4773         123 :       bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
    4774         123 :       bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
    4775         123 :       bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
    4776             :       // Ensure both shuffles have a zero input.
    4777         123 :       if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
    4778             :         assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!");
    4779             :         assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!");
    4780             :         const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
    4781             :         const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
    4782             :         bool CanFold = true;
    4783          66 :         int NumElts = VT.getVectorNumElements();
    4784         132 :         SmallVector<int, 4> Mask(NumElts);
    4785             : 
    4786         534 :         for (int i = 0; i != NumElts; ++i) {
    4787         239 :           int M0 = SV0->getMaskElt(i);
    4788         239 :           int M1 = SV1->getMaskElt(i);
    4789             : 
    4790             :           // Determine if either index is pointing to a zero vector.
    4791         239 :           bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
    4792         239 :           bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
    4793             : 
    4794             :           // If one element is zero and the otherside is undef, keep undef.
    4795             :           // This also handles the case that both are undef.
    4796         240 :           if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
    4797           2 :             Mask[i] = -1;
    4798           1 :             continue;
    4799             :           }
    4800             : 
    4801             :           // Make sure only one of the elements is zero.
    4802         238 :           if (M0Zero == M1Zero) {
    4803             :             CanFold = false;
    4804             :             break;
    4805             :           }
    4806             : 
    4807             :           assert((M0 >= 0 || M1 >= 0) && "Undef index!");
    4808             : 
    4809             :           // We have a zero and non-zero element. If the non-zero came from
    4810             :           // SV0 make the index a LHS index. If it came from SV1, make it
    4811             :           // a RHS index. We need to mod by NumElts because we don't care
    4812             :           // which operand it came from in the original shuffles.
    4813         466 :           Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
    4814             :         }
    4815             : 
    4816          66 :         if (CanFold) {
    4817         122 :           SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
    4818         122 :           SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
    4819             : 
    4820         122 :           bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
    4821          61 :           if (!LegalMask) {
    4822             :             std::swap(NewLHS, NewRHS);
    4823             :             ShuffleVectorSDNode::commuteMask(Mask);
    4824           0 :             LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
    4825             :           }
    4826             : 
    4827          61 :           if (LegalMask)
    4828         183 :             return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
    4829             :         }
    4830             :       }
    4831             :     }
    4832             :   }
    4833             : 
    4834             :   // fold (or c1, c2) -> c1|c2
    4835             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    4836             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    4837      123385 :   if (N0C && N1C && !N1C->isOpaque())
    4838         225 :     return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
    4839             :   // canonicalize constant to RHS
    4840      124405 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    4841        1172 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    4842        3498 :     return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
    4843             :   // fold (or x, 0) -> x
    4844      122067 :   if (isNullConstant(N1))
    4845         839 :     return N0;
    4846             :   // fold (or x, -1) -> -1
    4847      121228 :   if (isAllOnesConstant(N1))
    4848          76 :     return N1;
    4849             : 
    4850      121152 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    4851          41 :     return NewSel;
    4852             : 
    4853             :   // fold (or x, c) -> c iff (x & ~c) == 0
    4854      421100 :   if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
    4855           2 :     return N1;
    4856             : 
    4857      121109 :   if (SDValue Combined = visitORLike(N0, N1, N))
    4858         429 :     return Combined;
    4859             : 
    4860             :   // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
    4861      120680 :   if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
    4862           8 :     return BSwap;
    4863      120672 :   if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
    4864           7 :     return BSwap;
    4865             : 
    4866             :   // reassociate or
    4867      241330 :   if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
    4868          65 :     return ROR;
    4869             : 
    4870             :   // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
    4871             :   // iff (c1 & c2) != 0.
    4872         668 :   auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    4873        1336 :     return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
    4874         668 :   };
    4875      260728 :   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
    4876      260697 :       ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
    4877          62 :     if (SDValue COR = DAG.FoldConstantArithmetic(
    4878          93 :             ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
    4879          93 :       SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
    4880          31 :       AddToWorklist(IOR.getNode());
    4881          93 :       return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
    4882             :     }
    4883             :   }
    4884             : 
    4885             :   // Simplify: (or (op x...), (op y...))  -> (op (or x, y))
    4886      120569 :   if (N0.getOpcode() == N1.getOpcode())
    4887       17457 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    4888         424 :       return Tmp;
    4889             : 
    4890             :   // See if this is some rotate idiom.
    4891      240290 :   if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
    4892         834 :     return SDValue(Rot, 0);
    4893             : 
    4894      119311 :   if (SDValue Load = MatchLoadCombine(N))
    4895         184 :     return Load;
    4896             : 
    4897             :   // Simplify the operands using demanded-bits information.
    4898      119127 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    4899        5763 :     return SDValue(N, 0);
    4900             : 
    4901      113364 :   return SDValue();
    4902             : }
    4903             : 
    4904             : /// Match "(X shl/srl V1) & V2" where V2 may not be present.
    4905      105697 : bool DAGCombiner::MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
    4906      105697 :   if (Op.getOpcode() == ISD::AND) {
    4907       23182 :     if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
    4908        9586 :       Mask = Op.getOperand(1);
    4909        9586 :       Op = Op.getOperand(0);
    4910             :     } else {
    4911             :       return false;
    4912             :     }
    4913             :   }
    4914             : 
    4915      103692 :   if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
    4916       14630 :     Shift = Op;
    4917             :     return true;
    4918             :   }
    4919             : 
    4920             :   return false;
    4921             : }
    4922             : 
    4923             : // Return true if we can prove that, whenever Neg and Pos are both in the
    4924             : // range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos).  This means that
    4925             : // for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
    4926             : //
    4927             : //     (or (shift1 X, Neg), (shift2 X, Pos))
    4928             : //
    4929             : // reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
    4930             : // in direction shift1 by Neg.  The range [0, EltSize) means that we only need
    4931             : // to consider shift amounts with defined behavior.
    4932         446 : static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
    4933             :                            SelectionDAG &DAG) {
    4934             :   // If EltSize is a power of 2 then:
    4935             :   //
    4936             :   //  (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
    4937             :   //  (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
    4938             :   //
    4939             :   // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
    4940             :   // for the stronger condition:
    4941             :   //
    4942             :   //     Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1)    [A]
    4943             :   //
    4944             :   // for all Neg and Pos.  Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
    4945             :   // we can just replace Neg with Neg' for the rest of the function.
    4946             :   //
    4947             :   // In other cases we check for the even stronger condition:
    4948             :   //
    4949             :   //     Neg == EltSize - Pos                                    [B]
    4950             :   //
    4951             :   // for all Neg and Pos.  Note that the (or ...) then invokes undefined
    4952             :   // behavior if Pos == 0 (and consequently Neg == EltSize).
    4953             :   //
    4954             :   // We could actually use [A] whenever EltSize is a power of 2, but the
    4955             :   // only extra cases that it would match are those uninteresting ones
    4956             :   // where Neg and Pos are never in range at the same time.  E.g. for
    4957             :   // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
    4958             :   // as well as (sub 32, Pos), but:
    4959             :   //
    4960             :   //     (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
    4961             :   //
    4962             :   // always invokes undefined behavior for 32-bit X.
    4963             :   //
    4964             :   // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
    4965             :   unsigned MaskLoBits = 0;
    4966         446 :   if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
    4967          72 :     if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
    4968          72 :       KnownBits Known;
    4969          72 :       DAG.computeKnownBits(Neg.getOperand(0), Known);
    4970             :       unsigned Bits = Log2_64(EltSize);
    4971         288 :       if (NegC->getAPIntValue().getActiveBits() <= Bits &&
    4972         216 :           ((NegC->getAPIntValue() | Known.Zero).countTrailingOnes() >= Bits)) {
    4973          70 :         Neg = Neg.getOperand(0);
    4974             :         MaskLoBits = Bits;
    4975             :       }
    4976             :     }
    4977             :   }
    4978             : 
    4979             :   // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
    4980         446 :   if (Neg.getOpcode() != ISD::SUB)
    4981             :     return false;
    4982         324 :   ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
    4983         324 :   if (!NegC)
    4984             :     return false;
    4985         324 :   SDValue NegOp1 = Neg.getOperand(1);
    4986             : 
    4987             :   // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
    4988             :   // Pos'.  The truncation is redundant for the purpose of the equality.
    4989         376 :   if (MaskLoBits && Pos.getOpcode() == ISD::AND) {
    4990          46 :     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1))) {
    4991          46 :       KnownBits Known;
    4992          46 :       DAG.computeKnownBits(Pos.getOperand(0), Known);
    4993         184 :       if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
    4994         138 :           ((PosC->getAPIntValue() | Known.Zero).countTrailingOnes() >=
    4995             :            MaskLoBits))
    4996          34 :         Pos = Pos.getOperand(0);
    4997             :     }
    4998             :   }
    4999             : 
    5000             :   // The condition we need is now:
    5001             :   //
    5002             :   //     (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
    5003             :   //
    5004             :   // If NegOp1 == Pos then we need:
    5005             :   //
    5006             :   //              EltSize & Mask == NegC & Mask
    5007             :   //
    5008             :   // (because "x & Mask" is a truncation and distributes through subtraction).
    5009             :   APInt Width;
    5010             :   if (Pos == NegOp1)
    5011         634 :     Width = NegC->getAPIntValue();
    5012             : 
    5013             :   // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
    5014             :   // Then the condition we want to prove becomes:
    5015             :   //
    5016             :   //     (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
    5017             :   //
    5018             :   // which, again because "x & Mask" is a truncation, becomes:
    5019             :   //
    5020             :   //                NegC & Mask == (EltSize - PosC) & Mask
    5021             :   //             EltSize & Mask == (NegC + PosC) & Mask
    5022           7 :   else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
    5023           3 :     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
    5024           9 :       Width = PosC->getAPIntValue() + NegC->getAPIntValue();
    5025             :     else
    5026             :       return false;
    5027             :   } else
    5028             :     return false;
    5029             : 
    5030             :   // Now we just need to check that EltSize & Mask == Width & Mask.
    5031         320 :   if (MaskLoBits)
    5032             :     // EltSize & Mask is 0 since Mask is EltSize - 1.
    5033         104 :     return Width.getLoBits(MaskLoBits) == 0;
    5034         268 :   return Width == EltSize;
    5035             : }
    5036             : 
    5037             : // A subroutine of MatchRotate used once we have found an OR of two opposite
    5038             : // shifts of Shifted.  If Neg == <operand size> - Pos then the OR reduces
    5039             : // to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
    5040             : // former being preferred if supported.  InnerPos and InnerNeg are Pos and
    5041             : // Neg with outer conversions stripped away.
    5042         446 : SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
    5043             :                                        SDValue Neg, SDValue InnerPos,
    5044             :                                        SDValue InnerNeg, unsigned PosOpcode,
    5045             :                                        unsigned NegOpcode, const SDLoc &DL) {
    5046             :   // fold (or (shl x, (*ext y)),
    5047             :   //          (srl x, (*ext (sub 32, y)))) ->
    5048             :   //   (rotl x, y) or (rotr x, (sub 32, y))
    5049             :   //
    5050             :   // fold (or (shl x, (*ext (sub 32, y))),
    5051             :   //          (srl x, (*ext y))) ->
    5052             :   //   (rotr x, y) or (rotl x, (sub 32, y))
    5053         892 :   EVT VT = Shifted.getValueType();
    5054         892 :   if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
    5055         316 :     bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
    5056         948 :     return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
    5057         948 :                        HasPos ? Pos : Neg).getNode();
    5058             :   }
    5059             : 
    5060             :   return nullptr;
    5061             : }
    5062             : 
    5063             : // MatchRotate - Handle an 'or' of two operands.  If this is one of the many
    5064             : // idioms for rotate, and if the target supports rotation instructions, generate
    5065             : // a rot[lr].
    5066      120254 : SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
    5067             :   // Must be a legal type.  Expanded 'n promoted things won't work with rotates.
    5068      120254 :   EVT VT = LHS.getValueType();
    5069      120254 :   if (!TLI.isTypeLegal(VT)) return nullptr;
    5070             : 
    5071             :   // The target must have at least one rotate flavor.
    5072      117701 :   bool HasROTL = hasOperation(ISD::ROTL, VT);
    5073      117701 :   bool HasROTR = hasOperation(ISD::ROTR, VT);
    5074      117701 :   if (!HasROTL && !HasROTR) return nullptr;
    5075             : 
    5076             :   // Check for truncated rotate.
    5077       97015 :   if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
    5078         218 :       LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
    5079             :     assert(LHS.getValueType() == RHS.getValueType());
    5080         109 :     if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
    5081           4 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
    5082           2 :                          SDValue(Rot, 0)).getNode();
    5083             :     }
    5084             :   }
    5085             : 
    5086             :   // Match "(X shl/srl V1) & V2" where V2 may not be present.
    5087       96450 :   SDValue LHSShift;   // The shift.
    5088       96450 :   SDValue LHSMask;    // AND value if any.
    5089       96450 :   if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
    5090             :     return nullptr; // Not part of a rotate.
    5091             : 
    5092        9247 :   SDValue RHSShift;   // The shift.
    5093        9247 :   SDValue RHSMask;    // AND value if any.
    5094        9247 :   if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
    5095             :     return nullptr; // Not part of a rotate.
    5096             : 
    5097       10766 :   if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
    5098             :     return nullptr;   // Not shifting the same value.
    5099             : 
    5100        1131 :   if (LHSShift.getOpcode() == RHSShift.getOpcode())
    5101             :     return nullptr;   // Shifts must disagree.
    5102             : 
    5103             :   // Canonicalize shl to left side in a shl/srl pair.
    5104         888 :   if (RHSShift.getOpcode() == ISD::SHL) {
    5105             :     std::swap(LHS, RHS);
    5106             :     std::swap(LHSShift, RHSShift);
    5107             :     std::swap(LHSMask, RHSMask);
    5108             :   }
    5109             : 
    5110             :   unsigned EltSizeInBits = VT.getScalarSizeInBits();
    5111        1776 :   SDValue LHSShiftArg = LHSShift.getOperand(0);
    5112         888 :   SDValue LHSShiftAmt = LHSShift.getOperand(1);
    5113        1776 :   SDValue RHSShiftArg = RHSShift.getOperand(0);
    5114         888 :   SDValue RHSShiftAmt = RHSShift.getOperand(1);
    5115             : 
    5116             :   // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
    5117             :   // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
    5118             :   auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
    5119        2080 :                                         ConstantSDNode *RHS) {
    5120        8320 :     return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
    5121        4160 :   };
    5122        1776 :   if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
    5123         518 :     SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
    5124         518 :                               LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
    5125             : 
    5126             :     // If there is an AND of either shifted operand, apply it to the result.
    5127         518 :     if (LHSMask.getNode() || RHSMask.getNode()) {
    5128          69 :       SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
    5129          69 :       SDValue Mask = AllOnes;
    5130             : 
    5131          69 :       if (LHSMask.getNode()) {
    5132         138 :         SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
    5133         138 :         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
    5134         138 :                            DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
    5135             :       }
    5136          69 :       if (RHSMask.getNode()) {
    5137         112 :         SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
    5138         112 :         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
    5139         112 :                            DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
    5140             :       }
    5141             : 
    5142         138 :       Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
    5143             :     }
    5144             : 
    5145             :     return Rot.getNode();
    5146             :   }
    5147             : 
    5148             :   // If there is a mask here, and we have a variable shift, we can't be sure
    5149             :   // that we're masking out the right stuff.
    5150         370 :   if (LHSMask.getNode() || RHSMask.getNode())
    5151             :     return nullptr;
    5152             : 
    5153             :   // If the shift amount is sign/zext/any-extended just peel it off.
    5154         339 :   SDValue LExtOp0 = LHSShiftAmt;
    5155         339 :   SDValue RExtOp0 = RHSShiftAmt;
    5156         339 :   if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
    5157         328 :        LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
    5158         328 :        LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
    5159         418 :        LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
    5160          79 :       (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
    5161          68 :        RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
    5162          68 :        RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
    5163             :        RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
    5164          71 :     LExtOp0 = LHSShiftAmt.getOperand(0);
    5165          71 :     RExtOp0 = RHSShiftAmt.getOperand(0);
    5166             :   }
    5167             : 
    5168             :   SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
    5169         339 :                                    LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
    5170         339 :   if (TryL)
    5171             :     return TryL;
    5172             : 
    5173             :   SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
    5174         107 :                                    RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
    5175         107 :   if (TryR)
    5176             :     return TryR;
    5177             : 
    5178          23 :   return nullptr;
    5179             : }
    5180             : 
    5181             : namespace {
    5182             : 
    5183             : /// Represents known origin of an individual byte in load combine pattern. The
    5184             : /// value of the byte is either constant zero or comes from memory.
    5185             : struct ByteProvider {
    5186             :   // For constant zero providers Load is set to nullptr. For memory providers
    5187             :   // Load represents the node which loads the byte from memory.
    5188             :   // ByteOffset is the offset of the byte in the value produced by the load.
    5189             :   LoadSDNode *Load = nullptr;
    5190             :   unsigned ByteOffset = 0;
    5191             : 
    5192             :   ByteProvider() = default;
    5193             : 
    5194             :   static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
    5195             :     return ByteProvider(Load, ByteOffset);
    5196             :   }
    5197             : 
    5198             :   static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
    5199             : 
    5200             :   bool isConstantZero() const { return !Load; }
    5201             :   bool isMemory() const { return Load; }
    5202             : 
    5203             :   bool operator==(const ByteProvider &Other) const {
    5204             :     return Other.Load == Load && Other.ByteOffset == ByteOffset;
    5205             :   }
    5206             : 
    5207             : private:
    5208             :   ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
    5209             :       : Load(Load), ByteOffset(ByteOffset) {}
    5210             : };
    5211             : 
    5212             : } // end anonymous namespace
    5213             : 
    5214             : /// Recursively traverses the expression calculating the origin of the requested
    5215             : /// byte of the given value. Returns None if the provider can't be calculated.
    5216             : ///
    5217             : /// For all the values except the root of the expression verifies that the value
    5218             : /// has exactly one use and if it's not true return None. This way if the origin
    5219             : /// of the byte is returned it's guaranteed that the values which contribute to
    5220             : /// the byte are not used outside of this expression.
    5221             : ///
    5222             : /// Because the parts of the expression are not allowed to have more than one
    5223             : /// use this function iterates over trees, not DAGs. So it never visits the same
    5224             : /// node more than once.
    5225             : static const Optional<ByteProvider>
    5226      258703 : calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
    5227             :                       bool Root = false) {
    5228             :   // Typical i64 by i8 pattern requires recursion up to 8 calls depth
    5229      258703 :   if (Depth == 10)
    5230             :     return None;
    5231             : 
    5232      419795 :   if (!Root && !Op.hasOneUse())
    5233             :     return None;
    5234             : 
    5235             :   assert(Op.getValueType().isScalarInteger() && "can't handle other types");
    5236      232706 :   unsigned BitWidth = Op.getValueSizeInBits();
    5237      232706 :   if (BitWidth % 8 != 0)
    5238             :     return None;
    5239      232706 :   unsigned ByteWidth = BitWidth / 8;
    5240             :   assert(Index < ByteWidth && "invalid index requested");
    5241             :   (void) ByteWidth;
    5242             : 
    5243      465412 :   switch (Op.getOpcode()) {
    5244      137114 :   case ISD::OR: {
    5245      274228 :     auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
    5246      137114 :     if (!LHS)
    5247             :       return None;
    5248       28148 :     auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
    5249       14074 :     if (!RHS)
    5250             :       return None;
    5251             : 
    5252       11444 :     if (LHS->isConstantZero())
    5253             :       return RHS;
    5254        5388 :     if (RHS->isConstantZero())
    5255             :       return LHS;
    5256             :     return None;
    5257             :   }
    5258       15650 :   case ISD::SHL: {
    5259       15650 :     auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
    5260             :     if (!ShiftOp)
    5261             :       return None;
    5262             : 
    5263       15271 :     uint64_t BitShift = ShiftOp->getZExtValue();
    5264       15271 :     if (BitShift % 8 != 0)
    5265             :       return None;
    5266       12641 :     uint64_t ByteShift = BitShift / 8;
    5267             : 
    5268       12641 :     return Index < ByteShift
    5269             :                ? ByteProvider::getConstantZero()
    5270             :                : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
    5271       23857 :                                        Depth + 1);
    5272             :   }
    5273       12174 :   case ISD::ANY_EXTEND:
    5274             :   case ISD::SIGN_EXTEND:
    5275             :   case ISD::ZERO_EXTEND: {
    5276       12174 :     SDValue NarrowOp = Op->getOperand(0);
    5277       12174 :     unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
    5278       12174 :     if (NarrowBitWidth % 8 != 0)
    5279             :       return None;
    5280       11980 :     uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    5281             : 
    5282       11980 :     if (Index >= NarrowByteWidth)
    5283        1582 :       return Op.getOpcode() == ISD::ZERO_EXTEND
    5284             :                  ? Optional<ByteProvider>(ByteProvider::getConstantZero())
    5285        1582 :                  : None;
    5286       10398 :     return calculateByteProvider(NarrowOp, Index, Depth + 1);
    5287             :   }
    5288          42 :   case ISD::BSWAP:
    5289          84 :     return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
    5290         126 :                                  Depth + 1);
    5291       13746 :   case ISD::LOAD: {
    5292             :     auto L = cast<LoadSDNode>(Op.getNode());
    5293       27408 :     if (L->isVolatile() || L->isIndexed())
    5294             :       return None;
    5295             : 
    5296       13651 :     unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
    5297       13651 :     if (NarrowBitWidth % 8 != 0)
    5298             :       return None;
    5299       13432 :     uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    5300             : 
    5301       13432 :     if (Index >= NarrowByteWidth)
    5302             :       return L->getExtensionType() == ISD::ZEXTLOAD
    5303             :                  ? Optional<ByteProvider>(ByteProvider::getConstantZero())
    5304        4572 :                  : None;
    5305             :     return ByteProvider::getMemory(L, Index);
    5306             :   }
    5307             :   }
    5308             : 
    5309             :   return None;
    5310             : }
    5311             : 
    5312             : /// Match a pattern where a wide type scalar value is loaded by several narrow
    5313             : /// loads and combined by shifts and ors. Fold it into a single load or a load
    5314             : /// and a BSWAP if the targets supports it.
    5315             : ///
    5316             : /// Assuming little endian target:
    5317             : ///  i8 *a = ...
    5318             : ///  i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
    5319             : /// =>
    5320             : ///  i32 val = *((i32)a)
    5321             : ///
    5322             : ///  i8 *a = ...
    5323             : ///  i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
    5324             : /// =>
    5325             : ///  i32 val = BSWAP(*((i32)a))
    5326             : ///
    5327             : /// TODO: This rule matches complex patterns with OR node roots and doesn't
    5328             : /// interact well with the worklist mechanism. When a part of the pattern is
    5329             : /// updated (e.g. one of the loads) its direct users are put into the worklist,
    5330             : /// but the root node of the pattern which triggers the load combine is not
    5331             : /// necessarily a direct user of the changed node. For example, once the address
    5332             : /// of t28 load is reassociated load combine won't be triggered:
    5333             : ///             t25: i32 = add t4, Constant:i32<2>
    5334             : ///           t26: i64 = sign_extend t25
    5335             : ///        t27: i64 = add t2, t26
    5336             : ///       t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
    5337             : ///     t29: i32 = zero_extend t28
    5338             : ///   t32: i32 = shl t29, Constant:i8<8>
    5339             : /// t33: i32 = or t23, t32
    5340             : /// As a possible fix visitLoad can check if the load can be a part of a load
    5341             : /// combine pattern and add corresponding OR roots to the worklist.
    5342      119311 : SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
    5343             :   assert(N->getOpcode() == ISD::OR &&
    5344             :          "Can only match load combining against OR nodes");
    5345             : 
    5346             :   // Handles simple types only
    5347      238622 :   EVT VT = N->getValueType(0);
    5348             :   if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
    5349       20647 :     return SDValue();
    5350       98664 :   unsigned ByteWidth = VT.getSizeInBits() / 8;
    5351             : 
    5352       98664 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5353             :   // Before legalize we can introduce too wide illegal loads which will be later
    5354             :   // split into legal sized loads. This enables us to combine i64 load by i8
    5355             :   // patterns to a couple of i32 loads on 32 bit targets.
    5356       98664 :   if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
    5357       12796 :     return SDValue();
    5358             : 
    5359             :   std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
    5360             :     unsigned BW, unsigned i) { return i; };
    5361             :   std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
    5362        4166 :     unsigned BW, unsigned i) { return BW - i - 1; };
    5363             : 
    5364      171736 :   bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
    5365        7201 :   auto MemoryByteOffset = [&] (ByteProvider P) {
    5366             :     assert(P.isMemory() && "Must be a memory byte provider");
    5367        7201 :     unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
    5368             :     assert(LoadBitWidth % 8 == 0 &&
    5369             :            "can only analyze providers for individual bytes not bit");
    5370        7201 :     unsigned LoadByteWidth = LoadBitWidth / 8;
    5371        7201 :     return IsBigEndianTarget
    5372        7201 :             ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
    5373       13758 :             : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
    5374       85868 :   };
    5375             : 
    5376             :   Optional<BaseIndexOffset> Base;
    5377             :   SDValue Chain;
    5378             : 
    5379             :   SmallPtrSet<LoadSDNode *, 8> Loads;
    5380             :   Optional<ByteProvider> FirstByteProvider;
    5381             :   int64_t FirstOffset = INT64_MAX;
    5382             : 
    5383             :   // Check if all the bytes of the OR we are looking at are loaded from the same
    5384             :   // base address. Collect bytes offsets from Base address in ByteOffsets.
    5385      171736 :   SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
    5386       98686 :   for (unsigned i = 0; i < ByteWidth; i++) {
    5387       91467 :     auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
    5388       91467 :     if (!P || !P->isMemory()) // All the bytes must be loaded from memory
    5389       84634 :       return SDValue();
    5390             : 
    5391             :     LoadSDNode *L = P->Load;
    5392             :     assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&
    5393             :            "Must be enforced by calculateByteProvider");
    5394             :     assert(L->getOffset().isUndef() && "Unindexed load must have undef offset");
    5395             : 
    5396             :     // All loads must share the same chain
    5397        6833 :     SDValue LChain = L->getChain();
    5398        6833 :     if (!Chain)
    5399             :       Chain = LChain;
    5400             :     else if (Chain != LChain)
    5401         329 :       return SDValue();
    5402             : 
    5403             :     // Loads must share the same base address
    5404        6504 :     BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
    5405        6504 :     int64_t ByteOffsetFromBase = 0;
    5406        6504 :     if (!Base)
    5407             :       Base = Ptr;
    5408        4328 :     else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
    5409          95 :       return SDValue();
    5410             : 
    5411             :     // Calculate the offset of the current byte from the base address
    5412        6409 :     ByteOffsetFromBase += MemoryByteOffset(*P);
    5413       12818 :     ByteOffsets[i] = ByteOffsetFromBase;
    5414             : 
    5415             :     // Remember the first byte load
    5416        6409 :     if (ByteOffsetFromBase < FirstOffset) {
    5417             :       FirstByteProvider = P;
    5418             :       FirstOffset = ByteOffsetFromBase;
    5419             :     }
    5420             : 
    5421        6409 :     Loads.insert(L);
    5422             :   }
    5423             :   assert(!Loads.empty() && "All the bytes of the value must be loaded from "
    5424             :          "memory, so there must be at least one load which produces the value");
    5425             :   assert(Base && "Base address of the accessed memory location must be set");
    5426             :   assert(FirstOffset != INT64_MAX && "First byte offset must be set");
    5427             : 
    5428             :   // Check if the bytes of the OR we are looking at match with either big or
    5429             :   // little endian value load
    5430             :   bool BigEndian = true, LittleEndian = true;
    5431        7818 :   for (unsigned i = 0; i < ByteWidth; i++) {
    5432        7044 :     int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
    5433        3522 :     LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
    5434        3522 :     BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
    5435        3522 :     if (!BigEndian && !LittleEndian)
    5436          18 :       return SDValue();
    5437             :   }
    5438             :   assert((BigEndian != LittleEndian) && "should be either or");
    5439             :   assert(FirstByteProvider && "must be set");
    5440             : 
    5441             :   // Ensure that the first byte is loaded from zero offset of the first load.
    5442             :   // So the combined value can be loaded from the first load address.
    5443         792 :   if (MemoryByteOffset(*FirstByteProvider) != 0)
    5444          10 :     return SDValue();
    5445             :   LoadSDNode *FirstLoad = FirstByteProvider->Load;
    5446             : 
    5447             :   // The node we are looking at matches with the pattern, check if we can
    5448             :   // replace it with a single load and bswap if needed.
    5449             : 
    5450             :   // If the load needs byte swap check if the target supports it
    5451         782 :   bool NeedsBswap = IsBigEndianTarget != BigEndian;
    5452             : 
    5453             :   // Before legalize we can introduce illegal bswaps which will be later
    5454             :   // converted to an explicit bswap sequence. This way we end up with a single
    5455             :   // load and byte shuffling instead of several loads and byte shuffling.
    5456         782 :   if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
    5457           0 :     return SDValue();
    5458             : 
    5459             :   // Check that a load of the wide type is both allowed and fast on the target
    5460         782 :   bool Fast = false;
    5461        2346 :   bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
    5462             :                                         VT, FirstLoad->getAddressSpace(),
    5463         782 :                                         FirstLoad->getAlignment(), &Fast);
    5464         976 :   if (!Allowed || !Fast)
    5465         598 :     return SDValue();
    5466             : 
    5467             :   SDValue NewLoad =
    5468         368 :       DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
    5469         736 :                   FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
    5470             : 
    5471             :   // Transfer chain users from old loads to the new load.
    5472         184 :   for (LoadSDNode *L : Loads)
    5473        1402 :     DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
    5474             : 
    5475         552 :   return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
    5476             : }
    5477             : 
    5478             : // If the target has andn, bsl, or a similar bit-select instruction,
    5479             : // we want to unfold masked merge, with canonical pattern of:
    5480             : //   |        A  |  |B|
    5481             : //   ((x ^ y) & m) ^ y
    5482             : //    |  D  |
    5483             : // Into:
    5484             : //   (x & m) | (y & ~m)
    5485             : // If y is a constant, and the 'andn' does not work with immediates,
    5486             : // we unfold into a different pattern:
    5487             : //   ~(~x & m) & (m | y)
    5488             : // NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
    5489             : //       the very least that breaks andnpd / andnps patterns, and because those
    5490             : //       patterns are simplified in IR and shouldn't be created in the DAG
    5491       56698 : SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
    5492             :   assert(N->getOpcode() == ISD::XOR);
    5493             : 
    5494             :   // Don't touch 'not' (i.e. where y = -1).
    5495      113396 :   if (isAllOnesConstantOrAllOnesSplatConstant(N->getOperand(1)))
    5496       18328 :     return SDValue();
    5497             : 
    5498       76740 :   EVT VT = N->getValueType(0);
    5499             : 
    5500             :   // There are 3 commutable operators in the pattern,
    5501             :   // so we have to deal with 8 possible variants of the basic pattern.
    5502       38370 :   SDValue X, Y, M;
    5503      153735 :   auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
    5504      153570 :     if (And.getOpcode() != ISD::AND || !And.hasOneUse())
    5505             :       return false;
    5506        2198 :     SDValue Xor = And.getOperand(XorIdx);
    5507        3263 :     if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
    5508             :       return false;
    5509        1027 :     SDValue Xor0 = Xor.getOperand(0);
    5510        1027 :     SDValue Xor1 = Xor.getOperand(1);
    5511             :     // Don't touch 'not' (i.e. where y = -1).
    5512        1027 :     if (isAllOnesConstantOrAllOnesSplatConstant(Xor1))
    5513             :       return false;
    5514             :     if (Other == Xor0)
    5515             :       std::swap(Xor0, Xor1);
    5516             :     if (Other != Xor1)
    5517             :       return false;
    5518         965 :     X = Xor0;
    5519         965 :     Y = Xor1;
    5520        1930 :     M = And.getOperand(XorIdx ? 0 : 1);
    5521             :     return true;
    5522       38370 :   };
    5523             : 
    5524       38370 :   SDValue N0 = N->getOperand(0);
    5525       38370 :   SDValue N1 = N->getOperand(1);
    5526      113415 :   if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
    5527      113309 :       !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
    5528       37405 :     return SDValue();
    5529             : 
    5530             :   // Don't do anything if the mask is constant. This should not be reachable.
    5531             :   // InstCombine should have already unfolded this pattern, and DAGCombiner
    5532             :   // probably shouldn't produce it, too.
    5533         965 :   if (isa<ConstantSDNode>(M.getNode()))
    5534         205 :     return SDValue();
    5535             : 
    5536             :   // We can transform if the target has AndNot
    5537         760 :   if (!TLI.hasAndNot(M))
    5538         646 :     return SDValue();
    5539             : 
    5540             :   SDLoc DL(N);
    5541             : 
    5542             :   // If Y is a constant, check that 'andn' works with immediates.
    5543         114 :   if (!TLI.hasAndNot(Y)) {
    5544             :     assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.");
    5545             :     // If not, we need to do a bit more work to make sure andn is still used.
    5546           2 :     SDValue NotX = DAG.getNOT(DL, X, VT);
    5547           4 :     SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
    5548           2 :     SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
    5549           4 :     SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
    5550           4 :     return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
    5551             :   }
    5552             : 
    5553         224 :   SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
    5554         112 :   SDValue NotM = DAG.getNOT(DL, M, VT);
    5555         224 :   SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
    5556             : 
    5557         224 :   return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
    5558             : }
    5559             : 
    5560       99834 : SDValue DAGCombiner::visitXOR(SDNode *N) {
    5561       99834 :   SDValue N0 = N->getOperand(0);
    5562       99834 :   SDValue N1 = N->getOperand(1);
    5563      199668 :   EVT VT = N0.getValueType();
    5564             : 
    5565             :   // fold vector ops
    5566       99834 :   if (VT.isVector()) {
    5567       23079 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    5568           0 :       return FoldedVOp;
    5569             : 
    5570             :     // fold (xor x, 0) -> x, vector edition
    5571       23079 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    5572          18 :       return N1;
    5573       23061 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    5574          14 :       return N0;
    5575             :   }
    5576             : 
    5577             :   // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
    5578       99830 :   if (N0.isUndef() && N1.isUndef())
    5579           0 :     return DAG.getConstant(0, SDLoc(N), VT);
    5580             :   // fold (xor x, undef) -> undef
    5581       99802 :   if (N0.isUndef())
    5582          28 :     return N0;
    5583      199548 :   if (N1.isUndef())
    5584           0 :     return N1;
    5585             :   // fold (xor c1, c2) -> c1^c2
    5586             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    5587             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    5588       99774 :   if (N0C && N1C)
    5589        2709 :     return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
    5590             :   // canonicalize constant to RHS
    5591       99447 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    5592         576 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    5593        1725 :     return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
    5594             :   // fold (xor x, 0) -> x
    5595       98296 :   if (isNullConstant(N1))
    5596           0 :     return N0;
    5597             : 
    5598       98296 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    5599           6 :     return NewSel;
    5600             : 
    5601             :   // reassociate xor
    5602      196580 :   if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
    5603         156 :     return RXOR;
    5604             : 
    5605             :   // fold !(x cc y) -> (x !cc y)
    5606       98134 :   SDValue LHS, RHS, CC;
    5607       98134 :   if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
    5608       81384 :     bool isInt = LHS.getValueType().isInteger();
    5609       81384 :     ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
    5610       40692 :                                                isInt);
    5611             : 
    5612       40717 :     if (!LegalOperations ||
    5613          25 :         TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
    5614       40668 :       switch (N0.getOpcode()) {
    5615           0 :       default:
    5616           0 :         llvm_unreachable("Unhandled SetCC Equivalent!");
    5617       40668 :       case ISD::SETCC:
    5618      122004 :         return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
    5619           0 :       case ISD::SELECT_CC:
    5620           0 :         return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
    5621           0 :                                N0.getOperand(3), NotCC);
    5622             :       }
    5623             :     }
    5624             :   }
    5625             : 
    5626             :   // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
    5627       91602 :   if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
    5628       57473 :       N0.getNode()->hasOneUse() &&
    5629           7 :       isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
    5630           3 :     SDValue V = N0.getOperand(0);
    5631             :     SDLoc DL(N0);
    5632           6 :     V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
    5633           6 :                     DAG.getConstant(1, DL, V.getValueType()));
    5634           3 :     AddToWorklist(V.getNode());
    5635           9 :     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
    5636             :   }
    5637             : 
    5638             :   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
    5639       79695 :   if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
    5640        4905 :       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
    5641          79 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    5642          79 :     if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
    5643          50 :       unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
    5644         150 :       LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
    5645         150 :       RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
    5646          50 :       AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
    5647         150 :       return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
    5648             :     }
    5649             :   }
    5650             :   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
    5651       81297 :   if (isAllOnesConstant(N1) && N0.hasOneUse() &&
    5652        8971 :       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
    5653         584 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    5654             :     if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
    5655          60 :       unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
    5656         180 :       LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
    5657         180 :       RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
    5658          60 :       AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
    5659         180 :       return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
    5660             :     }
    5661             :   }
    5662             :   // fold (xor (and x, y), y) -> (and (not x), y)
    5663      114706 :   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
    5664        1713 :       N0->getOperand(1) == N1) {
    5665          74 :     SDValue X = N0->getOperand(0);
    5666         222 :     SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
    5667          74 :     AddToWorklist(NotX.getNode());
    5668         222 :     return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
    5669             :   }
    5670             : 
    5671             :   // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
    5672       57279 :   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
    5673       11906 :     SDValue A = N0.getOpcode() == ISD::ADD ? N0 : N1;
    5674       11906 :     SDValue S = N0.getOpcode() == ISD::SRA ? N0 : N1;
    5675       12300 :     if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
    5676         104 :       SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
    5677         104 :       SDValue S0 = S.getOperand(0);
    5678             :       if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0)) {
    5679             :         unsigned OpSizeInBits = VT.getScalarSizeInBits();
    5680          98 :         if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
    5681         192 :           if (C->getAPIntValue() == (OpSizeInBits - 1))
    5682         288 :             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
    5683             :       }
    5684             :     }
    5685             :   }
    5686             : 
    5687             :   // fold (xor x, x) -> 0
    5688             :   if (N0 == N1)
    5689          39 :     return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
    5690             : 
    5691             :   // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
    5692             :   // Here is a concrete example of this equivalence:
    5693             :   // i16   x ==  14
    5694             :   // i16 shl ==   1 << 14  == 16384 == 0b0100000000000000
    5695             :   // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
    5696             :   //
    5697             :   // =>
    5698             :   //
    5699             :   // i16     ~1      == 0b1111111111111110
    5700             :   // i16 rol(~1, 14) == 0b1011111111111111
    5701             :   //
    5702             :   // Some additional tips to help conceptualize this transform:
    5703             :   // - Try to see the operation as placing a single zero in a value of all ones.
    5704             :   // - There exists no value for x which would allow the result to contain zero.
    5705             :   // - Values of x larger than the bitwidth are undefined and do not require a
    5706             :   //   consistent result.
    5707             :   // - Pushing the zero left requires shifting one bits in from the right.
    5708             :   // A rotate left of ~1 is a nice way of achieving the desired result.
    5709       81177 :   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
    5710         626 :       && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
    5711             :     SDLoc DL(N);
    5712          59 :     return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
    5713          59 :                        N0.getOperand(1));
    5714             :   }
    5715             : 
    5716             :   // Simplify: xor (op x...), (op y...)  -> (op (xor x, y))
    5717       57111 :   if (N0.getOpcode() == N1.getOpcode())
    5718       12771 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    5719         413 :       return Tmp;
    5720             : 
    5721             :   // Unfold  ((x ^ y) & m) ^ y  into  (x & m) | (y & ~m)  if profitable
    5722       56698 :   if (SDValue MM = unfoldMaskedMerge(N))
    5723         114 :     return MM;
    5724             : 
    5725             :   // Simplify the expression using non-local knowledge.
    5726       56584 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    5727         251 :     return SDValue(N, 0);
    5728             : 
    5729       56333 :   return SDValue();
    5730             : }
    5731             : 
    5732             : /// Handle transforms common to the three shifts, when the shift amount is a
    5733             : /// constant.
    5734      270283 : SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
    5735      270283 :   SDNode *LHS = N->getOperand(0).getNode();
    5736      116654 :   if (!LHS->hasOneUse()) return SDValue();
    5737             : 
    5738             :   // We want to pull some binops through shifts, so that we have (and (shift))
    5739             :   // instead of (shift (and)), likewise for add, or, xor, etc.  This sort of
    5740             :   // thing happens with address calculations, so it's important to canonicalize
    5741             :   // it.
    5742             :   bool HighBitSet = false;  // Can we transform this if the high bit is set?
    5743             : 
    5744      307258 :   switch (LHS->getOpcode()) {
    5745      136473 :   default: return SDValue();
    5746             :   case ISD::OR:
    5747             :   case ISD::XOR:
    5748             :     HighBitSet = false; // We can only transform sra if the high bit is clear.
    5749             :     break;
    5750        8643 :   case ISD::AND:
    5751             :     HighBitSet = true;  // We can only transform sra if the high bit is set.
    5752             :     break;
    5753        7458 :   case ISD::ADD:
    5754        7458 :     if (N->getOpcode() != ISD::SHL)
    5755        6155 :       return SDValue(); // only shl(add) not sr[al](add).
    5756             :     HighBitSet = false; // We can only transform sra if the high bit is clear.
    5757             :     break;
    5758             :   }
    5759             : 
    5760             :   // We require the RHS of the binop to be a constant and not opaque as well.
    5761       11001 :   ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
    5762        2492 :   if (!BinOpCst) return SDValue();
    5763             : 
    5764             :   // FIXME: disable this unless the input to the binop is a shift by a constant
    5765             :   // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
    5766        8509 :   SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
    5767       16915 :   bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
    5768       16914 :                  BinOpLHSVal->getOpcode() == ISD::SRA ||
    5769             :                  BinOpLHSVal->getOpcode() == ISD::SRL;
    5770        8509 :   bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
    5771             :                         BinOpLHSVal->getOpcode() == ISD::SELECT;
    5772             : 
    5773       16248 :   if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
    5774             :       !isCopyOrSelect)
    5775        7082 :     return SDValue();
    5776             : 
    5777        1427 :   if (isCopyOrSelect && N->hasOneUse())
    5778         612 :     return SDValue();
    5779             : 
    5780        1630 :   EVT VT = N->getValueType(0);
    5781             : 
    5782             :   // If this is a signed shift right, and the high bit is modified by the
    5783             :   // logical operation, do not perform the transformation. The highBitSet
    5784             :   // boolean indicates the value of the high bit of the constant which would
    5785             :   // cause it to be modified for this operation.
    5786         815 :   if (N->getOpcode() == ISD::SRA) {
    5787           4 :     bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
    5788           4 :     if (BinOpRHSSignSet != HighBitSet)
    5789           4 :       return SDValue();
    5790             :   }
    5791             : 
    5792         811 :   if (!TLI.isDesirableToCommuteWithShift(LHS))
    5793           2 :     return SDValue();
    5794             : 
    5795             :   // Fold the constants, shifting the binop RHS by the shift amount.
    5796        1618 :   SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
    5797             :                                N->getValueType(0),
    5798        4854 :                                LHS->getOperand(1), N->getOperand(1));
    5799             :   assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!");
    5800             : 
    5801             :   // Create the new shift.
    5802         809 :   SDValue NewShift = DAG.getNode(N->getOpcode(),
    5803         809 :                                  SDLoc(LHS->getOperand(0)),
    5804        4045 :                                  VT, LHS->getOperand(0), N->getOperand(1));
    5805             : 
    5806             :   // Create the new binop.
    5807        3236 :   return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
    5808             : }
    5809             : 
    5810         436 : SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
    5811             :   assert(N->getOpcode() == ISD::TRUNCATE);
    5812             :   assert(N->getOperand(0).getOpcode() == ISD::AND);
    5813             : 
    5814             :   // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
    5815         838 :   if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
    5816         358 :     SDValue N01 = N->getOperand(0).getOperand(1);
    5817         179 :     if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
    5818             :       SDLoc DL(N);
    5819         352 :       EVT TruncVT = N->getValueType(0);
    5820         352 :       SDValue N00 = N->getOperand(0).getOperand(0);
    5821         352 :       SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
    5822         352 :       SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
    5823         176 :       AddToWorklist(Trunc00.getNode());
    5824         176 :       AddToWorklist(Trunc01.getNode());
    5825         352 :       return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
    5826             :     }
    5827             :   }
    5828             : 
    5829         260 :   return SDValue();
    5830             : }
    5831             : 
    5832        2106 : SDValue DAGCombiner::visitRotate(SDNode *N) {
    5833             :   SDLoc dl(N);
    5834        2106 :   SDValue N0 = N->getOperand(0);
    5835        2106 :   SDValue N1 = N->getOperand(1);
    5836        4212 :   EVT VT = N->getValueType(0);
    5837             :   unsigned Bitsize = VT.getScalarSizeInBits();
    5838             : 
    5839             :   // fold (rot x, 0) -> x
    5840        2106 :   if (isNullConstantOrNullSplatConstant(N1))
    5841           4 :     return N0;
    5842             : 
    5843             :   // fold (rot x, c) -> (rot x, c % BitSize)
    5844        2102 :   if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
    5845        2325 :     if (Cst->getAPIntValue().uge(Bitsize)) {
    5846           2 :       uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
    5847           2 :       return DAG.getNode(N->getOpcode(), dl, VT, N0,
    5848           4 :                          DAG.getConstant(RotAmt, dl, N1.getValueType()));
    5849             :     }
    5850             :   }
    5851             : 
    5852             :   // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
    5853        2307 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    5854         207 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    5855          44 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    5856         132 :       return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
    5857             :   }
    5858             : 
    5859             :   unsigned NextOp = N0.getOpcode();
    5860             :   // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
    5861        2056 :   if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
    5862          17 :     SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
    5863          34 :     SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
    5864          34 :     if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
    5865          17 :       EVT ShiftVT = C1->getValueType(0);
    5866          17 :       bool SameSide = (N->getOpcode() == NextOp);
    5867          17 :       unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
    5868          17 :       if (SDValue CombinedShift =
    5869          17 :               DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
    5870          17 :         SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
    5871          17 :         SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
    5872             :             ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
    5873          17 :             BitsizeC.getNode());
    5874          34 :         return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
    5875          34 :                            CombinedShiftNorm);
    5876             :       }
    5877             :     }
    5878             :   }
    5879        2039 :   return SDValue();
    5880             : }
    5881             : 
    5882      191913 : SDValue DAGCombiner::visitSHL(SDNode *N) {
    5883      191913 :   SDValue N0 = N->getOperand(0);
    5884      191913 :   SDValue N1 = N->getOperand(1);
    5885      191913 :   EVT VT = N0.getValueType();
    5886             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    5887             : 
    5888             :   // fold vector ops
    5889      191913 :   if (VT.isVector()) {
    5890        3127 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    5891           6 :       return FoldedVOp;
    5892             : 
    5893             :     BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
    5894             :     // If setcc produces all-one true value then:
    5895             :     // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
    5896        1738 :     if (N1CV && N1CV->isConstant()) {
    5897        1694 :       if (N0.getOpcode() == ISD::AND) {
    5898          16 :         SDValue N00 = N0->getOperand(0);
    5899          16 :         SDValue N01 = N0->getOperand(1);
    5900             :         BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
    5901             : 
    5902          34 :         if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
    5903          12 :             TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
    5904             :                 TargetLowering::ZeroOrNegativeOneBooleanContent) {
    5905          12 :           if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
    5906           8 :                                                      N01CV, N1CV))
    5907          12 :             return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
    5908             :         }
    5909             :       }
    5910             :     }
    5911             :   }
    5912             : 
    5913      191903 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    5914             : 
    5915             :   // fold (shl c1, c2) -> c1<<c2
    5916             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    5917      195667 :   if (N0C && N1C && !N1C->isOpaque())
    5918       11292 :     return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
    5919             :   // fold (shl 0, x) -> 0
    5920      188139 :   if (isNullConstantOrNullSplatConstant(N0))
    5921          78 :     return N0;
    5922             :   // fold (shl x, c >= size(x)) -> undef
    5923             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    5924      178499 :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    5925      356998 :     return Val->getAPIntValue().uge(OpSizeInBits);
    5926             :   };
    5927      376122 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    5928          46 :     return DAG.getUNDEF(VT);
    5929             :   // fold (shl x, 0) -> x
    5930      366001 :   if (N1C && N1C->isNullValue())
    5931         436 :     return N0;
    5932             :   // fold (shl undef, x) -> 0
    5933      187579 :   if (N0.isUndef())
    5934          72 :     return DAG.getConstant(0, SDLoc(N), VT);
    5935             : 
    5936      187555 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    5937       25231 :     return NewSel;
    5938             : 
    5939             :   // if (shl x, c) is known to be zero, return 0
    5940      486972 :   if (DAG.MaskedValueIsZero(SDValue(N, 0),
    5941      324648 :                             APInt::getAllOnesValue(OpSizeInBits)))
    5942        6351 :     return DAG.getConstant(0, SDLoc(N), VT);
    5943             :   // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
    5944      162679 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    5945        2472 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    5946         327 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    5947         258 :       return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
    5948             :   }
    5949             : 
    5950      315734 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    5951        5435 :     return SDValue(N, 0);
    5952             : 
    5953             :   // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
    5954      154686 :   if (N0.getOpcode() == ISD::SHL) {
    5955             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    5956         186 :                                           ConstantSDNode *RHS) {
    5957          93 :       APInt c1 = LHS->getAPIntValue();
    5958          93 :       APInt c2 = RHS->getAPIntValue();
    5959          93 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    5960         372 :       return (c1 + c2).uge(OpSizeInBits);
    5961             :     };
    5962         686 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    5963          15 :       return DAG.getConstant(0, SDLoc(N), VT);
    5964             : 
    5965             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    5966         196 :                                        ConstantSDNode *RHS) {
    5967          98 :       APInt c1 = LHS->getAPIntValue();
    5968          98 :       APInt c2 = RHS->getAPIntValue();
    5969          98 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    5970         392 :       return (c1 + c2).ult(OpSizeInBits);
    5971             :     };
    5972         676 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    5973             :       SDLoc DL(N);
    5974          77 :       EVT ShiftVT = N1.getValueType();
    5975         154 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    5976         154 :       return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
    5977             :     }
    5978             :   }
    5979             : 
    5980             :   // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
    5981             :   // For this to be valid, the second form must not preserve any of the bits
    5982             :   // that are shifted out by the inner shift in the first form.  This means
    5983             :   // the outer shift size must be >= the number of bits added by the ext.
    5984             :   // As a corollary, we don't care what kind of ext it is.
    5985      144667 :   if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
    5986       57852 :               N0.getOpcode() == ISD::ANY_EXTEND ||
    5987      244832 :               N0.getOpcode() == ISD::SIGN_EXTEND) &&
    5988       90228 :       N0.getOperand(0).getOpcode() == ISD::SHL) {
    5989          38 :     SDValue N0Op0 = N0.getOperand(0);
    5990          38 :     if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
    5991          30 :       APInt c1 = N0Op0C1->getAPIntValue();
    5992          30 :       APInt c2 = N1C->getAPIntValue();
    5993          30 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    5994             : 
    5995          30 :       EVT InnerShiftVT = N0Op0.getValueType();
    5996          30 :       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
    5997          60 :       if (c2.uge(OpSizeInBits - InnerShiftSize)) {
    5998             :         SDLoc DL(N0);
    5999          13 :         APInt Sum = c1 + c2;
    6000          13 :         if (Sum.uge(OpSizeInBits))
    6001           4 :           return DAG.getConstant(0, DL, VT);
    6002             : 
    6003           9 :         return DAG.getNode(
    6004             :             ISD::SHL, DL, VT,
    6005          18 :             DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
    6006          18 :             DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
    6007             :       }
    6008             :     }
    6009             :   }
    6010             : 
    6011             :   // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
    6012             :   // Only fold this if the inner zext has no other uses to avoid increasing
    6013             :   // the total number of instructions.
    6014      464539 :   if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
    6015       82240 :       N0.getOperand(0).getOpcode() == ISD::SRL) {
    6016             :     SDValue N0Op0 = N0.getOperand(0);
    6017          77 :     if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6018         154 :       if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
    6019          77 :         uint64_t c1 = N0Op0C1->getZExtValue();
    6020          77 :         uint64_t c2 = N1C->getZExtValue();
    6021          77 :         if (c1 == c2) {
    6022          13 :           SDValue NewOp0 = N0.getOperand(0);
    6023          26 :           EVT CountVT = NewOp0.getOperand(1).getValueType();
    6024             :           SDLoc DL(N);
    6025          13 :           SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
    6026             :                                        NewOp0,
    6027          26 :                                        DAG.getConstant(c2, DL, CountVT));
    6028          13 :           AddToWorklist(NewSHL.getNode());
    6029          39 :           return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
    6030             :         }
    6031             :       }
    6032             :     }
    6033             :   }
    6034             : 
    6035             :   // fold (shl (sr[la] exact X,  C1), C2) -> (shl    X, (C2-C1)) if C1 <= C2
    6036             :   // fold (shl (sr[la] exact X,  C1), C2) -> (sr[la] X, (C2-C1)) if C1  > C2
    6037      301670 :   if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
    6038        2451 :       N0->getFlags().hasExact()) {
    6039          97 :     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
    6040          97 :       uint64_t C1 = N0C1->getZExtValue();
    6041          97 :       uint64_t C2 = N1C->getZExtValue();
    6042             :       SDLoc DL(N);
    6043          97 :       if (C1 <= C2)
    6044          71 :         return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
    6045         142 :                            DAG.getConstant(C2 - C1, DL, N1.getValueType()));
    6046          26 :       return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
    6047          52 :                          DAG.getConstant(C1 - C2, DL, N1.getValueType()));
    6048             :     }
    6049             :   }
    6050             : 
    6051             :   // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
    6052             :   //                               (and (srl x, (sub c1, c2), MASK)
    6053             :   // Only fold this if the inner shift has no other uses -- if it does, folding
    6054             :   // this will increase the total number of instructions.
    6055      300661 :   if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    6056        1204 :     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
    6057        1052 :       uint64_t c1 = N0C1->getZExtValue();
    6058        1052 :       if (c1 < OpSizeInBits) {
    6059        1052 :         uint64_t c2 = N1C->getZExtValue();
    6060        1052 :         APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
    6061        1052 :         SDValue Shift;
    6062        1052 :         if (c2 > c1) {
    6063         102 :           Mask <<= c2 - c1;
    6064             :           SDLoc DL(N);
    6065         204 :           Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
    6066         204 :                               DAG.getConstant(c2 - c1, DL, N1.getValueType()));
    6067             :         } else {
    6068         950 :           Mask.lshrInPlace(c1 - c2);
    6069             :           SDLoc DL(N);
    6070        1900 :           Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
    6071        1900 :                               DAG.getConstant(c1 - c2, DL, N1.getValueType()));
    6072             :         }
    6073             :         SDLoc DL(N0);
    6074        1052 :         return DAG.getNode(ISD::AND, DL, VT, Shift,
    6075        1052 :                            DAG.getConstant(Mask, DL, VT));
    6076             :       }
    6077             :     }
    6078             :   }
    6079             : 
    6080             :   // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
    6081      153435 :   if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
    6082           6 :       isConstantOrConstantVector(N1, /* No Opaques */ true)) {
    6083             :     SDLoc DL(N);
    6084           6 :     SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
    6085          12 :     SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
    6086          12 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
    6087             :   }
    6088             : 
    6089             :   // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
    6090             :   // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
    6091             :   // Variant of version done on multiply, except mul by a power of 2 is turned
    6092             :   // into a shift.
    6093      150807 :   if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
    6094        2871 :       N0.getNode()->hasOneUse() &&
    6095      159143 :       isConstantOrConstantVector(N1, /* No Opaques */ true) &&
    6096        2849 :       isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
    6097        2760 :     SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
    6098        2760 :     SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
    6099         920 :     AddToWorklist(Shl0.getNode());
    6100         920 :     AddToWorklist(Shl1.getNode());
    6101        2760 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
    6102             :   }
    6103             : 
    6104             :   // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
    6105         102 :   if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
    6106      152703 :       isConstantOrConstantVector(N1, /* No Opaques */ true) &&
    6107          98 :       isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
    6108          78 :     SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
    6109          26 :     if (isConstantOrConstantVector(Shl))
    6110          78 :       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
    6111             :   }
    6112             : 
    6113      295029 :   if (N1C && !N1C->isOpaque())
    6114      142552 :     if (SDValue NewSHL = visitShiftByConstant(N, N1C))
    6115         208 :       return NewSHL;
    6116             : 
    6117      152269 :   return SDValue();
    6118             : }
    6119             : 
    6120       25700 : SDValue DAGCombiner::visitSRA(SDNode *N) {
    6121       25700 :   SDValue N0 = N->getOperand(0);
    6122       25700 :   SDValue N1 = N->getOperand(1);
    6123       25700 :   EVT VT = N0.getValueType();
    6124             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6125             : 
    6126             :   // Arithmetic shifting an all-sign-bit value is a no-op.
    6127             :   // fold (sra 0, x) -> 0
    6128             :   // fold (sra -1, x) -> -1
    6129       25700 :   if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
    6130         445 :     return N0;
    6131             : 
    6132             :   // fold vector ops
    6133       25255 :   if (VT.isVector())
    6134        2459 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6135           3 :       return FoldedVOp;
    6136             : 
    6137       25252 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6138             : 
    6139             :   // fold (sra c1, c2) -> (sra c1, c2)
    6140             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6141       25252 :   if (N0C && N1C && !N1C->isOpaque())
    6142           0 :     return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
    6143             :   // fold (sra x, c >= size(x)) -> undef
    6144             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6145       23137 :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6146       46274 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6147             :   };
    6148       50504 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6149          23 :     return DAG.getUNDEF(VT);
    6150             :   // fold (sra x, 0) -> x
    6151       48142 :   if (N1C && N1C->isNullValue())
    6152          15 :     return N0;
    6153             : 
    6154       25214 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6155           2 :     return NewSel;
    6156             : 
    6157             :   // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
    6158             :   // sext_inreg.
    6159       48108 :   if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
    6160        4392 :     unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
    6161        2196 :     EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
    6162        2196 :     if (VT.isVector())
    6163         139 :       ExtVT = EVT::getVectorVT(*DAG.getContext(),
    6164         139 :                                ExtVT, VT.getVectorNumElements());
    6165        2196 :     if ((!LegalOperations ||
    6166        1649 :          TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
    6167        1160 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    6168        1740 :                          N0.getOperand(0), DAG.getValueType(ExtVT));
    6169             :   }
    6170             : 
    6171             :   // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
    6172       24632 :   if (N0.getOpcode() == ISD::SRA) {
    6173             :     SDLoc DL(N);
    6174         286 :     EVT ShiftVT = N1.getValueType();
    6175             : 
    6176             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    6177         540 :                                           ConstantSDNode *RHS) {
    6178         270 :       APInt c1 = LHS->getAPIntValue();
    6179         270 :       APInt c2 = RHS->getAPIntValue();
    6180         270 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6181        1080 :       return (c1 + c2).uge(OpSizeInBits);
    6182             :     };
    6183         572 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    6184         243 :       return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
    6185         486 :                          DAG.getConstant(OpSizeInBits - 1, DL, ShiftVT));
    6186             : 
    6187             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    6188          68 :                                        ConstantSDNode *RHS) {
    6189          34 :       APInt c1 = LHS->getAPIntValue();
    6190          34 :       APInt c2 = RHS->getAPIntValue();
    6191          34 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6192         136 :       return (c1 + c2).ult(OpSizeInBits);
    6193             :     };
    6194          86 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    6195          20 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    6196          20 :       return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), Sum);
    6197             :     }
    6198             :   }
    6199             : 
    6200             :   // fold (sra (shl X, m), (sub result_size, n))
    6201             :   // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
    6202             :   // result_size - n != m.
    6203             :   // If truncate is free for the target sext(shl) is likely to result in better
    6204             :   // code.
    6205       24379 :   if (N0.getOpcode() == ISD::SHL && N1C) {
    6206             :     // Get the two constanst of the shifts, CN0 = m, CN = n.
    6207        3207 :     const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
    6208        3207 :     if (N01C) {
    6209        3207 :       LLVMContext &Ctx = *DAG.getContext();
    6210             :       // Determine what the truncate's result bitsize and type would be.
    6211        6414 :       EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue());
    6212             : 
    6213        3207 :       if (VT.isVector())
    6214          45 :         TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorNumElements());
    6215             : 
    6216             :       // Determine the residual right-shift amount.
    6217        9621 :       int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
    6218             : 
    6219             :       // If the shift is not a no-op (in which case this should be just a sign
    6220             :       // extend already), the truncated to type is legal, sign_extend is legal
    6221             :       // on that type, and the truncate to that type is both legal and free,
    6222             :       // perform the transform.
    6223             :       if ((ShiftAmt > 0) &&
    6224        1556 :           TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
    6225        3385 :           TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
    6226          89 :           TLI.isTruncateFree(VT, TruncVT)) {
    6227             :         SDLoc DL(N);
    6228          13 :         SDValue Amt = DAG.getConstant(ShiftAmt, DL,
    6229          26 :             getShiftAmountTy(N0.getOperand(0).getValueType()));
    6230          13 :         SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
    6231          13 :                                     N0.getOperand(0), Amt);
    6232          13 :         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
    6233          13 :                                     Shift);
    6234          13 :         return DAG.getNode(ISD::SIGN_EXTEND, DL,
    6235          26 :                            N->getValueType(0), Trunc);
    6236             :       }
    6237             :     }
    6238             :   }
    6239             : 
    6240             :   // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
    6241       24671 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6242         305 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6243          18 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6244          45 :       return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
    6245             :   }
    6246             : 
    6247             :   // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
    6248             :   //      if c1 is equal to the number of bits the trunc removes
    6249        5541 :   if (N0.getOpcode() == ISD::TRUNCATE &&
    6250        8831 :       (N0.getOperand(0).getOpcode() == ISD::SRL ||
    6251        2266 :        N0.getOperand(0).getOpcode() == ISD::SRA) &&
    6252        3998 :       N0.getOperand(0).hasOneUse() &&
    6253       27925 :       N0.getOperand(0).getOperand(1).hasOneUse() &&
    6254             :       N1C) {
    6255         104 :     SDValue N0Op0 = N0.getOperand(0);
    6256         104 :     if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6257         208 :       unsigned LargeShiftVal = LargeShift->getZExtValue();
    6258         104 :       EVT LargeVT = N0Op0.getValueType();
    6259             : 
    6260         104 :       if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) {
    6261             :         SDLoc DL(N);
    6262             :         SDValue Amt =
    6263         196 :           DAG.getConstant(LargeShiftVal + N1C->getZExtValue(), DL,
    6264         294 :                           getShiftAmountTy(N0Op0.getOperand(0).getValueType()));
    6265          98 :         SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT,
    6266          98 :                                   N0Op0.getOperand(0), Amt);
    6267         196 :         return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
    6268             :       }
    6269             :     }
    6270             :   }
    6271             : 
    6272             :   // Simplify, based on bits shifted out of the LHS.
    6273       47247 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6274        1036 :     return SDValue(N, 0);
    6275             : 
    6276             :   // If the sign bit is known to be zero, switch this to a SRL.
    6277       23217 :   if (DAG.SignBitIsZero(N0))
    6278         348 :     return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
    6279             : 
    6280       44023 :   if (N1C && !N1C->isOpaque())
    6281       20922 :     if (SDValue NewSRA = visitShiftByConstant(N, N1C))
    6282           0 :       return NewSRA;
    6283             : 
    6284       23101 :   return SDValue();
    6285             : }
    6286             : 
    6287      136433 : SDValue DAGCombiner::visitSRL(SDNode *N) {
    6288      136433 :   SDValue N0 = N->getOperand(0);
    6289      136433 :   SDValue N1 = N->getOperand(1);
    6290      136433 :   EVT VT = N0.getValueType();
    6291             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6292             : 
    6293             :   // fold vector ops
    6294      136433 :   if (VT.isVector())
    6295        3929 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6296           3 :       return FoldedVOp;
    6297             : 
    6298      136430 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6299             : 
    6300             :   // fold (srl c1, c2) -> c1 >>u c2
    6301             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6302      139067 :   if (N0C && N1C && !N1C->isOpaque())
    6303        7911 :     return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
    6304             :   // fold (srl 0, x) -> 0
    6305      133793 :   if (isNullConstantOrNullSplatConstant(N0))
    6306          55 :     return N0;
    6307             :   // fold (srl x, c >= size(x)) -> undef
    6308             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6309      125202 :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6310      250404 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6311             :   };
    6312      267476 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6313          23 :     return DAG.getUNDEF(VT);
    6314             :   // fold (srl x, 0) -> x
    6315      258566 :   if (N1C && N1C->isNullValue())
    6316        2060 :     return N0;
    6317             : 
    6318      131655 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6319           3 :     return NewSel;
    6320             : 
    6321             :   // if (srl x, c) is known to be zero, return 0
    6322      386092 :   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
    6323      386048 :                                    APInt::getAllOnesValue(OpSizeInBits)))
    6324         132 :     return DAG.getConstant(0, SDLoc(N), VT);
    6325             : 
    6326             :   // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
    6327      131608 :   if (N0.getOpcode() == ISD::SRL) {
    6328             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    6329        8860 :                                           ConstantSDNode *RHS) {
    6330        4430 :       APInt c1 = LHS->getAPIntValue();
    6331        4430 :       APInt c2 = RHS->getAPIntValue();
    6332        4430 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6333       17720 :       return (c1 + c2).uge(OpSizeInBits);
    6334             :     };
    6335       10564 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    6336          15 :       return DAG.getConstant(0, SDLoc(N), VT);
    6337             : 
    6338             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    6339        8870 :                                        ConstantSDNode *RHS) {
    6340        4435 :       APInt c1 = LHS->getAPIntValue();
    6341        4435 :       APInt c2 = RHS->getAPIntValue();
    6342        4435 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6343       17740 :       return (c1 + c2).ult(OpSizeInBits);
    6344             :     };
    6345       10554 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    6346             :       SDLoc DL(N);
    6347        4414 :       EVT ShiftVT = N1.getValueType();
    6348        8828 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    6349        8828 :       return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
    6350             :     }
    6351             :   }
    6352             : 
    6353             :   // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
    6354      258368 :   if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
    6355       12848 :       N0.getOperand(0).getOpcode() == ISD::SRL) {
    6356        4534 :     if (auto N001C = isConstOrConstSplat(N0.getOperand(0).getOperand(1))) {
    6357        4534 :       uint64_t c1 = N001C->getZExtValue();
    6358        4534 :       uint64_t c2 = N1C->getZExtValue();
    6359        9068 :       EVT InnerShiftVT = N0.getOperand(0).getValueType();
    6360        9068 :       EVT ShiftCountVT = N0.getOperand(0).getOperand(1).getValueType();
    6361        4534 :       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
    6362             :       // This is only valid if the OpSizeInBits + c1 = size of inner shift.
    6363        4534 :       if (c1 + OpSizeInBits == InnerShiftSize) {
    6364             :         SDLoc DL(N0);
    6365        3082 :         if (c1 + c2 >= InnerShiftSize)
    6366           0 :           return DAG.getConstant(0, DL, VT);
    6367        3082 :         return DAG.getNode(ISD::TRUNCATE, DL, VT,
    6368             :                            DAG.getNode(ISD::SRL, DL, InnerShiftVT,
    6369        3082 :                                        N0.getOperand(0).getOperand(0),
    6370             :                                        DAG.getConstant(c1 + c2, DL,
    6371        6164 :                                                        ShiftCountVT)));
    6372             :       }
    6373             :     }
    6374             :   }
    6375             : 
    6376             :   // fold (srl (shl x, c), c) -> (and x, cst2)
    6377      124523 :   if (N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
    6378         416 :       isConstantOrConstantVector(N1, /* NoOpaques */ true)) {
    6379             :     SDLoc DL(N);
    6380             :     SDValue Mask =
    6381         440 :         DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
    6382         220 :     AddToWorklist(Mask.getNode());
    6383         440 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask);
    6384             :   }
    6385             : 
    6386             :   // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
    6387      238919 :   if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    6388             :     // Shifting in all undef bits?
    6389         656 :     EVT SmallVT = N0.getOperand(0).getValueType();
    6390             :     unsigned BitSize = SmallVT.getScalarSizeInBits();
    6391         656 :     if (N1C->getZExtValue() >= BitSize)
    6392         121 :       return DAG.getUNDEF(VT);
    6393             : 
    6394         328 :     if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
    6395         121 :       uint64_t ShiftAmt = N1C->getZExtValue();
    6396             :       SDLoc DL0(N0);
    6397         121 :       SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
    6398             :                                        N0.getOperand(0),
    6399             :                           DAG.getConstant(ShiftAmt, DL0,
    6400         242 :                                           getShiftAmountTy(SmallVT)));
    6401         121 :       AddToWorklist(SmallShift.getNode());
    6402         121 :       APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt);
    6403             :       SDLoc DL(N);
    6404         121 :       return DAG.getNode(ISD::AND, DL, VT,
    6405         121 :                          DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
    6406         242 :                          DAG.getConstant(Mask, DL, VT));
    6407             :     }
    6408             :   }
    6409             : 
    6410             :   // fold (srl (sra X, Y), 31) -> (srl X, 31).  This srl only looks at the sign
    6411             :   // bit, which is unmodified by sra.
    6412      238677 :   if (N1C && N1C->getZExtValue() + 1 == OpSizeInBits) {
    6413        3440 :     if (N0.getOpcode() == ISD::SRA)
    6414         876 :       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
    6415             :   }
    6416             : 
    6417             :   // fold (srl (ctlz x), "5") -> x  iff x has one bit set (the low bit).
    6418      238253 :   if (N1C && N0.getOpcode() == ISD::CTLZ &&
    6419         320 :       N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
    6420         157 :     KnownBits Known;
    6421         320 :     DAG.computeKnownBits(N0.getOperand(0), Known);
    6422             : 
    6423             :     // If any of the input bits are KnownOne, then the input couldn't be all
    6424             :     // zeros, thus the result of the srl will always be zero.
    6425         160 :     if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
    6426             : 
    6427             :     // If all of the bits input the to ctlz node are known to be zero, then
    6428             :     // the result of the ctlz is "32" and the result of the shift is one.
    6429         160 :     APInt UnknownBits = ~Known.Zero;
    6430         160 :     if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
    6431             : 
    6432             :     // Otherwise, check to see if there is exactly one bit input to the ctlz.
    6433         160 :     if (UnknownBits.isPowerOf2()) {
    6434             :       // Okay, we know that only that the single bit specified by UnknownBits
    6435             :       // could be set on input to the CTLZ node. If this bit is set, the SRL
    6436             :       // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
    6437             :       // to an SRL/XOR pair, which is likely to simplify more.
    6438           3 :       unsigned ShAmt = UnknownBits.countTrailingZeros();
    6439           3 :       SDValue Op = N0.getOperand(0);
    6440             : 
    6441           3 :       if (ShAmt) {
    6442             :         SDLoc DL(N0);
    6443           6 :         Op = DAG.getNode(ISD::SRL, DL, VT, Op,
    6444             :                   DAG.getConstant(ShAmt, DL,
    6445           6 :                                   getShiftAmountTy(Op.getValueType())));
    6446           3 :         AddToWorklist(Op.getNode());
    6447             :       }
    6448             : 
    6449             :       SDLoc DL(N);
    6450           3 :       return DAG.getNode(ISD::XOR, DL, VT,
    6451           3 :                          Op, DAG.getConstant(1, DL, VT));
    6452             :     }
    6453             :   }
    6454             : 
    6455             :   // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
    6456      124717 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6457        1246 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6458          47 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6459          93 :       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
    6460             :   }
    6461             : 
    6462             :   // fold operands of srl based on knowledge that the low bits are not
    6463             :   // demanded.
    6464      245863 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6465        7807 :     return SDValue(N, 0);
    6466             : 
    6467      222442 :   if (N1C && !N1C->isOpaque())
    6468      106809 :     if (SDValue NewSRL = visitShiftByConstant(N, N1C))
    6469         601 :       return NewSRL;
    6470             : 
    6471             :   // Attempt to convert a srl of a load into a narrower zero-extending load.
    6472      115032 :   if (SDValue NarrowLoad = ReduceLoadWidth(N))
    6473         121 :     return NarrowLoad;
    6474             : 
    6475             :   // Here is a common situation. We want to optimize:
    6476             :   //
    6477             :   //   %a = ...
    6478             :   //   %b = and i32 %a, 2
    6479             :   //   %c = srl i32 %b, 1
    6480             :   //   brcond i32 %c ...
    6481             :   //
    6482             :   // into
    6483             :   //
    6484             :   //   %a = ...
    6485             :   //   %b = and %a, 2
    6486             :   //   %c = setcc eq %b, 0
    6487             :   //   brcond %c ...
    6488             :   //
    6489             :   // However when after the source operand of SRL is optimized into AND, the SRL
    6490             :   // itself may not be optimized further. Look for it and add the BRCOND into
    6491             :   // the worklist.
    6492             :   if (N->hasOneUse()) {
    6493             :     SDNode *Use = *N->use_begin();
    6494      217656 :     if (Use->getOpcode() == ISD::BRCOND)
    6495           4 :       AddToWorklist(Use);
    6496      108824 :     else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
    6497             :       // Also look pass the truncate.
    6498             :       Use = *Use->use_begin();
    6499       28067 :       if (Use->getOpcode() == ISD::BRCOND)
    6500           2 :         AddToWorklist(Use);
    6501             :     }
    6502             :   }
    6503             : 
    6504      114911 :   return SDValue();
    6505             : }
    6506             : 
    6507        1146 : SDValue DAGCombiner::visitABS(SDNode *N) {
    6508        1146 :   SDValue N0 = N->getOperand(0);
    6509        2292 :   EVT VT = N->getValueType(0);
    6510             : 
    6511             :   // fold (abs c1) -> c2
    6512        1146 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6513           0 :     return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0);
    6514             :   // fold (abs (abs x)) -> (abs x)
    6515        1146 :   if (N0.getOpcode() == ISD::ABS)
    6516          11 :     return N0;
    6517             :   // fold (abs x) -> x iff not-negative
    6518        1135 :   if (DAG.SignBitIsZero(N0))
    6519           8 :     return N0;
    6520        1127 :   return SDValue();
    6521             : }
    6522             : 
    6523        1408 : SDValue DAGCombiner::visitBSWAP(SDNode *N) {
    6524        1408 :   SDValue N0 = N->getOperand(0);
    6525        2816 :   EVT VT = N->getValueType(0);
    6526             : 
    6527             :   // fold (bswap c1) -> c2
    6528        1408 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6529           0 :     return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
    6530             :   // fold (bswap (bswap x)) -> x
    6531        1408 :   if (N0.getOpcode() == ISD::BSWAP)
    6532          28 :     return N0->getOperand(0);
    6533        1380 :   return SDValue();
    6534             : }
    6535             : 
    6536         543 : SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
    6537         543 :   SDValue N0 = N->getOperand(0);
    6538        1086 :   EVT VT = N->getValueType(0);
    6539             : 
    6540             :   // fold (bitreverse c1) -> c2
    6541         543 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6542           0 :     return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0);
    6543             :   // fold (bitreverse (bitreverse x)) -> x
    6544         543 :   if (N0.getOpcode() == ISD::BITREVERSE)
    6545           4 :     return N0.getOperand(0);
    6546         539 :   return SDValue();
    6547             : }
    6548             : 
    6549        1198 : SDValue DAGCombiner::visitCTLZ(SDNode *N) {
    6550        1198 :   SDValue N0 = N->getOperand(0);
    6551        2396 :   EVT VT = N->getValueType(0);
    6552             : 
    6553             :   // fold (ctlz c1) -> c2
    6554        1198 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6555           6 :     return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
    6556             : 
    6557             :   // If the value is known never to be zero, switch to the undef version.
    6558        1196 :   if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) {
    6559         404 :     if (DAG.isKnownNeverZero(N0))
    6560          21 :       return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    6561             :   }
    6562             : 
    6563        1189 :   return SDValue();
    6564             : }
    6565             : 
    6566         527 : SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
    6567         527 :   SDValue N0 = N->getOperand(0);
    6568        1054 :   EVT VT = N->getValueType(0);
    6569             : 
    6570             :   // fold (ctlz_zero_undef c1) -> c2
    6571         527 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6572           0 :     return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    6573         527 :   return SDValue();
    6574             : }
    6575             : 
    6576         467 : SDValue DAGCombiner::visitCTTZ(SDNode *N) {
    6577         467 :   SDValue N0 = N->getOperand(0);
    6578         934 :   EVT VT = N->getValueType(0);
    6579             : 
    6580             :   // fold (cttz c1) -> c2
    6581         467 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6582           0 :     return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
    6583             : 
    6584             :   // If the value is known never to be zero, switch to the undef version.
    6585         467 :   if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) {
    6586         317 :     if (DAG.isKnownNeverZero(N0))
    6587          42 :       return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    6588             :   }
    6589             : 
    6590         453 :   return SDValue();
    6591             : }
    6592             : 
    6593         557 : SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
    6594         557 :   SDValue N0 = N->getOperand(0);
    6595        1114 :   EVT VT = N->getValueType(0);
    6596             : 
    6597             :   // fold (cttz_zero_undef c1) -> c2
    6598         557 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6599           0 :     return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    6600         557 :   return SDValue();
    6601             : }
    6602             : 
    6603        1957 : SDValue DAGCombiner::visitCTPOP(SDNode *N) {
    6604        1957 :   SDValue N0 = N->getOperand(0);
    6605        3914 :   EVT VT = N->getValueType(0);
    6606             : 
    6607             :   // fold (ctpop c1) -> c2
    6608        1957 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6609           0 :     return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
    6610        1957 :   return SDValue();
    6611             : }
    6612             : 
    6613             : /// Generate Min/Max node
    6614          98 : static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
    6615             :                                    SDValue RHS, SDValue True, SDValue False,
    6616             :                                    ISD::CondCode CC, const TargetLowering &TLI,
    6617             :                                    SelectionDAG &DAG) {
    6618             :   if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
    6619          15 :     return SDValue();
    6620             : 
    6621             :   switch (CC) {
    6622             :   case ISD::SETOLT:
    6623             :   case ISD::SETOLE:
    6624             :   case ISD::SETLT:
    6625             :   case ISD::SETLE:
    6626             :   case ISD::SETULT:
    6627             :   case ISD::SETULE: {
    6628             :     unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
    6629             :     if (TLI.isOperationLegal(Opcode, VT))
    6630          15 :       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    6631          28 :     return SDValue();
    6632             :   }
    6633             :   case ISD::SETOGT:
    6634             :   case ISD::SETOGE:
    6635             :   case ISD::SETGT:
    6636             :   case ISD::SETGE:
    6637             :   case ISD::SETUGT:
    6638             :   case ISD::SETUGE: {
    6639             :     unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
    6640             :     if (TLI.isOperationLegal(Opcode, VT))
    6641          12 :       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    6642          28 :     return SDValue();
    6643             :   }
    6644           0 :   default:
    6645           0 :     return SDValue();
    6646             :   }
    6647             : }
    6648             : 
    6649       50757 : SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
    6650       50757 :   SDValue Cond = N->getOperand(0);
    6651       50757 :   SDValue N1 = N->getOperand(1);
    6652       50757 :   SDValue N2 = N->getOperand(2);
    6653      101514 :   EVT VT = N->getValueType(0);
    6654       50757 :   EVT CondVT = Cond.getValueType();
    6655             :   SDLoc DL(N);
    6656             : 
    6657       50757 :   if (!VT.isInteger())
    6658        5623 :     return SDValue();
    6659             : 
    6660             :   auto *C1 = dyn_cast<ConstantSDNode>(N1);
    6661             :   auto *C2 = dyn_cast<ConstantSDNode>(N2);
    6662       45134 :   if (!C1 || !C2)
    6663       15568 :     return SDValue();
    6664             : 
    6665             :   // Only do this before legalization to avoid conflicting with target-specific
    6666             :   // transforms in the other direction (create a select from a zext/sext). There
    6667             :   // is also a target-independent combine here in DAGCombiner in the other
    6668             :   // direction for (select Cond, -1, 0) when the condition is not i1.
    6669       28314 :   if (CondVT == MVT::i1 && !LegalOperations) {
    6670       52826 :     if (C1->isNullValue() && C2->isOne()) {
    6671             :       // select Cond, 0, 1 --> zext (!Cond)
    6672         226 :       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    6673             :       if (VT != MVT::i1)
    6674          64 :         NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond);
    6675         113 :       return NotCond;
    6676             :     }
    6677       26284 :     if (C1->isNullValue() && C2->isAllOnesValue()) {
    6678             :       // select Cond, 0, -1 --> sext (!Cond)
    6679          50 :       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    6680             :       if (VT != MVT::i1)
    6681          50 :         NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond);
    6682          25 :       return NotCond;
    6683             :     }
    6684       26325 :     if (C1->isOne() && C2->isNullValue()) {
    6685             :       // select Cond, 1, 0 --> zext (Cond)
    6686             :       if (VT != MVT::i1)
    6687         106 :         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
    6688          53 :       return Cond;
    6689             :     }
    6690       26237 :     if (C1->isAllOnesValue() && C2->isNullValue()) {
    6691             :       // select Cond, -1, 0 --> sext (Cond)
    6692             :       if (VT != MVT::i1)
    6693         114 :         Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
    6694          57 :       return Cond;
    6695             :     }
    6696             : 
    6697             :     // For any constants that differ by 1, we can transform the select into an
    6698             :     // extend and add. Use a target hook because some targets may prefer to
    6699             :     // transform in the other direction.
    6700       26068 :     if (TLI.convertSelectOfConstantsToMath(VT)) {
    6701      102608 :       if (C1->getAPIntValue() - 1 == C2->getAPIntValue()) {
    6702             :         // select Cond, C1, C1-1 --> add (zext Cond), C1-1
    6703             :         if (VT != MVT::i1)
    6704          44 :           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
    6705          44 :         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
    6706             :       }
    6707      102520 :       if (C1->getAPIntValue() + 1 == C2->getAPIntValue()) {
    6708             :         // select Cond, C1, C1+1 --> add (sext Cond), C1+1
    6709             :         if (VT != MVT::i1)
    6710         208 :           Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
    6711         208 :         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
    6712             :       }
    6713             :     }
    6714             : 
    6715       25942 :     return SDValue();
    6716             :   }
    6717             : 
    6718             :   // fold (select Cond, 0, 1) -> (xor Cond, 1)
    6719             :   // We can't do this reliably if integer based booleans have different contents
    6720             :   // to floating point based booleans. This is because we can't tell whether we
    6721             :   // have an integer-based boolean or a floating-point-based boolean unless we
    6722             :   // can find the SETCC that produced it and inspect its operands. This is
    6723             :   // fairly easy if C is the SETCC node, but it can potentially be
    6724             :   // undiscoverable (or not reasonably discoverable). For example, it could be
    6725             :   // in another basic block or it could require searching a complicated
    6726             :   // expression.
    6727        6500 :   if (CondVT.isInteger() &&
    6728        3250 :       TLI.getBooleanContents(false, true) ==
    6729        1105 :           TargetLowering::ZeroOrOneBooleanContent &&
    6730             :       TLI.getBooleanContents(false, false) ==
    6731        1105 :           TargetLowering::ZeroOrOneBooleanContent &&
    6732        4997 :       C1->isNullValue() && C2->isOne()) {
    6733             :     SDValue NotCond =
    6734           0 :         DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
    6735           0 :     if (VT.bitsEq(CondVT))
    6736           0 :       return NotCond;
    6737           0 :     return DAG.getZExtOrTrunc(NotCond, DL, VT);
    6738             :   }
    6739             : 
    6740        3250 :   return SDValue();
    6741             : }
    6742             : 
    6743       50924 : SDValue DAGCombiner::visitSELECT(SDNode *N) {
    6744       50924 :   SDValue N0 = N->getOperand(0);
    6745       50924 :   SDValue N1 = N->getOperand(1);
    6746       50924 :   SDValue N2 = N->getOperand(2);
    6747      101848 :   EVT VT = N->getValueType(0);
    6748             :   EVT VT0 = N0.getValueType();
    6749             :   SDLoc DL(N);
    6750             : 
    6751             :   // fold (select C, X, X) -> X
    6752             :   if (N1 == N2)
    6753          63 :     return N1;
    6754             : 
    6755             :   if (const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
    6756             :     // fold (select true, X, Y) -> X
    6757             :     // fold (select false, X, Y) -> Y
    6758         154 :     return !N0C->isNullValue() ? N1 : N2;
    6759             :   }
    6760             : 
    6761             :   // fold (select X, X, Y) -> (or X, Y)
    6762             :   // fold (select X, 1, Y) -> (or C, Y)
    6763         173 :   if (VT == VT0 && VT == MVT::i1 && (N0 == N1 || isOneConstant(N1)))
    6764          54 :     return DAG.getNode(ISD::OR, DL, VT, N0, N2);
    6765             : 
    6766       50757 :   if (SDValue V = foldSelectOfConstants(N))
    6767         374 :     return V;
    6768             : 
    6769             :   // fold (select C, 0, X) -> (and (not C), X)
    6770          71 :   if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) {
    6771          18 :     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
    6772           6 :     AddToWorklist(NOTNode.getNode());
    6773          12 :     return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2);
    6774             :   }
    6775             :   // fold (select C, X, 1) -> (or (not C), X)
    6776          65 :   if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) {
    6777           6 :     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
    6778           2 :     AddToWorklist(NOTNode.getNode());
    6779           4 :     return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1);
    6780             :   }
    6781             :   // fold (select X, Y, X) -> (and X, Y)
    6782             :   // fold (select X, Y, 0) -> (and X, Y)
    6783          61 :   if (VT == VT0 && VT == MVT::i1 && (N0 == N2 || isNullConstant(N2)))
    6784           6 :     return DAG.getNode(ISD::AND, DL, VT, N0, N1);
    6785             : 
    6786             :   // If we can fold this based on the true/false value, do so.
    6787       50372 :   if (SimplifySelectOps(N, N1, N2))
    6788          86 :     return SDValue(N, 0); // Don't revisit N.
    6789             : 
    6790             :   if (VT0 == MVT::i1) {
    6791             :     // The code in this block deals with the following 2 equivalences:
    6792             :     //    select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
    6793             :     //    select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
    6794             :     // The target can specify its preferred form with the
    6795             :     // shouldNormalizeToSelectSequence() callback. However we always transform
    6796             :     // to the right anyway if we find the inner select exists in the DAG anyway
    6797             :     // and we always transform to the left side if we know that we can further
    6798             :     // optimize the combination of the conditions.
    6799             :     bool normalizeToSequence =
    6800       42814 :         TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
    6801             :     // select (and Cond0, Cond1), X, Y
    6802             :     //   -> select Cond0, (select Cond1, X, Y), Y
    6803       42814 :     if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
    6804         320 :       SDValue Cond0 = N0->getOperand(0);
    6805         320 :       SDValue Cond1 = N0->getOperand(1);
    6806             :       SDValue InnerSelect =
    6807         960 :           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
    6808         615 :       if (normalizeToSequence || !InnerSelect.use_empty())
    6809          25 :         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
    6810          25 :                            InnerSelect, N2);
    6811             :     }
    6812             :     // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
    6813       42789 :     if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
    6814         176 :       SDValue Cond0 = N0->getOperand(0);
    6815         176 :       SDValue Cond1 = N0->getOperand(1);
    6816             :       SDValue InnerSelect =
    6817         528 :           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
    6818         304 :       if (normalizeToSequence || !InnerSelect.use_empty())
    6819          49 :         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
    6820          49 :                            InnerSelect);
    6821             :     }
    6822             : 
    6823             :     // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
    6824       42740 :     if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
    6825         896 :       SDValue N1_0 = N1->getOperand(0);
    6826         896 :       SDValue N1_1 = N1->getOperand(1);
    6827         896 :       SDValue N1_2 = N1->getOperand(2);
    6828           0 :       if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
    6829             :         // Create the actual and node if we can generate good code for it.
    6830          55 :         if (!normalizeToSequence) {
    6831           0 :           SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
    6832           0 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, N2);
    6833             :         }
    6834             :         // Otherwise see if we can optimize the "and" to a better pattern.
    6835          55 :         if (SDValue Combined = visitANDLike(N0, N1_0, N))
    6836           4 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
    6837           8 :                              N2);
    6838             :       }
    6839             :     }
    6840             :     // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
    6841       42736 :     if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
    6842        1055 :       SDValue N2_0 = N2->getOperand(0);
    6843        1055 :       SDValue N2_1 = N2->getOperand(1);
    6844        1055 :       SDValue N2_2 = N2->getOperand(2);
    6845           0 :       if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
    6846             :         // Create the actual or node if we can generate good code for it.
    6847         110 :         if (!normalizeToSequence) {
    6848          34 :           SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
    6849          34 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, N2_2);
    6850             :         }
    6851             :         // Otherwise see if we can optimize to a better pattern.
    6852          93 :         if (SDValue Combined = visitORLike(N0, N2_0, N))
    6853           6 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
    6854           6 :                              N2_2);
    6855             :       }
    6856             :     }
    6857             :   }
    6858             : 
    6859             :   // select (xor Cond, 1), X, Y -> select Cond, Y, X
    6860             :   if (VT0 == MVT::i1) {
    6861       42713 :     if (N0->getOpcode() == ISD::XOR) {
    6862         298 :       if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1))) {
    6863         158 :         SDValue Cond0 = N0->getOperand(0);
    6864         316 :         if (C->isOne())
    6865         474 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N2, N1);
    6866             :       }
    6867             :     }
    6868             :   }
    6869             : 
    6870             :   // fold selects based on a setcc into other things, such as min/max/abs
    6871       50027 :   if (N0.getOpcode() == ISD::SETCC) {
    6872             :     // select x, y (fcmp lt x, y) -> fminnum x, y
    6873             :     // select x, y (fcmp gt x, y) -> fmaxnum x, y
    6874             :     //
    6875             :     // This is OK if we don't care about what happens if either operand is a
    6876             :     // NaN.
    6877             :     //
    6878             : 
    6879             :     // FIXME: Instead of testing for UnsafeFPMath, this should be checking for
    6880             :     // no signed zeros as well as no nans.
    6881       42108 :     const TargetOptions &Options = DAG.getTarget().Options;
    6882       42902 :     if (Options.UnsafeFPMath && VT.isFloatingPoint() && N0.hasOneUse() &&
    6883       42424 :         DAG.isKnownNeverNaN(N1) && DAG.isKnownNeverNaN(N2)) {
    6884          98 :       ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    6885             : 
    6886          98 :       if (SDValue FMinMax = combineMinNumMaxNum(
    6887         196 :               DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG))
    6888          27 :         return FMinMax;
    6889             :     }
    6890             : 
    6891       42081 :     if ((!LegalOperations &&
    6892       42081 :          TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) ||
    6893       38724 :         TLI.isOperationLegal(ISD::SELECT_CC, VT))
    6894        3374 :       return DAG.getNode(ISD::SELECT_CC, DL, VT, N0.getOperand(0),
    6895        3374 :                          N0.getOperand(1), N1, N2, N0.getOperand(2));
    6896       38707 :     return SimplifySelect(DL, N0, N1, N2);
    6897             :   }
    6898             : 
    6899        7919 :   return SDValue();
    6900             : }
    6901             : 
    6902             : static
    6903          19 : std::pair<SDValue, SDValue> SplitVSETCC(const SDNode *N, SelectionDAG &DAG) {
    6904             :   SDLoc DL(N);
    6905             :   EVT LoVT, HiVT;
    6906          57 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    6907             : 
    6908             :   // Split the inputs.
    6909             :   SDValue Lo, Hi, LL, LH, RL, RH;
    6910          38 :   std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
    6911          38 :   std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
    6912             : 
    6913          57 :   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
    6914          57 :   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
    6915             : 
    6916          19 :   return std::make_pair(Lo, Hi);
    6917             : }
    6918             : 
    6919             : // This function assumes all the vselect's arguments are CONCAT_VECTOR
    6920             : // nodes and that the condition is a BV of ConstantSDNodes (or undefs).
    6921          43 : static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
    6922             :   SDLoc DL(N);
    6923          43 :   SDValue Cond = N->getOperand(0);
    6924          43 :   SDValue LHS = N->getOperand(1);
    6925          43 :   SDValue RHS = N->getOperand(2);
    6926          86 :   EVT VT = N->getValueType(0);
    6927          43 :   int NumElems = VT.getVectorNumElements();
    6928             :   assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
    6929             :          RHS.getOpcode() == ISD::CONCAT_VECTORS &&
    6930             :          Cond.getOpcode() == ISD::BUILD_VECTOR);
    6931             : 
    6932             :   // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
    6933             :   // binary ones here.
    6934          43 :   if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
    6935          11 :     return SDValue();
    6936             : 
    6937             :   // We're sure we have an even number of elements due to the
    6938             :   // concat_vectors we have as arguments to vselect.
    6939             :   // Skip BV elements until we find one that's not an UNDEF
    6940             :   // After we find an UNDEF element, keep looping until we get to half the
    6941             :   // length of the BV and see if all the non-undef nodes are the same.
    6942             :   ConstantSDNode *BottomHalf = nullptr;
    6943         164 :   for (int i = 0; i < NumElems / 2; ++i) {
    6944         170 :     if (Cond->getOperand(i)->isUndef())
    6945           0 :       continue;
    6946             : 
    6947          85 :     if (BottomHalf == nullptr)
    6948             :       BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    6949          53 :     else if (Cond->getOperand(i).getNode() != BottomHalf)
    6950          19 :       return SDValue();
    6951             :   }
    6952             : 
    6953             :   // Do the same for the second half of the BuildVector
    6954             :   ConstantSDNode *TopHalf = nullptr;
    6955          83 :   for (int i = NumElems / 2; i < NumElems; ++i) {
    6956          76 :     if (Cond->getOperand(i)->isUndef())
    6957           0 :       continue;
    6958             : 
    6959          38 :     if (TopHalf == nullptr)
    6960             :       TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    6961          25 :     else if (Cond->getOperand(i).getNode() != TopHalf)
    6962           3 :       return SDValue();
    6963             :   }
    6964             : 
    6965             :   assert(TopHalf && BottomHalf &&
    6966             :          "One half of the selector was all UNDEFs and the other was all the "
    6967             :          "same value. This should have been addressed before this function.");
    6968             :   return DAG.getNode(
    6969             :       ISD::CONCAT_VECTORS, DL, VT,
    6970          22 :       BottomHalf->isNullValue() ? RHS->getOperand(0) : LHS->getOperand(0),
    6971          38 :       TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
    6972             : }
    6973             : 
    6974         197 : SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
    6975         197 :   if (Level >= AfterLegalizeTypes)
    6976          88 :     return SDValue();
    6977             : 
    6978             :   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
    6979         109 :   SDValue Mask = MSC->getMask();
    6980         109 :   SDValue Data  = MSC->getValue();
    6981             :   SDLoc DL(N);
    6982             : 
    6983             :   // If the MSCATTER data type requires splitting and the mask is provided by a
    6984             :   // SETCC, then split both nodes and its operands before legalization. This
    6985             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    6986             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    6987         109 :   if (Mask.getOpcode() != ISD::SETCC)
    6988          84 :     return SDValue();
    6989             : 
    6990             :   // Check if any splitting is required.
    6991          75 :   if (TLI.getTypeAction(*DAG.getContext(), Data.getValueType()) !=
    6992             :       TargetLowering::TypeSplitVector)
    6993          20 :     return SDValue();
    6994             :   SDValue MaskLo, MaskHi, Lo, Hi;
    6995          10 :   std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    6996             : 
    6997             :   EVT LoVT, HiVT;
    6998          15 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0));
    6999             : 
    7000           5 :   SDValue Chain = MSC->getChain();
    7001             : 
    7002           5 :   EVT MemoryVT = MSC->getMemoryVT();
    7003           5 :   unsigned Alignment = MSC->getOriginalAlignment();
    7004             : 
    7005           5 :   EVT LoMemVT, HiMemVT;
    7006          10 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7007             : 
    7008             :   SDValue DataLo, DataHi;
    7009          10 :   std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    7010             : 
    7011           5 :   SDValue Scale = MSC->getScale();
    7012           5 :   SDValue BasePtr = MSC->getBasePtr();
    7013             :   SDValue IndexLo, IndexHi;
    7014          15 :   std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->getIndex(), DL);
    7015             : 
    7016           5 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    7017          10 :     getMachineMemOperand(MSC->getPointerInfo(),
    7018             :                           MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
    7019           5 :                           Alignment, MSC->getAAInfo(), MSC->getRanges());
    7020             : 
    7021           5 :   SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo, Scale };
    7022          10 :   Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
    7023          15 :                             DL, OpsLo, MMO);
    7024             : 
    7025           5 :   SDValue OpsHi[] = { Chain, DataHi, MaskHi, BasePtr, IndexHi, Scale };
    7026          10 :   Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
    7027          15 :                             DL, OpsHi, MMO);
    7028             : 
    7029           5 :   AddToWorklist(Lo.getNode());
    7030           5 :   AddToWorklist(Hi.getNode());
    7031             : 
    7032          10 :   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    7033             : }
    7034             : 
    7035         758 : SDValue DAGCombiner::visitMSTORE(SDNode *N) {
    7036         758 :   if (Level >= AfterLegalizeTypes)
    7037         453 :     return SDValue();
    7038             : 
    7039             :   MaskedStoreSDNode *MST = dyn_cast<MaskedStoreSDNode>(N);
    7040         305 :   SDValue Mask = MST->getMask();
    7041         305 :   SDValue Data  = MST->getValue();
    7042         610 :   EVT VT = Data.getValueType();
    7043             :   SDLoc DL(N);
    7044             : 
    7045             :   // If the MSTORE data type requires splitting and the mask is provided by a
    7046             :   // SETCC, then split both nodes and its operands before legalization. This
    7047             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7048             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7049         305 :   if (Mask.getOpcode() == ISD::SETCC) {
    7050             :     // Check if any splitting is required.
    7051          72 :     if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7052             :         TargetLowering::TypeSplitVector)
    7053          34 :       return SDValue();
    7054             : 
    7055             :     SDValue MaskLo, MaskHi, Lo, Hi;
    7056           4 :     std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7057             : 
    7058           2 :     SDValue Chain = MST->getChain();
    7059           2 :     SDValue Ptr   = MST->getBasePtr();
    7060             : 
    7061           2 :     EVT MemoryVT = MST->getMemoryVT();
    7062           2 :     unsigned Alignment = MST->getOriginalAlignment();
    7063             : 
    7064             :     // if Alignment is equal to the vector size,
    7065             :     // take the half of it for the second part
    7066             :     unsigned SecondHalfAlignment =
    7067           2 :       (Alignment == VT.getSizeInBits() / 8) ? Alignment / 2 : Alignment;
    7068             : 
    7069           2 :     EVT LoMemVT, HiMemVT;
    7070           4 :     std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7071             : 
    7072             :     SDValue DataLo, DataHi;
    7073           4 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    7074             : 
    7075           2 :     MachineMemOperand *MMO = DAG.getMachineFunction().
    7076           4 :       getMachineMemOperand(MST->getPointerInfo(),
    7077             :                            MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
    7078           2 :                            Alignment, MST->getAAInfo(), MST->getRanges());
    7079             : 
    7080           4 :     Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
    7081             :                             MST->isTruncatingStore(),
    7082           4 :                             MST->isCompressingStore());
    7083             : 
    7084           4 :     Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    7085           2 :                                      MST->isCompressingStore());
    7086             :     unsigned HiOffset = LoMemVT.getStoreSize();
    7087             : 
    7088           6 :     MMO = DAG.getMachineFunction().getMachineMemOperand(
    7089             :         MST->getPointerInfo().getWithOffset(HiOffset),
    7090             :         MachineMemOperand::MOStore, HiMemVT.getStoreSize(), SecondHalfAlignment,
    7091           2 :         MST->getAAInfo(), MST->getRanges());
    7092             : 
    7093           4 :     Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
    7094             :                             MST->isTruncatingStore(),
    7095           4 :                             MST->isCompressingStore());
    7096             : 
    7097           2 :     AddToWorklist(Lo.getNode());
    7098           2 :     AddToWorklist(Hi.getNode());
    7099             : 
    7100           4 :     return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    7101             :   }
    7102         269 :   return SDValue();
    7103             : }
    7104             : 
    7105         948 : SDValue DAGCombiner::visitMGATHER(SDNode *N) {
    7106         948 :   if (Level >= AfterLegalizeTypes)
    7107         487 :     return SDValue();
    7108             : 
    7109             :   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
    7110         461 :   SDValue Mask = MGT->getMask();
    7111             :   SDLoc DL(N);
    7112             : 
    7113             :   // If the MGATHER result requires splitting and the mask is provided by a
    7114             :   // SETCC, then split both nodes and its operands before legalization. This
    7115             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7116             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7117             : 
    7118         461 :   if (Mask.getOpcode() != ISD::SETCC)
    7119         431 :     return SDValue();
    7120             : 
    7121          60 :   EVT VT = N->getValueType(0);
    7122             : 
    7123             :   // Check if any splitting is required.
    7124          60 :   if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7125             :       TargetLowering::TypeSplitVector)
    7126          24 :     return SDValue();
    7127             : 
    7128             :   SDValue MaskLo, MaskHi, Lo, Hi;
    7129          12 :   std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7130             : 
    7131           6 :   SDValue Src0 = MGT->getValue();
    7132             :   SDValue Src0Lo, Src0Hi;
    7133          12 :   std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
    7134             : 
    7135             :   EVT LoVT, HiVT;
    7136          12 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
    7137             : 
    7138           6 :   SDValue Chain = MGT->getChain();
    7139           6 :   EVT MemoryVT = MGT->getMemoryVT();
    7140           6 :   unsigned Alignment = MGT->getOriginalAlignment();
    7141             : 
    7142           6 :   EVT LoMemVT, HiMemVT;
    7143          12 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7144             : 
    7145           6 :   SDValue Scale = MGT->getScale();
    7146           6 :   SDValue BasePtr = MGT->getBasePtr();
    7147           6 :   SDValue Index = MGT->getIndex();
    7148             :   SDValue IndexLo, IndexHi;
    7149          12 :   std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
    7150             : 
    7151           6 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    7152          12 :     getMachineMemOperand(MGT->getPointerInfo(),
    7153             :                           MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    7154           6 :                           Alignment, MGT->getAAInfo(), MGT->getRanges());
    7155             : 
    7156           6 :   SDValue OpsLo[] = { Chain, Src0Lo, MaskLo, BasePtr, IndexLo, Scale };
    7157          12 :   Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, DL, OpsLo,
    7158          12 :                            MMO);
    7159             : 
    7160           6 :   SDValue OpsHi[] = { Chain, Src0Hi, MaskHi, BasePtr, IndexHi, Scale };
    7161          12 :   Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, DL, OpsHi,
    7162          12 :                            MMO);
    7163             : 
    7164           6 :   AddToWorklist(Lo.getNode());
    7165           6 :   AddToWorklist(Hi.getNode());
    7166             : 
    7167             :   // Build a factor node to remember that this load is independent of the
    7168             :   // other one.
    7169          12 :   Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
    7170           6 :                       Hi.getValue(1));
    7171             : 
    7172             :   // Legalized the chain result - switch anything that used the old chain to
    7173             :   // use the new one.
    7174          12 :   DAG.ReplaceAllUsesOfValueWith(SDValue(MGT, 1), Chain);
    7175             : 
    7176          12 :   SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    7177             : 
    7178           6 :   SDValue RetOps[] = { GatherRes, Chain };
    7179          12 :   return DAG.getMergeValues(RetOps, DL);
    7180             : }
    7181             : 
    7182        1345 : SDValue DAGCombiner::visitMLOAD(SDNode *N) {
    7183        1345 :   if (Level >= AfterLegalizeTypes)
    7184         776 :     return SDValue();
    7185             : 
    7186             :   MaskedLoadSDNode *MLD = dyn_cast<MaskedLoadSDNode>(N);
    7187         569 :   SDValue Mask = MLD->getMask();
    7188             :   SDLoc DL(N);
    7189             : 
    7190             :   // If the MLOAD result requires splitting and the mask is provided by a
    7191             :   // SETCC, then split both nodes and its operands before legalization. This
    7192             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7193             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7194         569 :   if (Mask.getOpcode() == ISD::SETCC) {
    7195         152 :     EVT VT = N->getValueType(0);
    7196             : 
    7197             :     // Check if any splitting is required.
    7198         152 :     if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7199             :         TargetLowering::TypeSplitVector)
    7200          70 :       return SDValue();
    7201             : 
    7202             :     SDValue MaskLo, MaskHi, Lo, Hi;
    7203          12 :     std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7204             : 
    7205           6 :     SDValue Src0 = MLD->getSrc0();
    7206             :     SDValue Src0Lo, Src0Hi;
    7207          12 :     std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
    7208             : 
    7209             :     EVT LoVT, HiVT;
    7210          18 :     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
    7211             : 
    7212           6 :     SDValue Chain = MLD->getChain();
    7213           6 :     SDValue Ptr   = MLD->getBasePtr();
    7214           6 :     EVT MemoryVT = MLD->getMemoryVT();
    7215           6 :     unsigned Alignment = MLD->getOriginalAlignment();
    7216             : 
    7217             :     // if Alignment is equal to the vector size,
    7218             :     // take the half of it for the second part
    7219             :     unsigned SecondHalfAlignment =
    7220          12 :       (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
    7221             :          Alignment/2 : Alignment;
    7222             : 
    7223           6 :     EVT LoMemVT, HiMemVT;
    7224          12 :     std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7225             : 
    7226           6 :     MachineMemOperand *MMO = DAG.getMachineFunction().
    7227          12 :     getMachineMemOperand(MLD->getPointerInfo(),
    7228             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    7229           6 :                          Alignment, MLD->getAAInfo(), MLD->getRanges());
    7230             : 
    7231          12 :     Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
    7232           6 :                            ISD::NON_EXTLOAD, MLD->isExpandingLoad());
    7233             : 
    7234          12 :     Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    7235           6 :                                      MLD->isExpandingLoad());
    7236             :     unsigned HiOffset = LoMemVT.getStoreSize();
    7237             : 
    7238          18 :     MMO = DAG.getMachineFunction().getMachineMemOperand(
    7239             :         MLD->getPointerInfo().getWithOffset(HiOffset),
    7240             :         MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), SecondHalfAlignment,
    7241           6 :         MLD->getAAInfo(), MLD->getRanges());
    7242             : 
    7243          12 :     Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
    7244           6 :                            ISD::NON_EXTLOAD, MLD->isExpandingLoad());
    7245             : 
    7246           6 :     AddToWorklist(Lo.getNode());
    7247           6 :     AddToWorklist(Hi.getNode());
    7248             : 
    7249             :     // Build a factor node to remember that this load is independent of the
    7250             :     // other one.
    7251          12 :     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
    7252           6 :                         Hi.getValue(1));
    7253             : 
    7254             :     // Legalized the chain result - switch anything that used the old chain to
    7255             :     // use the new one.
    7256          12 :     DAG.ReplaceAllUsesOfValueWith(SDValue(MLD, 1), Chain);
    7257             : 
    7258          12 :     SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    7259             : 
    7260           6 :     SDValue RetOps[] = { LoadRes, Chain };
    7261          12 :     return DAG.getMergeValues(RetOps, DL);
    7262             :   }
    7263         493 :   return SDValue();
    7264             : }
    7265             : 
    7266             : /// A vector select of 2 constant vectors can be simplified to math/logic to
    7267             : /// avoid a variable select instruction and possibly avoid constant loads.
    7268       38012 : SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
    7269       38012 :   SDValue Cond = N->getOperand(0);
    7270       38012 :   SDValue N1 = N->getOperand(1);
    7271       38012 :   SDValue N2 = N->getOperand(2);
    7272       76024 :   EVT VT = N->getValueType(0);
    7273       98210 :   if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
    7274       28451 :       !TLI.convertSelectOfConstantsToMath(VT) ||
    7275       39318 :       !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) ||
    7276         110 :       !ISD::isBuildVectorOfConstantSDNodes(N2.getNode()))
    7277       37960 :     return SDValue();
    7278             : 
    7279             :   // Check if we can use the condition value to increment/decrement a single
    7280             :   // constant value. This simplifies a select to an add and removes a constant
    7281             :   // load/materialization from the general case.
    7282             :   bool AllAddOne = true;
    7283             :   bool AllSubOne = true;
    7284          52 :   unsigned Elts = VT.getVectorNumElements();
    7285         580 :   for (unsigned i = 0; i != Elts; ++i) {
    7286         264 :     SDValue N1Elt = N1.getOperand(i);
    7287         264 :     SDValue N2Elt = N2.getOperand(i);
    7288         528 :     if (N1Elt.isUndef() || N2Elt.isUndef())
    7289             :       continue;
    7290             : 
    7291         264 :     const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
    7292         264 :     const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
    7293         528 :     if (C1 != C2 + 1)
    7294             :       AllAddOne = false;
    7295         528 :     if (C1 != C2 - 1)
    7296             :       AllSubOne = false;
    7297             :   }
    7298             : 
    7299             :   // Further simplifications for the extra-special cases where the constants are
    7300             :   // all 0 or all -1 should be implemented as folds of these patterns.
    7301             :   SDLoc DL(N);
    7302          52 :   if (AllAddOne || AllSubOne) {
    7303             :     // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
    7304             :     // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
    7305          40 :     auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
    7306          80 :     SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
    7307          80 :     return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
    7308             :   }
    7309             : 
    7310             :   // The general case for select-of-constants:
    7311             :   // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
    7312             :   // ...but that only makes sense if a vselect is slower than 2 logic ops, so
    7313             :   // leave that to a machine-specific pass.
    7314          12 :   return SDValue();
    7315             : }
    7316             : 
    7317       38535 : SDValue DAGCombiner::visitVSELECT(SDNode *N) {
    7318       38535 :   SDValue N0 = N->getOperand(0);
    7319       38535 :   SDValue N1 = N->getOperand(1);
    7320       38535 :   SDValue N2 = N->getOperand(2);
    7321             :   SDLoc DL(N);
    7322             : 
    7323             :   // fold (vselect C, X, X) -> X
    7324             :   if (N1 == N2)
    7325           4 :     return N1;
    7326             : 
    7327             :   // Canonicalize integer abs.
    7328             :   // vselect (setg[te] X,  0),  X, -X ->
    7329             :   // vselect (setgt    X, -1),  X, -X ->
    7330             :   // vselect (setl[te] X,  0), -X,  X ->
    7331             :   // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
    7332       38531 :   if (N0.getOpcode() == ISD::SETCC) {
    7333        7562 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    7334        7562 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7335             :     bool isAbs = false;
    7336        7562 :     bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
    7337             : 
    7338       12029 :     if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
    7339        7242 :          (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
    7340        7918 :         N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
    7341         356 :       isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode());
    7342        4495 :     else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
    7343        7287 :              N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
    7344          81 :       isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
    7345             : 
    7346         437 :     if (isAbs) {
    7347         437 :       EVT VT = LHS.getValueType();
    7348         437 :       if (TLI.isOperationLegalOrCustom(ISD::ABS, VT))
    7349         712 :         return DAG.getNode(ISD::ABS, DL, VT, LHS);
    7350             : 
    7351          81 :       SDValue Shift = DAG.getNode(
    7352             :           ISD::SRA, DL, VT, LHS,
    7353          81 :           DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
    7354         162 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
    7355          81 :       AddToWorklist(Shift.getNode());
    7356          81 :       AddToWorklist(Add.getNode());
    7357         162 :       return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
    7358             :     }
    7359             : 
    7360             :     // If this select has a condition (setcc) with narrower operands than the
    7361             :     // select, try to widen the compare to match the select width.
    7362             :     // TODO: This should be extended to handle any constant.
    7363             :     // TODO: This could be extended to handle non-loading patterns, but that
    7364             :     //       requires thorough testing to avoid regressions.
    7365        7125 :     if (isNullConstantOrNullSplatConstant(RHS)) {
    7366             :       EVT NarrowVT = LHS.getValueType();
    7367        3191 :       EVT WideVT = N1.getValueType().changeVectorElementTypeToInteger();
    7368        3191 :       EVT SetCCVT = getSetCCResultType(LHS.getValueType());
    7369             :       unsigned SetCCWidth = SetCCVT.getScalarSizeInBits();
    7370             :       unsigned WideWidth = WideVT.getScalarSizeInBits();
    7371             :       bool IsSigned = isSignedIntSetCC(CC);
    7372        3191 :       auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
    7373          40 :       if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() &&
    7374          38 :           SetCCWidth != 1 && SetCCWidth < WideWidth &&
    7375        3209 :           TLI.isLoadExtLegalOrCustom(LoadExtOpcode, WideVT, NarrowVT) &&
    7376             :           TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) {
    7377             :         // Both compare operands can be widened for free. The LHS can use an
    7378             :         // extended load, and the RHS is a constant:
    7379             :         //   vselect (ext (setcc load(X), C)), N1, N2 -->
    7380             :         //   vselect (setcc extload(X), C'), N1, N2
    7381          10 :         auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    7382          20 :         SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
    7383          20 :         SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
    7384          10 :         EVT WideSetCCVT = getSetCCResultType(WideVT);
    7385          10 :         SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
    7386          20 :         return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
    7387             :       }
    7388             :     }
    7389             :   }
    7390             : 
    7391       38084 :   if (SimplifySelectOps(N, N1, N2))
    7392           6 :     return SDValue(N, 0);  // Don't revisit N.
    7393             : 
    7394             :   // Fold (vselect (build_vector all_ones), N1, N2) -> N1
    7395       38078 :   if (ISD::isBuildVectorAllOnes(N0.getNode()))
    7396          26 :     return N1;
    7397             :   // Fold (vselect (build_vector all_zeros), N1, N2) -> N2
    7398       38052 :   if (ISD::isBuildVectorAllZeros(N0.getNode()))
    7399          30 :     return N2;
    7400             : 
    7401             :   // The ConvertSelectToConcatVector function is assuming both the above
    7402             :   // checks for (vselect (build_vector all{ones,zeros) ...) have been made
    7403             :   // and addressed.
    7404         656 :   if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
    7405       38202 :       N2.getOpcode() == ISD::CONCAT_VECTORS &&
    7406         180 :       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
    7407          43 :     if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
    7408          10 :       return CV;
    7409             :   }
    7410             : 
    7411       38012 :   if (SDValue V = foldVSelectOfConstants(N))
    7412          40 :     return V;
    7413             : 
    7414       37972 :   return SDValue();
    7415             : }
    7416             : 
    7417       15730 : SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
    7418       15730 :   SDValue N0 = N->getOperand(0);
    7419       15730 :   SDValue N1 = N->getOperand(1);
    7420       15730 :   SDValue N2 = N->getOperand(2);
    7421       15730 :   SDValue N3 = N->getOperand(3);
    7422       15730 :   SDValue N4 = N->getOperand(4);
    7423       15730 :   ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
    7424             : 
    7425             :   // fold select_cc lhs, rhs, x, x, cc -> x
    7426             :   if (N2 == N3)
    7427          25 :     return N2;
    7428             : 
    7429             :   // Determine if the condition we're dealing with is constant
    7430       15705 :   if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
    7431       31410 :                                   CC, SDLoc(N), false)) {
    7432         291 :     AddToWorklist(SCC.getNode());
    7433             : 
    7434             :     if (ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
    7435          54 :       if (!SCCC->isNullValue())
    7436          17 :         return N2;    // cond always true -> true val
    7437             :       else
    7438          10 :         return N3;    // cond always false -> false val
    7439         264 :     } else if (SCC->isUndef()) {
    7440             :       // When the condition is UNDEF, just return the first operand. This is
    7441             :       // coherent the DAG creation, no setcc node is created in this case
    7442           0 :       return N2;
    7443         264 :     } else if (SCC.getOpcode() == ISD::SETCC) {
    7444             :       // Fold to a simpler select_cc
    7445         496 :       return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(),
    7446             :                          SCC.getOperand(0), SCC.getOperand(1), N2, N3,
    7447         248 :                          SCC.getOperand(2));
    7448             :     }
    7449             :   }
    7450             : 
    7451             :   // If we can fold this based on the true/false value, do so.
    7452       15430 :   if (SimplifySelectOps(N, N2, N3))
    7453           0 :     return SDValue(N, 0);  // Don't revisit N.
    7454             : 
    7455             :   // fold select_cc into other things, such as min/max/abs
    7456       30860 :   return SimplifySelectCC(SDLoc(N), N0, N1, N2, N3, CC);
    7457             : }
    7458             : 
    7459      178454 : SDValue DAGCombiner::visitSETCC(SDNode *N) {
    7460             :   // setcc is very commonly used as an argument to brcond. This pattern
    7461             :   // also lend itself to numerous combines and, as a result, it is desired
    7462             :   // we keep the argument to a brcond as a setcc as much as possible.
    7463             :   bool PreferSetCC =
    7464      139816 :       N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND;
    7465             : 
    7466             :   SDValue Combined = SimplifySetCC(
    7467             :       N->getValueType(0), N->getOperand(0), N->getOperand(1),
    7468      892270 :       cast<CondCodeSDNode>(N->getOperand(2))->get(), SDLoc(N), !PreferSetCC);
    7469             : 
    7470      178454 :   if (!Combined)
    7471      168095 :     return SDValue();
    7472             : 
    7473             :   // If we prefer to have a setcc, and we don't, we'll try our best to
    7474             :   // recreate one using rebuildSetCC.
    7475       17081 :   if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
    7476         225 :     SDValue NewSetCC = rebuildSetCC(Combined);
    7477             : 
    7478             :     // We don't have anything interesting to combine to.
    7479         225 :     if (NewSetCC.getNode() == N)
    7480          80 :       return SDValue();
    7481             : 
    7482         145 :     if (NewSetCC)
    7483           0 :       return NewSetCC;
    7484             :   }
    7485             : 
    7486       10279 :   return Combined;
    7487             : }
    7488             : 
    7489         268 : SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
    7490         268 :   SDValue LHS = N->getOperand(0);
    7491         268 :   SDValue RHS = N->getOperand(1);
    7492         268 :   SDValue Carry = N->getOperand(2);
    7493         268 :   SDValue Cond = N->getOperand(3);
    7494             : 
    7495             :   // If Carry is false, fold to a regular SETCC.
    7496         268 :   if (isNullConstant(Carry))
    7497          21 :     return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
    7498             : 
    7499         261 :   return SDValue();
    7500             : }
    7501             : 
    7502             : /// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
    7503             : /// a build_vector of constants.
    7504             : /// This function is called by the DAGCombiner when visiting sext/zext/aext
    7505             : /// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
    7506             : /// Vector extends are not folded if operations are legal; this is to
    7507             : /// avoid introducing illegal build_vector dag nodes.
    7508      243959 : static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
    7509             :                                          SelectionDAG &DAG, bool LegalTypes,
    7510             :                                          bool LegalOperations) {
    7511      243959 :   unsigned Opcode = N->getOpcode();
    7512      243959 :   SDValue N0 = N->getOperand(0);
    7513      487918 :   EVT VT = N->getValueType(0);
    7514             : 
    7515             :   assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
    7516             :          Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
    7517             :          Opcode == ISD::ZERO_EXTEND_VECTOR_INREG)
    7518             :          && "Expected EXTEND dag node in input!");
    7519             : 
    7520             :   // fold (sext c1) -> c1
    7521             :   // fold (zext c1) -> c1
    7522             :   // fold (aext c1) -> c1
    7523             :   if (isa<ConstantSDNode>(N0))
    7524        1118 :     return DAG.getNode(Opcode, SDLoc(N), VT, N0).getNode();
    7525             : 
    7526             :   // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
    7527             :   // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
    7528             :   // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
    7529      243400 :   EVT SVT = VT.getScalarType();
    7530      263693 :   if (!(VT.isVector() &&
    7531        8882 :       (!LegalTypes || (!LegalOperations && TLI.isTypeLegal(SVT))) &&
    7532       20293 :       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())))
    7533             :     return nullptr;
    7534             : 
    7535             :   // We can fold this node into a build_vector.
    7536          96 :   unsigned VTBits = SVT.getSizeInBits();
    7537         288 :   unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits();
    7538             :   SmallVector<SDValue, 8> Elts;
    7539          96 :   unsigned NumElts = VT.getVectorNumElements();
    7540             :   SDLoc DL(N);
    7541             : 
    7542         930 :   for (unsigned i=0; i != NumElts; ++i) {
    7543         834 :     SDValue Op = N0->getOperand(i);
    7544         503 :     if (Op->isUndef()) {
    7545          86 :       Elts.push_back(DAG.getUNDEF(SVT));
    7546          86 :       continue;
    7547             :     }
    7548             : 
    7549             :     SDLoc DL(Op);
    7550             :     // Get the constant value and if needed trunc it to the size of the type.
    7551             :     // Nodes like build_vector might have constants wider than the scalar type.
    7552         662 :     APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
    7553         331 :     if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
    7554         240 :       Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
    7555             :     else
    7556         422 :       Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
    7557             :   }
    7558             : 
    7559          96 :   return DAG.getBuildVector(VT, DL, Elts).getNode();
    7560             : }
    7561             : 
    7562             : // ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
    7563             : // "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
    7564             : // transformation. Returns true if extension are possible and the above
    7565             : // mentioned transformation is profitable.
    7566        1689 : static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
    7567             :                                     unsigned ExtOpc,
    7568             :                                     SmallVectorImpl<SDNode *> &ExtendNodes,
    7569             :                                     const TargetLowering &TLI) {
    7570             :   bool HasCopyToRegUses = false;
    7571        3378 :   bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType());
    7572        1689 :   for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
    7573             :                             UE = N0.getNode()->use_end();
    7574        5056 :        UI != UE; ++UI) {
    7575        3774 :     SDNode *User = *UI;
    7576        3774 :     if (User == N)
    7577        3030 :       continue;
    7578        2299 :     if (UI.getUse().getResNo() != N0.getResNo())
    7579             :       continue;
    7580             :     // FIXME: Only extend SETCC N, N and SETCC N, c for now.
    7581        1348 :     if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
    7582        1366 :       ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
    7583        1307 :       if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
    7584             :         // Sign bits will be lost after a zext.
    7585         407 :         return false;
    7586             :       bool Add = false;
    7587        3151 :       for (unsigned i = 0; i != 2; ++i) {
    7588        1301 :         SDValue UseOp = User->getOperand(i);
    7589         642 :         if (UseOp == N0)
    7590             :           continue;
    7591             :         if (!isa<ConstantSDNode>(UseOp))
    7592             :           return false;
    7593             :         Add = true;
    7594             :       }
    7595         604 :       if (Add)
    7596         604 :         ExtendNodes.push_back(User);
    7597         604 :       continue;
    7598             :     }
    7599             :     // If truncates aren't free and there are users we can't
    7600             :     // extend, it isn't worthwhile.
    7601         665 :     if (!isTruncFree)
    7602             :       return false;
    7603             :     // Remember if this value is live-out.
    7604         337 :     if (User->getOpcode() == ISD::CopyToReg)
    7605             :       HasCopyToRegUses = true;
    7606             :   }
    7607             : 
    7608        1282 :   if (HasCopyToRegUses) {
    7609             :     bool BothLiveOut = false;
    7610          60 :     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
    7611          86 :          UI != UE; ++UI) {
    7612             :       SDUse &Use = UI.getUse();
    7613          70 :       if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
    7614             :         BothLiveOut = true;
    7615             :         break;
    7616             :       }
    7617             :     }
    7618          60 :     if (BothLiveOut)
    7619             :       // Both unextended and extended values are live out. There had better be
    7620             :       // a good reason for the transformation.
    7621          44 :       return ExtendNodes.size();
    7622             :   }
    7623             :   return true;
    7624             : }
    7625             : 
    7626       12393 : void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
    7627             :                                   SDValue OrigLoad, SDValue ExtLoad,
    7628             :                                   ISD::NodeType ExtType) {
    7629             :   // Extend SetCC uses if necessary.
    7630             :   SDLoc DL(ExtLoad);
    7631       13597 :   for (SDNode *SetCC : SetCCs) {
    7632             :     SmallVector<SDValue, 4> Ops;
    7633             : 
    7634        3010 :     for (unsigned j = 0; j != 2; ++j) {
    7635        2408 :       SDValue SOp = SetCC->getOperand(j);
    7636             :       if (SOp == OrigLoad)
    7637         602 :         Ops.push_back(ExtLoad);
    7638             :       else
    7639        1806 :         Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
    7640             :     }
    7641             : 
    7642        1204 :     Ops.push_back(SetCC->getOperand(2));
    7643        2408 :     CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
    7644             :   }
    7645       12393 : }
    7646             : 
    7647             : // FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
    7648      167959 : SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
    7649      167959 :   SDValue N0 = N->getOperand(0);
    7650      335918 :   EVT DstVT = N->getValueType(0);
    7651             :   EVT SrcVT = N0.getValueType();
    7652             : 
    7653             :   assert((N->getOpcode() == ISD::SIGN_EXTEND ||
    7654             :           N->getOpcode() == ISD::ZERO_EXTEND) &&
    7655             :          "Unexpected node type (not an extend)!");
    7656             : 
    7657             :   // fold (sext (load x)) to multiple smaller sextloads; same for zext.
    7658             :   // For example, on a target with legal v4i32, but illegal v8i32, turn:
    7659             :   //   (v8i32 (sext (v8i16 (load x))))
    7660             :   // into:
    7661             :   //   (v8i32 (concat_vectors (v4i32 (sextload x)),
    7662             :   //                          (v4i32 (sextload (x + 16)))))
    7663             :   // Where uses of the original load, i.e.:
    7664             :   //   (v8i16 (load x))
    7665             :   // are replaced with:
    7666             :   //   (v8i16 (truncate
    7667             :   //     (v8i32 (concat_vectors (v4i32 (sextload x)),
    7668             :   //                            (v4i32 (sextload (x + 16)))))))
    7669             :   //
    7670             :   // This combine is only applicable to illegal, but splittable, vectors.
    7671             :   // All legal types, and illegal non-vector types, are handled elsewhere.
    7672             :   // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
    7673             :   //
    7674      335918 :   if (N0->getOpcode() != ISD::LOAD)
    7675      164219 :     return SDValue();
    7676             : 
    7677             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    7678             : 
    7679        3627 :   if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
    7680        9613 :       !N0.hasOneUse() || LN0->isVolatile() || !DstVT.isVector() ||
    7681        9018 :       !DstVT.isPow2VectorType() || !TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
    7682        3356 :     return SDValue();
    7683             : 
    7684             :   SmallVector<SDNode *, 4> SetCCs;
    7685         768 :   if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI))
    7686           0 :     return SDValue();
    7687             : 
    7688             :   ISD::LoadExtType ExtType =
    7689         384 :       N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
    7690             : 
    7691             :   // Try to split the vector types to get down to legal types.
    7692         384 :   EVT SplitSrcVT = SrcVT;
    7693         384 :   EVT SplitDstVT = DstVT;
    7694        2149 :   while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
    7695         723 :          SplitSrcVT.getVectorNumElements() > 1) {
    7696         521 :     SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
    7697         521 :     SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
    7698             :   }
    7699             : 
    7700             :   if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
    7701         202 :     return SDValue();
    7702             : 
    7703             :   SDLoc DL(N);
    7704             :   const unsigned NumSplits =
    7705         182 :       DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
    7706             :   const unsigned Stride = SplitSrcVT.getStoreSize();
    7707             :   SmallVector<SDValue, 4> Loads;
    7708             :   SmallVector<SDValue, 4> Chains;
    7709             : 
    7710         182 :   SDValue BasePtr = LN0->getBasePtr();
    7711         940 :   for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
    7712         379 :     const unsigned Offset = Idx * Stride;
    7713         758 :     const unsigned Align = MinAlign(LN0->getAlignment(), Offset);
    7714             : 
    7715         379 :     SDValue SplitLoad = DAG.getExtLoad(
    7716         379 :         ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), BasePtr,
    7717             :         LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT, Align,
    7718        1895 :         LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    7719             : 
    7720         758 :     BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
    7721         758 :                           DAG.getConstant(Stride, DL, BasePtr.getValueType()));
    7722             : 
    7723         379 :     Loads.push_back(SplitLoad.getValue(0));
    7724         379 :     Chains.push_back(SplitLoad.getValue(1));
    7725             :   }
    7726             : 
    7727         364 :   SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
    7728         364 :   SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
    7729             : 
    7730             :   // Simplify TF.
    7731         182 :   AddToWorklist(NewChain.getNode());
    7732             : 
    7733             :   CombineTo(N, NewValue);
    7734             : 
    7735             :   // Replace uses of the original load (before extension)
    7736             :   // with a truncate of the concatenated sextloaded vectors.
    7737             :   SDValue Trunc =
    7738         546 :       DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
    7739         364 :   ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode());
    7740             :   CombineTo(N0.getNode(), Trunc, NewChain);
    7741         182 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    7742             : }
    7743             : 
    7744             : // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
    7745             : //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
    7746      141028 : SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
    7747             :   assert(N->getOpcode() == ISD::ZERO_EXTEND);
    7748      282056 :   EVT VT = N->getValueType(0);
    7749             : 
    7750             :   // and/or/xor
    7751      141028 :   SDValue N0 = N->getOperand(0);
    7752      138802 :   if (!(N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    7753       35153 :         N0.getOpcode() == ISD::XOR) ||
    7754      208925 :       N0.getOperand(1).getOpcode() != ISD::Constant ||
    7755       36878 :       (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT)))
    7756      108284 :     return SDValue();
    7757             : 
    7758             :   // shl/shr
    7759       32744 :   SDValue N1 = N0->getOperand(0);
    7760       33749 :   if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) ||
    7761       34784 :       N1.getOperand(1).getOpcode() != ISD::Constant ||
    7762         315 :       (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT)))
    7763       31724 :     return SDValue();
    7764             : 
    7765             :   // load
    7766        1020 :   if (!isa<LoadSDNode>(N1.getOperand(0)))
    7767         988 :     return SDValue();
    7768             :   LoadSDNode *Load = cast<LoadSDNode>(N1.getOperand(0));
    7769             :   EVT MemVT = Load->getMemoryVT();
    7770          60 :   if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) ||
    7771          59 :       Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
    7772           5 :     return SDValue();
    7773             : 
    7774             : 
    7775             :   // If the shift op is SHL, the logic op must be AND, otherwise the result
    7776             :   // will be wrong.
    7777          30 :   if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND)
    7778           2 :     return SDValue();
    7779             : 
    7780          50 :   if (!N0.hasOneUse() || !N1.hasOneUse())
    7781           0 :     return SDValue();
    7782             : 
    7783             :   SmallVector<SDNode*, 4> SetCCs;
    7784          50 :   if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0),
    7785             :                                ISD::ZERO_EXTEND, SetCCs, TLI))
    7786           9 :     return SDValue();
    7787             : 
    7788             :   // Actually do the transformation.
    7789          32 :   SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
    7790             :                                    Load->getChain(), Load->getBasePtr(),
    7791          32 :                                    Load->getMemoryVT(), Load->getMemOperand());
    7792             : 
    7793             :   SDLoc DL1(N1);
    7794          16 :   SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
    7795          16 :                               N1.getOperand(1));
    7796             : 
    7797          16 :   APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    7798          32 :   Mask = Mask.zext(VT.getSizeInBits());
    7799             :   SDLoc DL0(N0);
    7800          16 :   SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
    7801          32 :                             DAG.getConstant(Mask, DL0, VT));
    7802             : 
    7803          16 :   ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
    7804             :   CombineTo(N, And);
    7805          16 :   if (SDValue(Load, 0).hasOneUse()) {
    7806          30 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
    7807             :   } else {
    7808           2 :     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
    7809           3 :                                 Load->getValueType(0), ExtLoad);
    7810             :     CombineTo(Load, Trunc, ExtLoad.getValue(1));
    7811             :   }
    7812          16 :   return SDValue(N,0); // Return N so it doesn't get rechecked!
    7813             : }
    7814             : 
    7815             : /// If we're narrowing or widening the result of a vector select and the final
    7816             : /// size is the same size as a setcc (compare) feeding the select, then try to
    7817             : /// apply the cast operation to the select's operands because matching vector
    7818             : /// sizes for a select condition and other operands should be more efficient.
    7819      331706 : SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
    7820      331706 :   unsigned CastOpcode = Cast->getOpcode();
    7821             :   assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
    7822             :           CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
    7823             :           CastOpcode == ISD::FP_ROUND) &&
    7824             :          "Unexpected opcode for vector select narrowing/widening");
    7825             : 
    7826             :   // We only do this transform before legal ops because the pattern may be
    7827             :   // obfuscated by target-specific operations after legalization. Do not create
    7828             :   // an illegal select op, however, because that may be difficult to lower.
    7829      663412 :   EVT VT = Cast->getValueType(0);
    7830      331706 :   if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
    7831      166455 :     return SDValue();
    7832             : 
    7833      165251 :   SDValue VSel = Cast->getOperand(0);
    7834      165319 :   if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
    7835          34 :       VSel.getOperand(0).getOpcode() != ISD::SETCC)
    7836      165217 :     return SDValue();
    7837             : 
    7838             :   // Does the setcc have the same vector size as the casted select?
    7839          34 :   SDValue SetCC = VSel.getOperand(0);
    7840          68 :   EVT SetCCVT = getSetCCResultType(SetCC.getOperand(0).getValueType());
    7841          34 :   if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
    7842          18 :     return SDValue();
    7843             : 
    7844             :   // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
    7845          16 :   SDValue A = VSel.getOperand(1);
    7846          16 :   SDValue B = VSel.getOperand(2);
    7847          16 :   SDValue CastA, CastB;
    7848             :   SDLoc DL(Cast);
    7849          16 :   if (CastOpcode == ISD::FP_ROUND) {
    7850             :     // FP_ROUND (fptrunc) has an extra flag operand to pass along.
    7851          12 :     CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
    7852          12 :     CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
    7853             :   } else {
    7854          24 :     CastA = DAG.getNode(CastOpcode, DL, VT, A);
    7855          24 :     CastB = DAG.getNode(CastOpcode, DL, VT, B);
    7856             :   }
    7857          32 :   return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
    7858             : }
    7859             : 
    7860             : // fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    7861             : // fold ([s|z]ext (     extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    7862      167730 : static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
    7863             :                                      const TargetLowering &TLI, EVT VT,
    7864             :                                      bool LegalOperations, SDNode *N,
    7865             :                                      SDValue N0, ISD::LoadExtType ExtLoadType) {
    7866             :   SDNode *N0Node = N0.getNode();
    7867      167730 :   bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD) ? ISD::isSEXTLoad(N0Node)
    7868             :                                                    : ISD::isZEXTLoad(N0Node);
    7869             :   if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) ||
    7870      167924 :       !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse())
    7871      167651 :     return {};
    7872             : 
    7873             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    7874          79 :   EVT MemVT = LN0->getMemoryVT();
    7875         158 :   if ((LegalOperations || LN0->isVolatile()) &&
    7876          71 :       !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT))
    7877          58 :     return {};
    7878             : 
    7879             :   SDValue ExtLoad =
    7880          21 :       DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
    7881          42 :                      LN0->getBasePtr(), MemVT, LN0->getMemOperand());
    7882             :   Combiner.CombineTo(N, ExtLoad);
    7883          42 :   DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    7884          21 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    7885             : }
    7886             : 
    7887             : // fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    7888             : // Only generate vector extloads when 1) they're legal, and 2) they are
    7889             : // deemed desirable by the target.
    7890      176507 : static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
    7891             :                                   const TargetLowering &TLI, EVT VT,
    7892             :                                   bool LegalOperations, SDNode *N, SDValue N0,
    7893             :                                   ISD::LoadExtType ExtLoadType,
    7894             :                                   ISD::NodeType ExtOpc) {
    7895             :   if (!ISD::isNON_EXTLoad(N0.getNode()) ||
    7896       12175 :       !ISD::isUNINDEXEDLoad(N0.getNode()) ||
    7897       19042 :       ((LegalOperations || VT.isVector() ||
    7898        4559 :         cast<LoadSDNode>(N0)->isVolatile()) &&
    7899        4794 :        !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType())))
    7900      166253 :     return {};
    7901             : 
    7902             :   bool DoXform = true;
    7903             :   SmallVector<SDNode *, 4> SetCCs;
    7904       10254 :   if (!N0.hasOneUse())
    7905        1030 :     DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI);
    7906       10254 :   if (VT.isVector())
    7907        4838 :     DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
    7908       10254 :   if (!DoXform)
    7909        1706 :     return {};
    7910             : 
    7911             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    7912        8548 :   SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
    7913             :                                    LN0->getBasePtr(), N0.getValueType(),
    7914       17096 :                                    LN0->getMemOperand());
    7915        8548 :   Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
    7916             :   // If the load value is used only by N, replace it via CombineTo N.
    7917             :   bool NoReplaceTrunc = SDValue(LN0, 0).hasOneUse();
    7918             :   Combiner.CombineTo(N, ExtLoad);
    7919        8548 :   if (NoReplaceTrunc) {
    7920        8352 :     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    7921             :   } else {
    7922             :     SDValue Trunc =
    7923         392 :         DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
    7924             :     Combiner.CombineTo(LN0, Trunc, ExtLoad.getValue(1));
    7925             :   }
    7926        8548 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    7927             : }
    7928             : 
    7929       38030 : SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
    7930       38030 :   SDValue N0 = N->getOperand(0);
    7931       76060 :   EVT VT = N->getValueType(0);
    7932             :   SDLoc DL(N);
    7933             : 
    7934       76060 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    7935       76060 :                                               LegalOperations))
    7936          73 :     return SDValue(Res, 0);
    7937             : 
    7938             :   // fold (sext (sext x)) -> (sext x)
    7939             :   // fold (sext (aext x)) -> (sext x)
    7940       75914 :   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    7941          12 :     return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
    7942             : 
    7943       37951 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    7944             :     // fold (sext (truncate (load x))) -> (sext (smaller load x))
    7945             :     // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
    7946        6487 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    7947          26 :       SDNode *oye = N0.getOperand(0).getNode();
    7948          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    7949             :         CombineTo(N0.getNode(), NarrowLoad);
    7950             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    7951          13 :         AddToWorklist(oye);
    7952             :       }
    7953          13 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    7954             :     }
    7955             : 
    7956             :     // See if the value being truncated is already sign extended.  If so, just
    7957             :     // eliminate the trunc/sext pair.
    7958       12948 :     SDValue Op = N0.getOperand(0);
    7959        6474 :     unsigned OpBits   = Op.getScalarValueSizeInBits();
    7960        6474 :     unsigned MidBits  = N0.getScalarValueSizeInBits();
    7961             :     unsigned DestBits = VT.getScalarSizeInBits();
    7962        6474 :     unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
    7963             : 
    7964        6474 :     if (OpBits == DestBits) {
    7965             :       // Op is i32, Mid is i8, and Dest is i32.  If Op has more than 24 sign
    7966             :       // bits, it is already ready.
    7967        3641 :       if (NumSignBits > DestBits-MidBits)
    7968         838 :         return Op;
    7969        2833 :     } else if (OpBits < DestBits) {
    7970             :       // Op is i32, Mid is i8, and Dest is i64.  If Op has more than 24 sign
    7971             :       // bits, just sext from i32.
    7972        2528 :       if (NumSignBits > OpBits-MidBits)
    7973          52 :         return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
    7974             :     } else {
    7975             :       // Op is i64, Mid is i8, and Dest is i32.  If Op has more than 56 sign
    7976             :       // bits, just truncate to i32.
    7977         305 :       if (NumSignBits > OpBits-MidBits)
    7978          38 :         return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
    7979             :     }
    7980             : 
    7981             :     // fold (sext (truncate x)) -> (sextinreg x).
    7982        5591 :     if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
    7983             :                                                  N0.getValueType())) {
    7984        5591 :       if (OpBits < DestBits)
    7985        7506 :         Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
    7986        3089 :       else if (OpBits > DestBits)
    7987         858 :         Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
    7988        5591 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
    7989       11182 :                          DAG.getValueType(N0.getValueType()));
    7990             :     }
    7991             :   }
    7992             : 
    7993             :   // Try to simplify (sext (load x)).
    7994       31464 :   if (SDValue foldedExt =
    7995       31464 :           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
    7996       31464 :                              ISD::SEXTLOAD, ISD::SIGN_EXTEND))
    7997        4621 :     return foldedExt;
    7998             : 
    7999             :   // fold (sext (load x)) to multiple smaller sextloads.
    8000             :   // Only on illegal but splittable vectors.
    8001       26843 :   if (SDValue ExtLoad = CombineExtLoad(N))
    8002         125 :     return ExtLoad;
    8003             : 
    8004             :   // Try to simplify (sext (sextload x)).
    8005       26718 :   if (SDValue foldedExt = tryToFoldExtOfExtload(
    8006       26718 :           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
    8007           4 :     return foldedExt;
    8008             : 
    8009             :   // fold (sext (and/or/xor (load x), cst)) ->
    8010             :   //      (and/or/xor (sextload x), (sext cst))
    8011       53069 :   if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8012        1030 :        N0.getOpcode() == ISD::XOR) &&
    8013           9 :       isa<LoadSDNode>(N0.getOperand(0)) &&
    8014       26723 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8015           0 :       (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    8016             :     LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    8017             :     EVT MemVT = LN00->getMemoryVT();
    8018           0 :     if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) &&
    8019           0 :       LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
    8020             :       SmallVector<SDNode*, 4> SetCCs;
    8021             :       bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
    8022           0 :                                              ISD::SIGN_EXTEND, SetCCs, TLI);
    8023           0 :       if (DoXform) {
    8024           0 :         SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
    8025             :                                          LN00->getChain(), LN00->getBasePtr(),
    8026             :                                          LN00->getMemoryVT(),
    8027           0 :                                          LN00->getMemOperand());
    8028           0 :         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8029           0 :         Mask = Mask.sext(VT.getSizeInBits());
    8030           0 :         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
    8031           0 :                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
    8032           0 :         ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND);
    8033           0 :         bool NoReplaceTruncAnd = !N0.hasOneUse();
    8034             :         bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
    8035             :         CombineTo(N, And);
    8036             :         // If N0 has multiple uses, change other uses as well.
    8037           0 :         if (NoReplaceTruncAnd) {
    8038             :           SDValue TruncAnd =
    8039           0 :               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
    8040           0 :           CombineTo(N0.getNode(), TruncAnd);
    8041             :         }
    8042           0 :         if (NoReplaceTrunc) {
    8043           0 :           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
    8044             :         } else {
    8045           0 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
    8046           0 :                                       LN00->getValueType(0), ExtLoad);
    8047             :           CombineTo(LN00, Trunc, ExtLoad.getValue(1));
    8048             :         }
    8049           0 :         return SDValue(N,0); // Return N so it doesn't get rechecked!
    8050             :       }
    8051             :     }
    8052             :   }
    8053             : 
    8054       53428 :   if (N0.getOpcode() == ISD::SETCC) {
    8055        4902 :     SDValue N00 = N0.getOperand(0);
    8056        4902 :     SDValue N01 = N0.getOperand(1);
    8057        4902 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    8058        9804 :     EVT N00VT = N0.getOperand(0).getValueType();
    8059             : 
    8060             :     // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
    8061             :     // Only do this before legalize for now.
    8062        7578 :     if (VT.isVector() && !LegalOperations &&
    8063        2676 :         TLI.getBooleanContents(N00VT) ==
    8064             :             TargetLowering::ZeroOrNegativeOneBooleanContent) {
    8065             :       // On some architectures (such as SSE/NEON/etc) the SETCC result type is
    8066             :       // of the same size as the compared operands. Only optimize sext(setcc())
    8067             :       // if this is the case.
    8068        2647 :       EVT SVT = getSetCCResultType(N00VT);
    8069             : 
    8070             :       // We know that the # elements of the results is the same as the
    8071             :       // # elements of the compare (and the # elements of the compare result
    8072             :       // for that matter).  Check to see that they are the same size.  If so,
    8073             :       // we know that the element size of the sext'd result matches the
    8074             :       // element size of the compare operands.
    8075        2647 :       if (VT.getSizeInBits() == SVT.getSizeInBits())
    8076        4637 :         return DAG.getSetCC(DL, VT, N00, N01, CC);
    8077             : 
    8078             :       // If the desired elements are smaller or larger than the source
    8079             :       // elements, we can use a matching integer vector type and then
    8080             :       // truncate/sign extend.
    8081         425 :       EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
    8082         428 :       if (SVT == MatchingVecType) {
    8083         193 :         SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
    8084         193 :         return DAG.getSExtOrTrunc(VsetCC, DL, VT);
    8085             :       }
    8086             :     }
    8087             : 
    8088             :     // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
    8089             :     // Here, T can be 1 or -1, depending on the type of the setcc and
    8090             :     // getBooleanContents().
    8091        2487 :     unsigned SetCCWidth = N0.getScalarValueSizeInBits();
    8092             : 
    8093             :     // To determine the "true" side of the select, we need to know the high bit
    8094             :     // of the value returned by the setcc if it evaluates to true.
    8095             :     // If the type of the setcc is i1, then the true case of the select is just
    8096             :     // sext(i1 1), that is, -1.
    8097             :     // If the type of the setcc is larger (say, i8) then the value of the high
    8098             :     // bit depends on getBooleanContents(), so ask TLI for a real "true" value
    8099             :     // of the appropriate width.
    8100             :     SDValue ExtTrueVal = (SetCCWidth == 1)
    8101        2486 :                              ? DAG.getAllOnesConstant(DL, VT)
    8102        4973 :                              : DAG.getBoolConstant(true, DL, VT, N00VT);
    8103        2487 :     SDValue Zero = DAG.getConstant(0, DL, VT);
    8104        2487 :     if (SDValue SCC =
    8105        2487 :             SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC, true))
    8106          25 :       return SCC;
    8107             : 
    8108        2462 :     if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) {
    8109         778 :       EVT SetCCVT = getSetCCResultType(N00VT);
    8110             :       // Don't do this transform for i1 because there's a select transform
    8111             :       // that would reverse it.
    8112             :       // TODO: We should not do this transform at all without a target hook
    8113             :       // because a sext is likely cheaper than a select?
    8114         885 :       if (SetCCVT.getScalarSizeInBits() != 1 &&
    8115         107 :           (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) {
    8116         107 :         SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
    8117         107 :         return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
    8118             :       }
    8119             :     }
    8120             :   }
    8121             : 
    8122             :   // fold (sext x) -> (zext x) if the sign bit is known zero.
    8123       47662 :   if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
    8124       23495 :       DAG.SignBitIsZero(N0))
    8125         766 :     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0);
    8126             : 
    8127       23784 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    8128           4 :     return NewVSel;
    8129             : 
    8130       23780 :   return SDValue();
    8131             : }
    8132             : 
    8133             : // isTruncateOf - If N is a truncate of some other value, return true, record
    8134             : // the value being truncated in Op and which of Op's bits are zero/one in Known.
    8135             : // This function computes KnownBits to avoid a duplicated call to
    8136             : // computeKnownBits in the caller.
    8137      147940 : static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
    8138             :                          KnownBits &Known) {
    8139      295880 :   if (N->getOpcode() == ISD::TRUNCATE) {
    8140        8407 :     Op = N->getOperand(0);
    8141        8407 :     DAG.computeKnownBits(Op, Known);
    8142             :     return true;
    8143             :   }
    8144             : 
    8145      168136 :   if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
    8146       57206 :       cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
    8147             :     return false;
    8148             : 
    8149        7700 :   SDValue Op0 = N->getOperand(0);
    8150        7700 :   SDValue Op1 = N->getOperand(1);
    8151             :   assert(Op0.getValueType() == Op1.getValueType());
    8152             : 
    8153        7700 :   if (isNullConstant(Op0))
    8154           0 :     Op = Op1;
    8155        7700 :   else if (isNullConstant(Op1))
    8156        5158 :     Op = Op0;
    8157             :   else
    8158             :     return false;
    8159             : 
    8160        5158 :   DAG.computeKnownBits(Op, Known);
    8161             : 
    8162       20632 :   if (!(Known.Zero | 1).isAllOnesValue())
    8163             :     return false;
    8164             : 
    8165             :   return true;
    8166             : }
    8167             : 
    8168      154812 : SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
    8169      154812 :   SDValue N0 = N->getOperand(0);
    8170      309624 :   EVT VT = N->getValueType(0);
    8171             : 
    8172      309624 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8173      309624 :                                               LegalOperations))
    8174         443 :     return SDValue(Res, 0);
    8175             : 
    8176             :   // fold (zext (zext x)) -> (zext x)
    8177             :   // fold (zext (aext x)) -> (zext x)
    8178      308738 :   if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    8179         550 :     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
    8180         550 :                        N0.getOperand(0));
    8181             : 
    8182             :   // fold (zext (truncate x)) -> (zext x) or
    8183             :   //      (zext (truncate x)) -> (truncate x)
    8184             :   // This is valid when the truncated bits of x are already zero.
    8185             :   // FIXME: We should extend this to work for vectors too.
    8186      154094 :   SDValue Op;
    8187      154094 :   KnownBits Known;
    8188      154094 :   if (!VT.isVector() && isTruncateOf(DAG, N0, Op, Known)) {
    8189             :     APInt TruncatedBits =
    8190        8576 :       (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ?
    8191             :       APInt(Op.getValueSizeInBits(), 0) :
    8192             :       APInt::getBitsSet(Op.getValueSizeInBits(),
    8193             :                         N0.getValueSizeInBits(),
    8194       17150 :                         std::min(Op.getValueSizeInBits(),
    8195       34300 :                                  VT.getSizeInBits()));
    8196        8576 :     if (TruncatedBits.isSubsetOf(Known.Zero))
    8197       14151 :       return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
    8198             :   }
    8199             : 
    8200             :   // fold (zext (truncate x)) -> (and x, mask)
    8201      298754 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8202             :     // fold (zext (truncate (load x))) -> (zext (smaller load x))
    8203             :     // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
    8204        4044 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8205          26 :       SDNode *oye = N0.getOperand(0).getNode();
    8206          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8207             :         CombineTo(N0.getNode(), NarrowLoad);
    8208             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8209          13 :         AddToWorklist(oye);
    8210             :       }
    8211          13 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8212             :     }
    8213             : 
    8214       12093 :     EVT SrcVT = N0.getOperand(0).getValueType();
    8215        8062 :     EVT MinVT = N0.getValueType();
    8216             : 
    8217             :     // Try to mask before the extension to avoid having to generate a larger mask,
    8218             :     // possibly over several sub-vectors.
    8219        5027 :     if (SrcVT.bitsLT(VT) && VT.isVector()) {
    8220          51 :       if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
    8221             :                                TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) {
    8222          51 :         SDValue Op = N0.getOperand(0);
    8223         153 :         Op = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
    8224          51 :         AddToWorklist(Op.getNode());
    8225         153 :         SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
    8226             :         // Transfer the debug info; the new node is equivalent to N0.
    8227          51 :         DAG.transferDbgValues(N0, ZExtOrTrunc);
    8228          51 :         return ZExtOrTrunc;
    8229             :       }
    8230             :     }
    8231             : 
    8232        3980 :     if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
    8233       15920 :       SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
    8234        3980 :       AddToWorklist(Op.getNode());
    8235       11940 :       SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
    8236             :       // We may safely transfer the debug info describing the truncate node over
    8237             :       // to the equivalent and operation.
    8238        3980 :       DAG.transferDbgValues(N0, And);
    8239        3980 :       return And;
    8240             :     }
    8241             :   }
    8242             : 
    8243             :   // Fold (zext (and (trunc x), cst)) -> (and x, cst),
    8244             :   // if either of the casts is not free.
    8245      147869 :   if (N0.getOpcode() == ISD::AND &&
    8246        2976 :       N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
    8247      146066 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8248        1172 :       (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
    8249         336 :                            N0.getValueType()) ||
    8250          86 :        !TLI.isZExtFree(N0.getValueType(), VT))) {
    8251         870 :     SDValue X = N0.getOperand(0).getOperand(0);
    8252         870 :     X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
    8253         580 :     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8254         580 :     Mask = Mask.zext(VT.getSizeInBits());
    8255             :     SDLoc DL(N);
    8256         290 :     return DAG.getNode(ISD::AND, DL, VT,
    8257         290 :                        X, DAG.getConstant(Mask, DL, VT));
    8258             :   }
    8259             : 
    8260             :   // Try to simplify (zext (load x)).
    8261      145043 :   if (SDValue foldedExt =
    8262      145043 :           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
    8263      145043 :                              ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
    8264        3927 :     return foldedExt;
    8265             : 
    8266             :   // fold (zext (load x)) to multiple smaller zextloads.
    8267             :   // Only on illegal but splittable vectors.
    8268      141116 :   if (SDValue ExtLoad = CombineExtLoad(N))
    8269          57 :     return ExtLoad;
    8270             : 
    8271             :   // fold (zext (and/or/xor (load x), cst)) ->
    8272             :   //      (and/or/xor (zextload x), (zext cst))
    8273             :   // Unless (and (load x) cst) will match as a zextload already and has
    8274             :   // additional users.
    8275      279872 :   if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8276       35184 :        N0.getOpcode() == ISD::XOR) &&
    8277         262 :       isa<LoadSDNode>(N0.getOperand(0)) &&
    8278      141394 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8279         116 :       (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    8280             :     LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    8281             :     EVT MemVT = LN00->getMemoryVT();
    8282          31 :     if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) &&
    8283          68 :         LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
    8284             :       bool DoXform = true;
    8285             :       SmallVector<SDNode*, 4> SetCCs;
    8286          62 :       if (!N0.hasOneUse()) {
    8287          16 :         if (N0.getOpcode() == ISD::AND) {
    8288             :           auto *AndC = cast<ConstantSDNode>(N0.getOperand(1));
    8289           1 :           EVT LoadResultTy = AndC->getValueType(0);
    8290           1 :           EVT ExtVT;
    8291           1 :           if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
    8292             :             DoXform = false;
    8293             :         }
    8294             :       }
    8295             :       if (DoXform)
    8296          62 :         DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
    8297             :                                           ISD::ZERO_EXTEND, SetCCs, TLI);
    8298          31 :       if (DoXform) {
    8299          62 :         SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
    8300             :                                          LN00->getChain(), LN00->getBasePtr(),
    8301             :                                          LN00->getMemoryVT(),
    8302          62 :                                          LN00->getMemOperand());
    8303          62 :         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8304          62 :         Mask = Mask.zext(VT.getSizeInBits());
    8305             :         SDLoc DL(N);
    8306          31 :         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
    8307          62 :                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
    8308          62 :         ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
    8309          31 :         bool NoReplaceTruncAnd = !N0.hasOneUse();
    8310             :         bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
    8311             :         CombineTo(N, And);
    8312             :         // If N0 has multiple uses, change other uses as well.
    8313          31 :         if (NoReplaceTruncAnd) {
    8314             :           SDValue TruncAnd =
    8315          24 :               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
    8316           8 :           CombineTo(N0.getNode(), TruncAnd);
    8317             :         }
    8318          31 :         if (NoReplaceTrunc) {
    8319          56 :           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
    8320             :         } else {
    8321           6 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
    8322           9 :                                       LN00->getValueType(0), ExtLoad);
    8323             :           CombineTo(LN00, Trunc, ExtLoad.getValue(1));
    8324             :         }
    8325          31 :         return SDValue(N,0); // Return N so it doesn't get rechecked!
    8326             :       }
    8327             :     }
    8328             :   }
    8329             : 
    8330             :   // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
    8331             :   //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
    8332      141028 :   if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
    8333          16 :     return ZExtLoad;
    8334             : 
    8335             :   // Try to simplify (zext (zextload x)).
    8336      141012 :   if (SDValue foldedExt = tryToFoldExtOfExtload(
    8337      141012 :           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
    8338          17 :     return foldedExt;
    8339             : 
    8340      281990 :   if (N0.getOpcode() == ISD::SETCC) {
    8341             :     // Only do this before legalize for now.
    8342      100247 :     if (!LegalOperations && VT.isVector() &&
    8343       51184 :         N0.getValueType().getVectorElementType() == MVT::i1) {
    8344         660 :       EVT N00VT = N0.getOperand(0).getValueType();
    8345         660 :       if (getSetCCResultType(N00VT) == N0.getValueType())
    8346         116 :         return SDValue();
    8347             : 
    8348             :       // We know that the # elements of the results is the same as the #
    8349             :       // elements of the compare (and the # elements of the compare result for
    8350             :       // that matter). Check to see that they are the same size. If so, we know
    8351             :       // that the element size of the sext'd result matches the element size of
    8352             :       // the compare operands.
    8353             :       SDLoc DL(N);
    8354         104 :       SDValue VecOnes = DAG.getConstant(1, DL, VT);
    8355         104 :       if (VT.getSizeInBits() == N00VT.getSizeInBits()) {
    8356             :         // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
    8357          94 :         SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
    8358         188 :                                      N0.getOperand(1), N0.getOperand(2));
    8359         188 :         return DAG.getNode(ISD::AND, DL, VT, VSetCC, VecOnes);
    8360             :       }
    8361             : 
    8362             :       // If the desired elements are smaller or larger than the source
    8363             :       // elements we can use a matching integer vector type and then
    8364             :       // truncate/sign extend.
    8365          10 :       EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
    8366             :       SDValue VsetCC =
    8367          10 :           DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
    8368          20 :                       N0.getOperand(1), N0.getOperand(2));
    8369          10 :       return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT),
    8370          10 :                          VecOnes);
    8371             :     }
    8372             : 
    8373             :     // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    8374             :     SDLoc DL(N);
    8375       50522 :     if (SDValue SCC = SimplifySelectCC(
    8376      101044 :             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
    8377       50522 :             DAG.getConstant(0, DL, VT),
    8378      252610 :             cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
    8379          93 :       return SCC;
    8380             :   }
    8381             : 
    8382             :   // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
    8383      278537 :   if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
    8384        6869 :       isa<ConstantSDNode>(N0.getOperand(1)) &&
    8385      147562 :       N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
    8386          11 :       N0.hasOneUse()) {
    8387          20 :     SDValue ShAmt = N0.getOperand(1);
    8388          20 :     unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
    8389          10 :     if (N0.getOpcode() == ISD::SHL) {
    8390           9 :       SDValue InnerZExt = N0.getOperand(0);
    8391             :       // If the original shl may be shifting out bits, do not perform this
    8392             :       // transformation.
    8393           9 :       unsigned KnownZeroBits = InnerZExt.getValueSizeInBits() -
    8394          18 :         InnerZExt.getOperand(0).getValueSizeInBits();
    8395           9 :       if (ShAmtVal > KnownZeroBits)
    8396           0 :         return SDValue();
    8397             :     }
    8398             : 
    8399             :     SDLoc DL(N);
    8400             : 
    8401             :     // Ensure that the shift amount is wide enough for the shifted value.
    8402          10 :     if (VT.getSizeInBits() >= 256)
    8403           0 :       ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
    8404             : 
    8405          10 :     return DAG.getNode(N0.getOpcode(), DL, VT,
    8406          10 :                        DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
    8407          20 :                        ShAmt);
    8408             :   }
    8409             : 
    8410      140672 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    8411           2 :     return NewVSel;
    8412             : 
    8413      140670 :   return SDValue();
    8414             : }
    8415             : 
    8416       42906 : SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
    8417       42906 :   SDValue N0 = N->getOperand(0);
    8418       85812 :   EVT VT = N->getValueType(0);
    8419             : 
    8420       85812 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8421       85812 :                                               LegalOperations))
    8422          78 :     return SDValue(Res, 0);
    8423             : 
    8424             :   // fold (aext (aext x)) -> (aext x)
    8425             :   // fold (aext (zext x)) -> (zext x)
    8426             :   // fold (aext (sext x)) -> (sext x)
    8427       42828 :   if (N0.getOpcode() == ISD::ANY_EXTEND  ||
    8428       85648 :       N0.getOpcode() == ISD::ZERO_EXTEND ||
    8429             :       N0.getOpcode() == ISD::SIGN_EXTEND)
    8430          39 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
    8431             : 
    8432             :   // fold (aext (truncate (load x))) -> (aext (smaller load x))
    8433             :   // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
    8434       42815 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8435        6634 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8436          13 :       SDNode *oye = N0.getOperand(0).getNode();
    8437          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8438             :         CombineTo(N0.getNode(), NarrowLoad);
    8439             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8440          13 :         AddToWorklist(oye);
    8441             :       }
    8442          13 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8443             :     }
    8444             :   }
    8445             : 
    8446             :   // fold (aext (truncate x))
    8447       42802 :   if (N0.getOpcode() == ISD::TRUNCATE)
    8448       19863 :     return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
    8449             : 
    8450             :   // Fold (aext (and (trunc x), cst)) -> (and x, cst)
    8451             :   // if the trunc is not free.
    8452        1779 :   if (N0.getOpcode() == ISD::AND &&
    8453        2699 :       N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
    8454       37912 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8455        2433 :       !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
    8456         811 :                           N0.getValueType())) {
    8457             :     SDLoc DL(N);
    8458          24 :     SDValue X = N0.getOperand(0).getOperand(0);
    8459          12 :     X = DAG.getAnyExtOrTrunc(X, DL, VT);
    8460          12 :     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8461          24 :     Mask = Mask.zext(VT.getSizeInBits());
    8462          12 :     return DAG.getNode(ISD::AND, DL, VT,
    8463          12 :                        X, DAG.getConstant(Mask, DL, VT));
    8464             :   }
    8465             : 
    8466             :   // fold (aext (load x)) -> (aext (truncate (extload x)))
    8467             :   // None of the supported targets knows how to perform load and any_ext
    8468             :   // on vectors in one instruction.  We only perform this transformation on
    8469             :   // scalars.
    8470        4091 :   if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
    8471        3990 :       ISD::isUNINDEXEDLoad(N0.getNode()) &&
    8472        4039 :       TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) {
    8473             :     bool DoXform = true;
    8474             :     SmallVector<SDNode*, 4> SetCCs;
    8475        3787 :     if (!N0.hasOneUse())
    8476         219 :       DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs,
    8477             :                                         TLI);
    8478         219 :     if (DoXform) {
    8479             :       LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8480        7232 :       SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
    8481             :                                        LN0->getChain(),
    8482             :                                        LN0->getBasePtr(), N0.getValueType(),
    8483        7232 :                                        LN0->getMemOperand());
    8484        3616 :       ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
    8485             :       // If the load value is used only by N, replace it via CombineTo N.
    8486             :       bool NoReplaceTrunc = N0.hasOneUse();
    8487             :       CombineTo(N, ExtLoad);
    8488        3616 :       if (NoReplaceTrunc) {
    8489        7136 :         DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8490             :       } else {
    8491          96 :         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
    8492          96 :                                     N0.getValueType(), ExtLoad);
    8493             :         CombineTo(LN0, Trunc, ExtLoad.getValue(1));
    8494             :       }
    8495        3616 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8496             :     }
    8497             :   }
    8498             : 
    8499             :   // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
    8500             :   // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
    8501             :   // fold (aext ( extload x)) -> (aext (truncate (extload  x)))
    8502             :   if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) &&
    8503       32852 :       ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
    8504             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8505             :     ISD::LoadExtType ExtType = LN0->getExtensionType();
    8506         148 :     EVT MemVT = LN0->getMemoryVT();
    8507         193 :     if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) {
    8508         254 :       SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
    8509             :                                        VT, LN0->getChain(), LN0->getBasePtr(),
    8510         254 :                                        MemVT, LN0->getMemOperand());
    8511             :       CombineTo(N, ExtLoad);
    8512         254 :       DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8513         127 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8514             :     }
    8515             :   }
    8516             : 
    8517       32426 :   if (N0.getOpcode() == ISD::SETCC) {
    8518             :     // For vectors:
    8519             :     // aext(setcc) -> vsetcc
    8520             :     // aext(setcc) -> truncate(vsetcc)
    8521             :     // aext(setcc) -> aext(vsetcc)
    8522             :     // Only do this before legalize for now.
    8523        1994 :     if (VT.isVector() && !LegalOperations) {
    8524         734 :       EVT N00VT = N0.getOperand(0).getValueType();
    8525         734 :       if (getSetCCResultType(N00VT) == N0.getValueType())
    8526          81 :         return SDValue();
    8527             : 
    8528             :       // We know that the # elements of the results is the same as the
    8529             :       // # elements of the compare (and the # elements of the compare result
    8530             :       // for that matter).  Check to see that they are the same size.  If so,
    8531             :       // we know that the element size of the sext'd result matches the
    8532             :       // element size of the compare operands.
    8533         286 :       if (VT.getSizeInBits() == N00VT.getSizeInBits())
    8534         360 :         return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0),
    8535             :                              N0.getOperand(1),
    8536         360 :                              cast<CondCodeSDNode>(N0.getOperand(2))->get());
    8537             :       // If the desired elements are smaller or larger than the source
    8538             :       // elements we can use a matching integer vector type and then
    8539             :       // truncate/any extend
    8540             :       else {
    8541         106 :         EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
    8542             :         SDValue VsetCC =
    8543         212 :           DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0),
    8544             :                         N0.getOperand(1),
    8545         212 :                         cast<CondCodeSDNode>(N0.getOperand(2))->get());
    8546         318 :         return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT);
    8547             :       }
    8548             :     }
    8549             : 
    8550             :     // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    8551             :     SDLoc DL(N);
    8552        1627 :     if (SDValue SCC = SimplifySelectCC(
    8553        1627 :             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
    8554        1627 :             DAG.getConstant(0, DL, VT),
    8555        6508 :             cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
    8556          16 :       return SCC;
    8557             :   }
    8558             : 
    8559       32043 :   return SDValue();
    8560             : }
    8561             : 
    8562       68314 : SDValue DAGCombiner::visitAssertExt(SDNode *N) {
    8563       68314 :   unsigned Opcode = N->getOpcode();
    8564       68314 :   SDValue N0 = N->getOperand(0);
    8565       68314 :   SDValue N1 = N->getOperand(1);
    8566       68314 :   EVT AssertVT = cast<VTSDNode>(N1)->getVT();
    8567             : 
    8568             :   // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt)
    8569       68314 :   if (N0.getOpcode() == Opcode &&
    8570             :       AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT())
    8571           0 :     return N0;
    8572             : 
    8573       73234 :   if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
    8574        2460 :       N0.getOperand(0).getOpcode() == Opcode) {
    8575             :     // We have an assert, truncate, assert sandwich. Make one stronger assert
    8576             :     // by asserting on the smallest asserted type to the larger source type.
    8577             :     // This eliminates the later assert:
    8578             :     // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN
    8579             :     // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN
    8580        2400 :     SDValue BigA = N0.getOperand(0);
    8581        2400 :     EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
    8582             :     assert(BigA_AssertVT.bitsLE(N0.getValueType()) &&
    8583             :            "Asserting zero/sign-extended bits to a type larger than the "
    8584             :            "truncated destination does not provide information");
    8585             : 
    8586             :     SDLoc DL(N);
    8587        2400 :     EVT MinAssertVT = AssertVT.bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
    8588        2400 :     SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
    8589        2400 :     SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
    8590        2400 :                                     BigA.getOperand(0), MinAssertVTVal);
    8591        7200 :     return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
    8592             :   }
    8593             : 
    8594       65914 :   return SDValue();
    8595             : }
    8596             : 
    8597             : /// If the result of a wider load is shifted to right of N  bits and then
    8598             : /// truncated to a narrower type and where N is a multiple of number of bits of
    8599             : /// the narrower type, transform it to a narrower load from address + N / num of
    8600             : /// bits of new type. Also narrow the load if the result is masked with an AND
    8601             : /// to effectively produce a smaller type. If the result is to be extended, also
    8602             : /// fold the extension to form a extending load.
    8603      348277 : SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
    8604      348277 :   unsigned Opc = N->getOpcode();
    8605             : 
    8606             :   ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
    8607      348277 :   SDValue N0 = N->getOperand(0);
    8608      696554 :   EVT VT = N->getValueType(0);
    8609      348277 :   EVT ExtVT = VT;
    8610             : 
    8611             :   // This transformation isn't valid for vector loads.
    8612      348277 :   if (VT.isVector())
    8613       14989 :     return SDValue();
    8614             : 
    8615             :   // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
    8616             :   // extended to VT.
    8617      333288 :   if (Opc == ISD::SIGN_EXTEND_INREG) {
    8618             :     ExtType = ISD::SEXTLOAD;
    8619       38530 :     ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    8620      294758 :   } else if (Opc == ISD::SRL) {
    8621             :     // Another special-case: SRL is basically zero-extending a narrower value,
    8622             :     // or it maybe shifting a higher subword, half or byte into the lowest
    8623             :     // bits.
    8624             :     ExtType = ISD::ZEXTLOAD;
    8625      111598 :     N0 = SDValue(N, 0);
    8626             : 
    8627             :     auto *LN0 = dyn_cast<LoadSDNode>(N0.getOperand(0));
    8628             :     auto *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    8629      111598 :     if (!N01 || !LN0)
    8630       98952 :       return SDValue();
    8631             : 
    8632       12646 :     uint64_t ShiftAmt = N01->getZExtValue();
    8633       12646 :     uint64_t MemoryWidth = LN0->getMemoryVT().getSizeInBits();
    8634       12646 :     if (LN0->getExtensionType() != ISD::SEXTLOAD && MemoryWidth > ShiftAmt)
    8635       12477 :       ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShiftAmt);
    8636             :     else
    8637         169 :       ExtVT = EVT::getIntegerVT(*DAG.getContext(),
    8638         169 :                                 VT.getSizeInBits() - ShiftAmt);
    8639      183160 :   } else if (Opc == ISD::AND) {
    8640             :     // An AND with a constant mask is the same as a truncate + zero-extend.
    8641             :     auto AndC = dyn_cast<ConstantSDNode>(N->getOperand(1));
    8642       39326 :     if (!AndC || !AndC->getAPIntValue().isMask())
    8643        6238 :       return SDValue();
    8644             : 
    8645             :     unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
    8646             :     ExtType = ISD::ZEXTLOAD;
    8647       13425 :     ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    8648             :   }
    8649             : 
    8650             :   unsigned ShAmt = 0;
    8651      514206 :   if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    8652       50635 :     SDValue SRL = N0;
    8653             :     if (auto *ConstShift = dyn_cast<ConstantSDNode>(SRL.getOperand(1))) {
    8654       98496 :       ShAmt = ConstShift->getZExtValue();
    8655       49248 :       unsigned EVTBits = ExtVT.getSizeInBits();
    8656             :       // Is the shift amount a multiple of size of VT?
    8657       49248 :       if ((ShAmt & (EVTBits-1)) == 0) {
    8658       42383 :         N0 = N0.getOperand(0);
    8659             :         // Is the load width a multiple of size of VT?
    8660       42383 :         if ((N0.getValueSizeInBits() & (EVTBits-1)) != 0)
    8661          21 :           return SDValue();
    8662             :       }
    8663             : 
    8664             :       // At this point, we must have a load or else we can't do the transform.
    8665       49227 :       if (!isa<LoadSDNode>(N0)) return SDValue();
    8666             : 
    8667             :       auto *LN0 = cast<LoadSDNode>(N0);
    8668             : 
    8669             :       // Because a SRL must be assumed to *need* to zero-extend the high bits
    8670             :       // (as opposed to anyext the high bits), we can't combine the zextload
    8671             :       // lowering of SRL and an sextload.
    8672       13736 :       if (LN0->getExtensionType() == ISD::SEXTLOAD)
    8673         188 :         return SDValue();
    8674             : 
    8675             :       // If the shift amount is larger than the input type then we're not
    8676             :       // accessing any of the loaded bytes.  If the load was a zextload/extload
    8677             :       // then the result of the shift+trunc is zero/undef (handled elsewhere).
    8678       13548 :       if (ShAmt >= LN0->getMemoryVT().getSizeInBits())
    8679           4 :         return SDValue();
    8680             : 
    8681             :       // If the SRL is only used by a masking AND, we may be able to adjust
    8682             :       // the ExtVT to make the AND redundant.
    8683       13544 :       SDNode *Mask = *(SRL->use_begin());
    8684       13544 :       if (Mask->getOpcode() == ISD::AND &&
    8685        4743 :           isa<ConstantSDNode>(Mask->getOperand(1))) {
    8686             :         const APInt &ShiftMask =
    8687        4738 :           cast<ConstantSDNode>(Mask->getOperand(1))->getAPIntValue();
    8688        4738 :         if (ShiftMask.isMask()) {
    8689        4505 :           EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(),
    8690        4505 :                                            ShiftMask.countTrailingOnes());
    8691             :           // Recompute the type.
    8692       13345 :           if (TLI.isLoadExtLegal(ExtType, N0.getValueType(), MaskedVT))
    8693         396 :             ExtVT = MaskedVT;
    8694             :         }
    8695             :       }
    8696             :     }
    8697             :   }
    8698             : 
    8699             :   // If the load is shifted left (and the result isn't shifted back right),
    8700             :   // we can fold the truncate through the shift.
    8701             :   unsigned ShLeftAmt = 0;
    8702      359293 :   if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
    8703       17812 :       ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) {
    8704         386 :     if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    8705          18 :       ShLeftAmt = N01->getZExtValue();
    8706           9 :       N0 = N0.getOperand(0);
    8707             :     }
    8708             :   }
    8709             : 
    8710             :   // If we haven't found a load, we can't narrow it.
    8711      192394 :   if (!isa<LoadSDNode>(N0))
    8712      158409 :     return SDValue();
    8713             : 
    8714             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8715       33985 :   if (!isLegalNarrowLoad(LN0, ExtType, ExtVT, ShAmt))
    8716       32614 :     return SDValue();
    8717             : 
    8718             :   // For big endian targets, we need to adjust the offset to the pointer to
    8719             :   // load the correct bytes.
    8720        2742 :   if (DAG.getDataLayout().isBigEndian()) {
    8721         316 :     unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
    8722             :     unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
    8723         158 :     ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
    8724             :   }
    8725             : 
    8726        4113 :   EVT PtrType = N0.getOperand(1).getValueType();
    8727        1371 :   uint64_t PtrOff = ShAmt / 8;
    8728        2742 :   unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
    8729             :   SDLoc DL(LN0);
    8730             :   // The original load itself didn't wrap, so an offset within it doesn't.
    8731             :   SDNodeFlags Flags;
    8732             :   Flags.setNoUnsignedWrap(true);
    8733        1371 :   SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
    8734             :                                PtrType, LN0->getBasePtr(),
    8735             :                                DAG.getConstant(PtrOff, DL, PtrType),
    8736        2742 :                                Flags);
    8737        1371 :   AddToWorklist(NewPtr.getNode());
    8738             : 
    8739             :   SDValue Load;
    8740        1371 :   if (ExtType == ISD::NON_EXTLOAD)
    8741        2694 :     Load = DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr,
    8742             :                        LN0->getPointerInfo().getWithOffset(PtrOff), NewAlign,
    8743        3592 :                        LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    8744             :   else
    8745        1419 :     Load = DAG.getExtLoad(ExtType, SDLoc(N0), VT, LN0->getChain(), NewPtr,
    8746             :                           LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
    8747         473 :                           NewAlign, LN0->getMemOperand()->getFlags(),
    8748        1892 :                           LN0->getAAInfo());
    8749             : 
    8750             :   // Replace the old load's chain with the new load's chain.
    8751             :   WorklistRemover DeadNodes(*this);
    8752        1371 :   DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
    8753             : 
    8754             :   // Shift the result left, if we've swallowed a left shift.
    8755        1371 :   SDValue Result = Load;
    8756        1371 :   if (ShLeftAmt != 0) {
    8757           9 :     EVT ShImmTy = getShiftAmountTy(Result.getValueType());
    8758           9 :     if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt))
    8759           0 :       ShImmTy = VT;
    8760             :     // If the shift amount is as large as the result size (but, presumably,
    8761             :     // no larger than the source) then the useful bits of the result are
    8762             :     // zero; we can't simply return the shortened shift, because the result
    8763             :     // of that operation is undefined.
    8764             :     SDLoc DL(N0);
    8765           9 :     if (ShLeftAmt >= VT.getSizeInBits())
    8766           7 :       Result = DAG.getConstant(0, DL, VT);
    8767             :     else
    8768           4 :       Result = DAG.getNode(ISD::SHL, DL, VT,
    8769           4 :                           Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
    8770             :   }
    8771             : 
    8772             :   // Return the new loaded value.
    8773        1371 :   return Result;
    8774             : }
    8775             : 
    8776       43574 : SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
    8777       43574 :   SDValue N0 = N->getOperand(0);
    8778       43574 :   SDValue N1 = N->getOperand(1);
    8779       87148 :   EVT VT = N->getValueType(0);
    8780       43574 :   EVT EVT = cast<VTSDNode>(N1)->getVT();
    8781             :   unsigned VTBits = VT.getScalarSizeInBits();
    8782             :   unsigned EVTBits = EVT.getScalarSizeInBits();
    8783             : 
    8784       43574 :   if (N0.isUndef())
    8785           1 :     return DAG.getUNDEF(VT);
    8786             : 
    8787             :   // fold (sext_in_reg c1) -> c1
    8788       43573 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    8789          36 :     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
    8790             : 
    8791             :   // If the input is already sign extended, just drop the extension.
    8792       43561 :   if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
    8793        1405 :     return N0;
    8794             : 
    8795             :   // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
    8796       42158 :   if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
    8797           2 :       EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
    8798           4 :     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    8799           4 :                        N0.getOperand(0), N1);
    8800             : 
    8801             :   // fold (sext_in_reg (sext x)) -> (sext x)
    8802             :   // fold (sext_in_reg (aext x)) -> (sext x)
    8803             :   // if x is small enough.
    8804       84308 :   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
    8805        9097 :     SDValue N00 = N0.getOperand(0);
    8806        9102 :     if (N00.getScalarValueSizeInBits() <= EVTBits &&
    8807           8 :         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
    8808           9 :       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
    8809             :   }
    8810             : 
    8811             :   // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
    8812       42137 :   if ((N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
    8813       42137 :        N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
    8814       42165 :        N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
    8815          14 :       N0.getOperand(0).getScalarValueSizeInBits() == EVTBits) {
    8816           0 :     if (!LegalOperations ||
    8817           0 :         TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))
    8818           0 :       return DAG.getSignExtendVectorInReg(N0.getOperand(0), SDLoc(N), VT);
    8819             :   }
    8820             : 
    8821             :   // fold (sext_in_reg (zext x)) -> (sext x)
    8822             :   // iff we are extending the source sign bit.
    8823       42151 :   if (N0.getOpcode() == ISD::ZERO_EXTEND) {
    8824           5 :     SDValue N00 = N0.getOperand(0);
    8825           9 :     if (N00.getScalarValueSizeInBits() == EVTBits &&
    8826           4 :         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
    8827          12 :       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
    8828             :   }
    8829             : 
    8830             :   // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
    8831       84294 :   if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, EVTBits - 1)))
    8832           6 :     return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT.getScalarType());
    8833             : 
    8834             :   // fold operands of sext_in_reg based on knowledge that the top bits are not
    8835             :   // demanded.
    8836       42145 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    8837        2093 :     return SDValue(N, 0);
    8838             : 
    8839             :   // fold (sext_in_reg (load x)) -> (smaller sextload x)
    8840             :   // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
    8841       40052 :   if (SDValue NarrowLoad = ReduceLoadWidth(N))
    8842         165 :     return NarrowLoad;
    8843             : 
    8844             :   // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
    8845             :   // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
    8846             :   // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
    8847       39887 :   if (N0.getOpcode() == ISD::SRL) {
    8848             :     if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
    8849       19750 :       if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
    8850             :         // We can turn this into an SRA iff the input to the SRL is already sign
    8851             :         // extended enough.
    8852       19748 :         unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
    8853       19748 :         if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
    8854        4516 :           return DAG.getNode(ISD::SRA, SDLoc(N), VT,
    8855        4516 :                              N0.getOperand(0), N0.getOperand(1));
    8856             :       }
    8857             :   }
    8858             : 
    8859             :   // fold (sext_inreg (extload x)) -> (sextload x)
    8860             :   // If sextload is not supported by target, we can only do the combine when
    8861             :   // load has one use. Doing otherwise can block folding the extload with other
    8862             :   // extends that the target does support.
    8863             :   if (ISD::isEXTLoad(N0.getNode()) &&
    8864          27 :       ISD::isUNINDEXEDLoad(N0.getNode()) &&
    8865        1614 :       EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
    8866        3059 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile() &&
    8867         936 :         N0.hasOneUse()) ||
    8868         936 :        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
    8869             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8870        1502 :     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
    8871             :                                      LN0->getChain(),
    8872             :                                      LN0->getBasePtr(), EVT,
    8873        1502 :                                      LN0->getMemOperand());
    8874             :     CombineTo(N, ExtLoad);
    8875             :     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    8876         751 :     AddToWorklist(ExtLoad.getNode());
    8877         751 :     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8878             :   }
    8879             :   // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
    8880          63 :   if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
    8881           0 :       N0.hasOneUse() &&
    8882           0 :       EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
    8883           0 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
    8884           0 :        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
    8885             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8886           0 :     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
    8887             :                                      LN0->getChain(),
    8888             :                                      LN0->getBasePtr(), EVT,
    8889           0 :                                      LN0->getMemOperand());
    8890             :     CombineTo(N, ExtLoad);
    8891             :     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    8892           0 :     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8893             :   }
    8894             : 
    8895             :   // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
    8896       72955 :   if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
    8897          58 :     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
    8898          58 :                                            N0.getOperand(1), false))
    8899          16 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    8900          16 :                          BSwap, N1);
    8901             :   }
    8902             : 
    8903       36870 :   return SDValue();
    8904             : }
    8905             : 
    8906        2770 : SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
    8907        2770 :   SDValue N0 = N->getOperand(0);
    8908        5540 :   EVT VT = N->getValueType(0);
    8909             : 
    8910        2770 :   if (N0.isUndef())
    8911           0 :     return DAG.getUNDEF(VT);
    8912             : 
    8913        5540 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8914        5540 :                                               LegalOperations))
    8915          30 :     return SDValue(Res, 0);
    8916             : 
    8917        2740 :   return SDValue();
    8918             : }
    8919             : 
    8920        5441 : SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
    8921        5441 :   SDValue N0 = N->getOperand(0);
    8922       10882 :   EVT VT = N->getValueType(0);
    8923             : 
    8924        5441 :   if (N0.isUndef())
    8925           0 :     return DAG.getUNDEF(VT);
    8926             : 
    8927       10882 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8928       10882 :                                               LegalOperations))
    8929          31 :     return SDValue(Res, 0);
    8930             : 
    8931        5410 :   return SDValue();
    8932             : }
    8933             : 
    8934      194064 : SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
    8935      194064 :   SDValue N0 = N->getOperand(0);
    8936      388128 :   EVT VT = N->getValueType(0);
    8937      388128 :   bool isLE = DAG.getDataLayout().isLittleEndian();
    8938             : 
    8939             :   // noop truncate
    8940      388568 :   if (N0.getValueType() == N->getValueType(0))
    8941           0 :     return N0;
    8942             : 
    8943             :   // fold (truncate (truncate x)) -> (truncate x)
    8944      194064 :   if (N0.getOpcode() == ISD::TRUNCATE)
    8945        6753 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
    8946             : 
    8947             :   // fold (truncate c1) -> c1
    8948      191813 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
    8949        9720 :     SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
    8950        3240 :     if (C.getNode() != N)
    8951        3218 :       return C;
    8952             :   }
    8953             : 
    8954             :   // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
    8955      375510 :   if (N0.getOpcode() == ISD::ZERO_EXTEND ||
    8956      374292 :       N0.getOpcode() == ISD::SIGN_EXTEND ||
    8957             :       N0.getOpcode() == ISD::ANY_EXTEND) {
    8958             :     // if the source is smaller than the dest, we still need an extend.
    8959        8048 :     if (N0.getOperand(0).getValueType().bitsLT(VT))
    8960        4844 :       return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
    8961             :     // if the source is larger than the dest, than we just need the truncate.
    8962        2813 :     if (N0.getOperand(0).getValueType().bitsGT(VT))
    8963        1803 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
    8964             :     // if the source and dest are the same type, we can drop both the extend
    8965             :     // and the truncate.
    8966        2212 :     return N0.getOperand(0);
    8967             :   }
    8968             : 
    8969             :   // If this is anyext(trunc), don't fold it, allow ourselves to be folded.
    8970      171951 :   if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND))
    8971        1131 :     return SDValue();
    8972             : 
    8973             :   // Fold extract-and-trunc into a narrow extract. For example:
    8974             :   //   i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
    8975             :   //   i32 y = TRUNCATE(i64 x)
    8976             :   //        -- becomes --
    8977             :   //   v16i8 b = BITCAST (v2i64 val)
    8978             :   //   i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
    8979             :   //
    8980             :   // Note: We only run this optimization after type legalization (which often
    8981             :   // creates this pattern) and before operation legalization after which
    8982             :   // we need to be more careful about the vector instructions that we generate.
    8983       16109 :   if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
    8984      221780 :       LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) {
    8985       12704 :     EVT VecTy = N0.getOperand(0).getValueType();
    8986       12704 :     EVT ExTy = N0.getValueType();
    8987       12704 :     EVT TrTy = N->getValueType(0);
    8988             : 
    8989        6352 :     unsigned NumElem = VecTy.getVectorNumElements();
    8990        6352 :     unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits();
    8991             : 
    8992        6352 :     EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
    8993             :     assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
    8994             : 
    8995        6352 :     SDValue EltNo = N0->getOperand(1);
    8996        6352 :     if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
    8997        6336 :       int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
    8998       12672 :       EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
    8999        6336 :       int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
    9000             : 
    9001             :       SDLoc DL(N);
    9002        6336 :       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
    9003        6336 :                          DAG.getBitcast(NVT, N0.getOperand(0)),
    9004       12672 :                          DAG.getConstant(Index, DL, IndexTy));
    9005             :     }
    9006             :   }
    9007             : 
    9008             :   // trunc (select c, a, b) -> select c, (trunc a), (trunc b)
    9009      354388 :   if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) {
    9010         228 :     EVT SrcVT = N0.getValueType();
    9011         228 :     if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) &&
    9012         114 :         TLI.isTruncateFree(SrcVT, VT)) {
    9013             :       SDLoc SL(N0);
    9014         104 :       SDValue Cond = N0.getOperand(0);
    9015         104 :       SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
    9016         156 :       SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
    9017         156 :       return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1);
    9018             :     }
    9019             :   }
    9020             : 
    9021             :   // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits()
    9022      180522 :   if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
    9023      179940 :       (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) &&
    9024        1120 :       TLI.isTypeDesirableForOp(ISD::SHL, VT)) {
    9025        1826 :     SDValue Amt = N0.getOperand(1);
    9026         419 :     KnownBits Known;
    9027         913 :     DAG.computeKnownBits(Amt, Known);
    9028             :     unsigned Size = VT.getScalarSizeInBits();
    9029        1826 :     if (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size)) {
    9030             :       SDLoc SL(N);
    9031         988 :       EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    9032             : 
    9033        1482 :       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
    9034         494 :       if (AmtVT != Amt.getValueType()) {
    9035           7 :         Amt = DAG.getZExtOrTrunc(Amt, SL, AmtVT);
    9036           7 :         AddToWorklist(Amt.getNode());
    9037             :       }
    9038         988 :       return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt);
    9039             :     }
    9040             :   }
    9041             : 
    9042             :   // Fold a series of buildvector, bitcast, and truncate if possible.
    9043             :   // For example fold
    9044             :   //   (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to
    9045             :   //   (2xi32 (buildvector x, y)).
    9046      190109 :   if (Level == AfterLegalizeVectorOps && VT.isVector() &&
    9047        1260 :       N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
    9048      176629 :       N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
    9049           1 :       N0.getOperand(0).hasOneUse()) {
    9050           2 :     SDValue BuildVect = N0.getOperand(0);
    9051           1 :     EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType();
    9052           1 :     EVT TruncVecEltTy = VT.getVectorElementType();
    9053             : 
    9054             :     // Check that the element types match.
    9055           1 :     if (BuildVectEltTy == TruncVecEltTy) {
    9056             :       // Now we only need to compute the offset of the truncated elements.
    9057             :       unsigned BuildVecNumElts =  BuildVect.getNumOperands();
    9058           1 :       unsigned TruncVecNumElts = VT.getVectorNumElements();
    9059           1 :       unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
    9060             : 
    9061             :       assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
    9062             :              "Invalid number of elements");
    9063             : 
    9064             :       SmallVector<SDValue, 8> Opnds;
    9065           5 :       for (unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset)
    9066           2 :         Opnds.push_back(BuildVect.getOperand(i));
    9067             : 
    9068           3 :       return DAG.getBuildVector(VT, SDLoc(N), Opnds);
    9069             :     }
    9070             :   }
    9071             : 
    9072             :   // See if we can simplify the input to this truncate through knowledge that
    9073             :   // only the low bits are being used.
    9074             :   // For example "trunc (or (shl x, 8), y)" // -> trunc y
    9075             :   // Currently we only perform this optimization on scalars because vectors
    9076             :   // may have different active low bits.
    9077      176557 :   if (!VT.isVector()) {
    9078             :     APInt Mask =
    9079      166889 :         APInt::getLowBitsSet(N0.getValueSizeInBits(), VT.getSizeInBits());
    9080      166889 :     if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
    9081        4737 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter);
    9082             :   }
    9083             : 
    9084             :   // fold (truncate (load x)) -> (smaller load x)
    9085             :   // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
    9086      295568 :   if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
    9087      156365 :     if (SDValue Reduced = ReduceLoadWidth(N))
    9088         859 :       return Reduced;
    9089             : 
    9090             :     // Handle the case where the load remains an extending load even
    9091             :     // after truncation.
    9092      311012 :     if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
    9093             :       LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9094        3200 :       if (!LN0->isVolatile() &&
    9095        4554 :           LN0->getMemoryVT().getStoreSizeInBits() < VT.getSizeInBits()) {
    9096         380 :         SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
    9097             :                                          VT, LN0->getChain(), LN0->getBasePtr(),
    9098             :                                          LN0->getMemoryVT(),
    9099         380 :                                          LN0->getMemOperand());
    9100         190 :         DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
    9101         190 :         return NewLoad;
    9102             :       }
    9103             :     }
    9104             :   }
    9105             : 
    9106             :   // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
    9107             :   // where ... are all 'undef'.
    9108      347858 :   if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
    9109             :     SmallVector<EVT, 8> VTs;
    9110             :     SDValue V;
    9111             :     unsigned Idx = 0;
    9112             :     unsigned NumDefs = 0;
    9113             : 
    9114         613 :     for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
    9115         780 :       SDValue X = N0.getOperand(i);
    9116         390 :       if (!X.isUndef()) {
    9117             :         V = X;
    9118             :         Idx = i;
    9119         364 :         NumDefs++;
    9120             :       }
    9121             :       // Stop if more than one members are non-undef.
    9122         390 :       if (NumDefs > 1)
    9123             :         break;
    9124         213 :       VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
    9125             :                                      VT.getVectorElementType(),
    9126         426 :                                      X.getValueType().getVectorNumElements()));
    9127             :     }
    9128             : 
    9129         187 :     if (NumDefs == 0)
    9130           0 :       return DAG.getUNDEF(VT);
    9131             : 
    9132         187 :     if (NumDefs == 1) {
    9133             :       assert(V.getNode() && "The single defined operand is empty!");
    9134             :       SmallVector<SDValue, 8> Opnds;
    9135          46 :       for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
    9136          62 :         if (i != Idx) {
    9137          52 :           Opnds.push_back(DAG.getUNDEF(VTs[i]));
    9138          26 :           continue;
    9139             :         }
    9140          40 :         SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
    9141          10 :         AddToWorklist(NV.getNode());
    9142          10 :         Opnds.push_back(NV);
    9143             :       }
    9144          30 :       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds);
    9145             :     }
    9146             :   }
    9147             : 
    9148             :   // Fold truncate of a bitcast of a vector to an extract of the low vector
    9149             :   // element.
    9150             :   //
    9151             :   // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx
    9152      364543 :   if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) {
    9153       16486 :     SDValue VecSrc = N0.getOperand(0);
    9154       16486 :     EVT SrcVT = VecSrc.getValueType();
    9155       44413 :     if (SrcVT.isVector() && SrcVT.getScalarType() == VT &&
    9156       11441 :         (!LegalOperations ||
    9157       10553 :          TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, SrcVT))) {
    9158             :       SDLoc SL(N);
    9159             : 
    9160       20786 :       EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
    9161       10393 :       unsigned Idx = isLE ? 0 : SrcVT.getVectorNumElements() - 1;
    9162       10393 :       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT,
    9163       10393 :                          VecSrc, DAG.getConstant(Idx, SL, IdxVT));
    9164             :     }
    9165             :   }
    9166             : 
    9167             :   // Simplify the operands using demanded-bits information.
    9168      317423 :   if (!VT.isVector() &&
    9169      317423 :       SimplifyDemandedBits(SDValue(N, 0)))
    9170        7888 :     return SDValue(N, 0);
    9171             : 
    9172             :   // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
    9173             :   // (trunc addcarry(X, Y, Carry)) -> (addcarry trunc(X), trunc(Y), Carry)
    9174             :   // When the adde's carry is not used.
    9175      311299 :   if ((N0.getOpcode() == ISD::ADDE || N0.getOpcode() == ISD::ADDCARRY) &&
    9176      155703 :       N0.hasOneUse() && !N0.getNode()->hasAnyUseOfValue(1) &&
    9177          23 :       (!LegalOperations || TLI.isOperationLegal(N0.getOpcode(), VT))) {
    9178             :     SDLoc SL(N);
    9179          60 :     auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
    9180          60 :     auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
    9181          40 :     auto VTs = DAG.getVTList(VT, N0->getValueType(1));
    9182          40 :     return DAG.getNode(N0.getOpcode(), SL, VTs, X, Y, N0.getOperand(2));
    9183             :   }
    9184             : 
    9185             :   // fold (truncate (extract_subvector(ext x))) ->
    9186             :   //      (extract_subvector x)
    9187             :   // TODO: This can be generalized to cover cases where the truncate and extract
    9188             :   // do not fully cancel each other out.
    9189      261216 :   if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
    9190         143 :     SDValue N00 = N0.getOperand(0);
    9191          15 :     if (N00.getOpcode() == ISD::SIGN_EXTEND ||
    9192         150 :         N00.getOpcode() == ISD::ZERO_EXTEND ||
    9193             :         N00.getOpcode() == ISD::ANY_EXTEND) {
    9194         544 :       if (N00.getOperand(0)->getValueType(0).getVectorElementType() ==
    9195             :           VT.getVectorElementType())
    9196         272 :         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
    9197         408 :                            N00.getOperand(0), N0.getOperand(1));
    9198             :     }
    9199             :   }
    9200             : 
    9201      155482 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    9202           4 :     return NewVSel;
    9203             : 
    9204      155478 :   return SDValue();
    9205             : }
    9206             : 
    9207             : static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
    9208       47616 :   SDValue Elt = N->getOperand(i);
    9209       47616 :   if (Elt.getOpcode() != ISD::MERGE_VALUES)
    9210             :     return Elt.getNode();
    9211         109 :   return Elt.getOperand(Elt.getResNo()).getNode();
    9212             : }
    9213             : 
    9214             : /// build_pair (load, load) -> load
    9215             : /// if load locations are consecutive.
    9216       23808 : SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
    9217             :   assert(N->getOpcode() == ISD::BUILD_PAIR);
    9218             : 
    9219             :   LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
    9220             :   LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
    9221             : 
    9222             :   // A BUILD_PAIR is always having the least significant part in elt 0 and the
    9223             :   // most significant part in elt 1. So when combining into one large load, we
    9224             :   // need to consider the endianness.
    9225       47616 :   if (DAG.getDataLayout().isBigEndian())
    9226             :     std::swap(LD1, LD2);
    9227             : 
    9228       26097 :   if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
    9229             :       LD1->getAddressSpace() != LD2->getAddressSpace())
    9230       21519 :     return SDValue();
    9231        4578 :   EVT LD1VT = LD1->getValueType(0);
    9232             :   unsigned LD1Bytes = LD1VT.getStoreSize();
    9233        2289 :   if (ISD::isNON_EXTLoad(LD2) && LD2->hasOneUse() &&
    9234        2289 :       DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) {
    9235        2165 :     unsigned Align = LD1->getAlignment();
    9236        4330 :     unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
    9237        4330 :         VT.getTypeForEVT(*DAG.getContext()));
    9238             : 
    9239        4141 :     if (NewAlign <= Align &&
    9240        1976 :         (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
    9241        3952 :       return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
    9242        5928 :                          LD1->getPointerInfo(), Align);
    9243             :   }
    9244             : 
    9245         313 :   return SDValue();
    9246             : }
    9247             : 
    9248             : static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
    9249             :   // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi
    9250             :   // and Lo parts; on big-endian machines it doesn't.
    9251          20 :   return DAG.getDataLayout().isBigEndian() ? 1 : 0;
    9252             : }
    9253             : 
    9254      481055 : static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
    9255             :                                     const TargetLowering &TLI) {
    9256             :   // If this is not a bitcast to an FP type or if the target doesn't have
    9257             :   // IEEE754-compliant FP logic, we're done.
    9258      962110 :   EVT VT = N->getValueType(0);
    9259      481055 :   if (!VT.isFloatingPoint() || !TLI.hasBitPreservingFPLogic(VT))
    9260      435007 :     return SDValue();
    9261             : 
    9262             :   // TODO: Use splat values for the constant-checking below and remove this
    9263             :   // restriction.
    9264       46048 :   SDValue N0 = N->getOperand(0);
    9265       46048 :   EVT SourceVT = N0.getValueType();
    9266       46048 :   if (SourceVT.isVector())
    9267       27197 :     return SDValue();
    9268             : 
    9269             :   unsigned FPOpcode;
    9270             :   APInt SignMask;
    9271       18851 :   switch (N0.getOpcode()) {
    9272          61 :   case ISD::AND:
    9273             :     FPOpcode = ISD::FABS;
    9274         122 :     SignMask = ~APInt::getSignMask(SourceVT.getSizeInBits());
    9275          61 :     break;
    9276          38 :   case ISD::XOR:
    9277             :     FPOpcode = ISD::FNEG;
    9278          76 :     SignMask = APInt::getSignMask(SourceVT.getSizeInBits());
    9279          38 :     break;
    9280             :   // TODO: ISD::OR --> ISD::FNABS?
    9281       18752 :   default:
    9282       18752 :     return SDValue();
    9283             :   }
    9284             : 
    9285             :   // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X
    9286             :   // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X
    9287          99 :   SDValue LogicOp0 = N0.getOperand(0);
    9288             :   ConstantSDNode *LogicOp1 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    9289         116 :   if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask &&
    9290           0 :       LogicOp0.getOpcode() == ISD::BITCAST &&
    9291          16 :       LogicOp0->getOperand(0).getValueType() == VT)
    9292          16 :     return DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0->getOperand(0));
    9293             : 
    9294          91 :   return SDValue();
    9295             : }
    9296             : 
    9297      587130 : SDValue DAGCombiner::visitBITCAST(SDNode *N) {
    9298      587130 :   SDValue N0 = N->getOperand(0);
    9299     1174260 :   EVT VT = N->getValueType(0);
    9300             : 
    9301      587130 :   if (N0.isUndef())
    9302         195 :     return DAG.getUNDEF(VT);
    9303             : 
    9304             :   // If the input is a BUILD_VECTOR with all constant elements, fold this now.
    9305             :   // Only do this before legalize, since afterward the target may be depending
    9306             :   // on the bitconvert.
    9307             :   // First check to see if this is all constant.
    9308      711411 :   if (!LegalTypes &&
    9309      657100 :       N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
    9310             :       VT.isVector()) {
    9311       69301 :     bool isSimple = cast<BuildVectorSDNode>(N0)->isConstant();
    9312             : 
    9313      138602 :     EVT DestEltVT = N->getValueType(0).getVectorElementType();
    9314             :     assert(!DestEltVT.isVector() &&
    9315             :            "Element type of vector ValueType must not be vector!");
    9316       69301 :     if (isSimple)
    9317       68372 :       return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), DestEltVT);
    9318             :   }
    9319             : 
    9320             :   // If the input is a constant, let getNode fold it.
    9321             :   // We always need to check that this is just a fp -> int or int -> conversion
    9322             :   // otherwise we will get back N which will confuse the caller into thinking
    9323             :   // we used CombineTo. This can block target combines from running. If we can't
    9324             :   // allowed legal operations, we need to ensure the resulting operation will be
    9325             :   // legal.
    9326             :   // TODO: Maybe we should check that the return value isn't N explicitly?
    9327         996 :   if ((isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() &&
    9328          74 :        (!LegalOperations || TLI.isOperationLegal(ISD::ConstantFP, VT))) ||
    9329         222 :       (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() &&
    9330          79 :        (!LegalOperations || TLI.isOperationLegal(ISD::Constant, VT))))
    9331         117 :     return DAG.getBitcast(VT, N0);
    9332             : 
    9333             :   // (conv (conv x, t1), t2) -> (conv x, t2)
    9334      518446 :   if (N0.getOpcode() == ISD::BITCAST)
    9335       65072 :     return DAG.getBitcast(VT, N0.getOperand(0));
    9336             : 
    9337             :   // fold (conv (load x)) -> (load (conv*)x)
    9338             :   // If the resultant load doesn't need a higher alignment than the original!
    9339      163012 :   if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
    9340             :       // Do not change the width of a volatile load.
    9341       75823 :       !cast<LoadSDNode>(N0)->isVolatile() &&
    9342             :       // Do not remove the cast if the types differ in endian layout.
    9343       75823 :       TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) ==
    9344      227467 :           TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) &&
    9345      153126 :       (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
    9346       22746 :       TLI.isLoadBitCastBeneficial(N0.getValueType(), VT)) {
    9347             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9348        4956 :     unsigned OrigAlign = LN0->getAlignment();
    9349             : 
    9350        4956 :     bool Fast = false;
    9351       14868 :     if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
    9352        4956 :                                LN0->getAddressSpace(), OrigAlign, &Fast) &&
    9353             :         Fast) {
    9354             :       SDValue Load =
    9355        9710 :           DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
    9356             :                       LN0->getPointerInfo(), OrigAlign,
    9357       19420 :                       LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    9358        4855 :       DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
    9359        4855 :       return Load;
    9360             :     }
    9361             :   }
    9362             : 
    9363      481055 :   if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
    9364           8 :     return V;
    9365             : 
    9366             :   // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
    9367             :   // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
    9368             :   //
    9369             :   // For ppc_fp128:
    9370             :   // fold (bitcast (fneg x)) ->
    9371             :   //     flipbit = signbit
    9372             :   //     (xor (bitcast x) (build_pair flipbit, flipbit))
    9373             :   //
    9374             :   // fold (bitcast (fabs x)) ->
    9375             :   //     flipbit = (and (extract_element (bitcast x), 0), signbit)
    9376             :   //     (xor (bitcast x) (build_pair flipbit, flipbit))
    9377             :   // This often reduces constant pool loads.
    9378      481691 :   if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) ||
    9379         314 :        (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) &&
    9380         333 :       N0.getNode()->hasOneUse() && VT.isInteger() &&
    9381      481292 :       !VT.isVector() && !N0.getValueType().isVector()) {
    9382          94 :     SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
    9383          47 :     AddToWorklist(NewConv.getNode());
    9384             : 
    9385             :     SDLoc DL(N);
    9386          10 :     if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
    9387             :       assert(VT.getSizeInBits() == 128);
    9388          10 :       SDValue SignBit = DAG.getConstant(
    9389          40 :           APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64);
    9390          10 :       SDValue FlipBit;
    9391          10 :       if (N0.getOpcode() == ISD::FNEG) {
    9392           5 :         FlipBit = SignBit;
    9393           5 :         AddToWorklist(FlipBit.getNode());
    9394             :       } else {
    9395             :         assert(N0.getOpcode() == ISD::FABS);
    9396             :         SDValue Hi =
    9397          10 :             DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv,
    9398           5 :                         DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
    9399          20 :                                               SDLoc(NewConv)));
    9400           5 :         AddToWorklist(Hi.getNode());
    9401          15 :         FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit);
    9402           5 :         AddToWorklist(FlipBit.getNode());
    9403             :       }
    9404             :       SDValue FlipBits =
    9405          30 :           DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
    9406          10 :       AddToWorklist(FlipBits.getNode());
    9407          20 :       return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits);
    9408             :     }
    9409          37 :     APInt SignBit = APInt::getSignMask(VT.getSizeInBits());
    9410          37 :     if (N0.getOpcode() == ISD::FNEG)
    9411          27 :       return DAG.getNode(ISD::XOR, DL, VT,
    9412          27 :                          NewConv, DAG.getConstant(SignBit, DL, VT));
    9413             :     assert(N0.getOpcode() == ISD::FABS);
    9414          10 :     return DAG.getNode(ISD::AND, DL, VT,
    9415          30 :                        NewConv, DAG.getConstant(~SignBit, DL, VT));
    9416             :   }
    9417             : 
    9418             :   // fold (bitconvert (fcopysign cst, x)) ->
    9419             :   //         (or (and (bitconvert x), sign), (and cst, (not sign)))
    9420             :   // Note that we don't handle (copysign x, cst) because this can always be
    9421             :   // folded to an fneg or fabs.
    9422             :   //
    9423             :   // For ppc_fp128:
    9424             :   // fold (bitcast (fcopysign cst, x)) ->
    9425             :   //     flipbit = (and (extract_element
    9426             :   //                     (xor (bitcast cst), (bitcast x)), 0),
    9427             :   //                    signbit)
    9428             :   //     (xor (bitcast cst) (build_pair flipbit, flipbit))
    9429             :   if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() &&
    9430           6 :       isa<ConstantFPSDNode>(N0.getOperand(0)) &&
    9431      481012 :       VT.isInteger() && !VT.isVector()) {
    9432           6 :     unsigned OrigXWidth = N0.getOperand(1).getValueSizeInBits();
    9433           6 :     EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
    9434           6 :     if (isTypeLegal(IntXVT)) {
    9435          12 :       SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
    9436           6 :       AddToWorklist(X.getNode());
    9437             : 
    9438             :       // If X has a different width than the result/lhs, sext it or truncate it.
    9439           6 :       unsigned VTWidth = VT.getSizeInBits();
    9440           6 :       if (OrigXWidth < VTWidth) {
    9441           0 :         X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X);
    9442           0 :         AddToWorklist(X.getNode());
    9443           6 :       } else if (OrigXWidth > VTWidth) {
    9444             :         // To get the sign bit in the right place, we have to shift it right
    9445             :         // before truncating.
    9446             :         SDLoc DL(X);
    9447           0 :         X = DAG.getNode(ISD::SRL, DL,
    9448             :                         X.getValueType(), X,
    9449           0 :                         DAG.getConstant(OrigXWidth-VTWidth, DL,
    9450           0 :                                         X.getValueType()));
    9451           0 :         AddToWorklist(X.getNode());
    9452           0 :         X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
    9453           0 :         AddToWorklist(X.getNode());
    9454             :       }
    9455             : 
    9456           5 :       if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
    9457           5 :         APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2);
    9458          10 :         SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
    9459           5 :         AddToWorklist(Cst.getNode());
    9460          10 :         SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
    9461           5 :         AddToWorklist(X.getNode());
    9462          15 :         SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
    9463           5 :         AddToWorklist(XorResult.getNode());
    9464           5 :         SDValue XorResult64 = DAG.getNode(
    9465           5 :             ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult,
    9466           5 :             DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
    9467          20 :                                   SDLoc(XorResult)));
    9468           5 :         AddToWorklist(XorResult64.getNode());
    9469             :         SDValue FlipBit =
    9470          10 :             DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64,
    9471          20 :                         DAG.getConstant(SignBit, SDLoc(XorResult64), MVT::i64));
    9472           5 :         AddToWorklist(FlipBit.getNode());
    9473             :         SDValue FlipBits =
    9474          15 :             DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
    9475           5 :         AddToWorklist(FlipBits.getNode());
    9476          15 :         return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits);
    9477             :       }
    9478           1 :       APInt SignBit = APInt::getSignMask(VT.getSizeInBits());
    9479           3 :       X = DAG.getNode(ISD::AND, SDLoc(X), VT,
    9480           2 :                       X, DAG.getConstant(SignBit, SDLoc(X), VT));
    9481           1 :       AddToWorklist(X.getNode());
    9482             : 
    9483           2 :       SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
    9484           3 :       Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
    9485           4 :                         Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT));
    9486           1 :       AddToWorklist(Cst.getNode());
    9487             : 
    9488           3 :       return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
    9489             :     }
    9490             :   }
    9491             : 
    9492             :   // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
    9493      480994 :   if (N0.getOpcode() == ISD::BUILD_PAIR)
    9494        4462 :     if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT))
    9495          38 :       return CombineLD;
    9496             : 
    9497             :   // Remove double bitcasts from shuffles - this is often a legacy of
    9498             :   // XformToShuffleWithZero being used to combine bitmaskings (of
    9499             :   // float vectors bitcast to integer vectors) into shuffles.
    9500             :   // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1)
    9501      926833 :   if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() &&
    9502      143018 :       N0->getOpcode() == ISD::VECTOR_SHUFFLE &&
    9503      991930 :       VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() &&
    9504      481686 :       !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) {
    9505             :     ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N0);
    9506             : 
    9507             :     // If operands are a bitcast, peek through if it casts the original VT.
    9508             :     // If operands are a constant, just bitcast back to original VT.
    9509        1460 :     auto PeekThroughBitcast = [&](SDValue Op) {
    9510        1460 :       if (Op.getOpcode() == ISD::BITCAST &&
    9511        2577 :           Op.getOperand(0).getValueType() == VT)
    9512         350 :         return SDValue(Op.getOperand(0));
    9513        2426 :       if (Op.isUndef() || ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
    9514         645 :           ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
    9515         930 :         return DAG.getBitcast(VT, Op);
    9516         645 :       return SDValue();
    9517         730 :     };
    9518             : 
    9519             :     // FIXME: If either input vector is bitcast, try to convert the shuffle to
    9520             :     // the result type of this bitcast. This would eliminate at least one
    9521             :     // bitcast. See the transform in InstCombine.
    9522        1460 :     SDValue SV0 = PeekThroughBitcast(N0->getOperand(0));
    9523        1460 :     SDValue SV1 = PeekThroughBitcast(N0->getOperand(1));
    9524         730 :     if (!(SV0 && SV1))
    9525         509 :       return SDValue();
    9526             : 
    9527             :     int MaskScale =
    9528         442 :         VT.getVectorNumElements() / N0.getValueType().getVectorNumElements();
    9529             :     SmallVector<int, 8> NewMask;
    9530        1997 :     for (int M : SVN->getMask())
    9531        5740 :       for (int i = 0; i != MaskScale; ++i)
    9532        2426 :         NewMask.push_back(M < 0 ? -1 : M * MaskScale + i);
    9533             : 
    9534         442 :     bool LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
    9535         221 :     if (!LegalMask) {
    9536             :       std::swap(SV0, SV1);
    9537             :       ShuffleVectorSDNode::commuteMask(NewMask);
    9538          10 :       LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
    9539             :     }
    9540             : 
    9541         221 :     if (LegalMask)
    9542         648 :       return DAG.getVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask);
    9543             :   }
    9544             : 
    9545      480231 :   return SDValue();
    9546             : }
    9547             : 
    9548             : SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
    9549       38692 :   EVT VT = N->getValueType(0);
    9550       19346 :   return CombineConsecutiveLoads(N, VT);
    9551             : }
    9552             : 
    9553             : /// We know that BV is a build_vector node with Constant, ConstantFP or Undef
    9554             : /// operands. DstEltVT indicates the destination element value type.
    9555       68498 : SDValue DAGCombiner::
    9556             : ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
    9557      136996 :   EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
    9558             : 
    9559             :   // If this is already the right type, we're done.
    9560       68498 :   if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
    9561             : 
    9562       68498 :   unsigned SrcBitSize = SrcEltVT.getSizeInBits();
    9563       68498 :   unsigned DstBitSize = DstEltVT.getSizeInBits();
    9564             : 
    9565             :   // If this is a conversion of N elements of one type to N elements of another
    9566             :   // type, convert each element.  This handles FP<->INT cases.
    9567       68498 :   if (SrcBitSize == DstBitSize) {
    9568         202 :     EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
    9569         606 :                               BV->getValueType(0).getVectorNumElements());
    9570             : 
    9571             :     // Due to the FP element handling below calling this routine recursively,
    9572             :     // we can end up with a scalar-to-vector node here.
    9573         202 :     if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR)
    9574           0 :       return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BV), VT,
    9575           0 :                          DAG.getBitcast(DstEltVT, BV->getOperand(0)));
    9576             : 
    9577             :     SmallVector<SDValue, 8> Ops;
    9578         878 :     for (SDValue Op : BV->op_values()) {
    9579             :       // If the vector element type is not legal, the BUILD_VECTOR operands
    9580             :       // are promoted and implicitly truncated.  Make that explicit here.
    9581         676 :       if (Op.getValueType() != SrcEltVT)
    9582           0 :         Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op);
    9583         676 :       Ops.push_back(DAG.getBitcast(DstEltVT, Op));
    9584         676 :       AddToWorklist(Ops.back().getNode());
    9585             :     }
    9586         606 :     return DAG.getBuildVector(VT, SDLoc(BV), Ops);
    9587             :   }
    9588             : 
    9589             :   // Otherwise, we're growing or shrinking the elements.  To avoid having to
    9590             :   // handle annoying details of growing/shrinking FP values, we convert them to
    9591             :   // int first.
    9592       68296 :   if (SrcEltVT.isFloatingPoint()) {
    9593             :     // Convert the input float vector to a int vector where the elements are the
    9594             :     // same sizes.
    9595          44 :     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
    9596          44 :     BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
    9597          44 :     SrcEltVT = IntVT;
    9598             :   }
    9599             : 
    9600             :   // Now we know the input is an integer vector.  If the output is a FP type,
    9601             :   // convert to integer first, then to FP of the right size.
    9602       68296 :   if (DstEltVT.isFloatingPoint()) {
    9603          41 :     EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
    9604          41 :     SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
    9605             : 
    9606             :     // Next, convert to FP elements of the same size.
    9607          41 :     return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
    9608             :   }
    9609             : 
    9610             :   SDLoc DL(BV);
    9611             : 
    9612             :   // Okay, we know the src/dst types are both integers of differing types.
    9613             :   // Handling growing first.
    9614             :   assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
    9615       68255 :   if (SrcBitSize < DstBitSize) {
    9616       67895 :     unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
    9617             : 
    9618             :     SmallVector<SDValue, 8> Ops;
    9619      407706 :     for (unsigned i = 0, e = BV->getNumOperands(); i != e;
    9620      135958 :          i += NumInputsPerOutput) {
    9621      271916 :       bool isLE = DAG.getDataLayout().isLittleEndian();
    9622             :       APInt NewBits = APInt(DstBitSize, 0);
    9623             :       bool EltIsUndef = true;
    9624      680686 :       for (unsigned j = 0; j != NumInputsPerOutput; ++j) {
    9625             :         // Shift the previously computed bits over.
    9626      272364 :         NewBits <<= SrcBitSize;
    9627      544728 :         SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
    9628      272364 :         if (Op.isUndef()) continue;
    9629             :         EltIsUndef = false;
    9630             : 
    9631      272214 :         NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue().
    9632      816642 :                    zextOrTrunc(SrcBitSize).zext(DstBitSize);
    9633             :       }
    9634             : 
    9635      135958 :       if (EltIsUndef)
    9636          12 :         Ops.push_back(DAG.getUNDEF(DstEltVT));
    9637             :       else
    9638      135946 :         Ops.push_back(DAG.getConstant(NewBits, DL, DstEltVT));
    9639             :     }
    9640             : 
    9641       67895 :     EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
    9642      135790 :     return DAG.getBuildVector(VT, DL, Ops);
    9643             :   }
    9644             : 
    9645             :   // Finally, this must be the case where we are shrinking elements: each input
    9646             :   // turns into multiple outputs.
    9647         360 :   unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
    9648         360 :   EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
    9649        1080 :                             NumOutputsPerInput*BV->getNumOperands());
    9650             :   SmallVector<SDValue, 8> Ops;
    9651             : 
    9652        1892 :   for (const SDValue &Op : BV->op_values()) {
    9653        3077 :     if (Op.isUndef()) {
    9654          13 :       Ops.append(NumOutputsPerInput, DAG.getUNDEF(DstEltVT));
    9655          13 :       continue;
    9656             :     }
    9657             : 
    9658             :     APInt OpVal = cast<ConstantSDNode>(Op)->
    9659        3038 :                   getAPIntValue().zextOrTrunc(SrcBitSize);
    9660             : 
    9661       11959 :     for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
    9662        5220 :       APInt ThisVal = OpVal.trunc(DstBitSize);
    9663        5220 :       Ops.push_back(DAG.getConstant(ThisVal, DL, DstEltVT));
    9664             :       OpVal.lshrInPlace(DstBitSize);
    9665             :     }
    9666             : 
    9667             :     // For big endian targets, swap the order of the pieces of each element.
    9668        3038 :     if (DAG.getDataLayout().isBigEndian())
    9669          76 :       std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
    9670             :   }
    9671             : 
    9672         720 :   return DAG.getBuildVector(VT, DL, Ops);
    9673             : }
    9674             : 
    9675             : static bool isContractable(SDNode *N) {
    9676             :   SDNodeFlags F = N->getFlags();
    9677       35168 :   return F.hasAllowContract() || F.hasAllowReassociation();
    9678             : }
    9679             : 
    9680             : /// Try to perform FMA combining on a given FADD node.
    9681       36222 : SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
    9682       36222 :   SDValue N0 = N->getOperand(0);
    9683       36222 :   SDValue N1 = N->getOperand(1);
    9684       72444 :   EVT VT = N->getValueType(0);
    9685             :   SDLoc SL(N);
    9686             : 
    9687       36222 :   const TargetOptions &Options = DAG.getTarget().Options;
    9688             : 
    9689             :   // Floating-point multiply-add with intermediate rounding.
    9690       36222 :   bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT));
    9691             : 
    9692             :   // Floating-point multiply-add without intermediate rounding.
    9693             :   bool HasFMA =
    9694       47433 :       TLI.isFMAFasterThanFMulAndFAdd(VT) &&
    9695       16219 :       (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT));
    9696             : 
    9697             :   // No valid opcode, do not combine.
    9698       36222 :   if (!HasFMAD && !HasFMA)
    9699       19502 :     return SDValue();
    9700             : 
    9701       16720 :   SDNodeFlags Flags = N->getFlags();
    9702       16720 :   bool CanFuse = Options.UnsafeFPMath || isContractable(N);
    9703       16720 :   bool AllowFusionGlobally = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
    9704       16720 :                               CanFuse || HasFMAD);
    9705             :   // If the addition is not contractable, do not combine.
    9706             :   if (!AllowFusionGlobally && !isContractable(N))
    9707        9034 :     return SDValue();
    9708             : 
    9709       15372 :   const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo();
    9710       15372 :   if (STI && STI->generateFMAsInMachineCombiner(OptLevel))
    9711          56 :     return SDValue();
    9712             : 
    9713             :   // Always prefer FMAD to FMA for precision.
    9714       15260 :   unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
    9715        7630 :   bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
    9716             : 
    9717             :   // Is the node an FMUL and contractable either due to global flags or
    9718             :   // SDNodeFlags.
    9719             :   auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
    9720       18682 :     if (N.getOpcode() != ISD::FMUL)
    9721             :       return false;
    9722        3882 :     return AllowFusionGlobally || isContractable(N.getNode());
    9723             :   };
    9724             :   // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
    9725             :   // prefer to fold the multiply with fewer uses.
    9726        7630 :   if (Aggressive && isContractableFMUL(N0) && isContractableFMUL(N1)) {
    9727         224 :     if (N0.getNode()->use_size() > N1.getNode()->use_size())
    9728             :       std::swap(N0, N1);
    9729             :   }
    9730             : 
    9731             :   // fold (fadd (fmul x, y), z) -> (fma x, y, z)
    9732        1950 :   if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) {
    9733        1931 :     return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9734        1931 :                        N0.getOperand(0), N0.getOperand(1), N1, Flags);
    9735             :   }
    9736             : 
    9737             :   // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
    9738             :   // Note: Commutes FADD operands.
    9739         326 :   if (isContractableFMUL(N1) && (Aggressive || N1->hasOneUse())) {
    9740         299 :     return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9741         299 :                        N1.getOperand(0), N1.getOperand(1), N0, Flags);
    9742             :   }
    9743             : 
    9744             :   // Look through FP_EXTEND nodes to do more combining.
    9745             : 
    9746             :   // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
    9747        5400 :   if (N0.getOpcode() == ISD::FP_EXTEND) {
    9748          17 :     SDValue N00 = N0.getOperand(0);
    9749           4 :     if (isContractableFMUL(N00) &&
    9750           8 :         TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) {
    9751           3 :       return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9752           3 :                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
    9753             :                                      N00.getOperand(0)),
    9754             :                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
    9755           6 :                                      N00.getOperand(1)), N1, Flags);
    9756             :     }
    9757             :   }
    9758             : 
    9759             :   // fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
    9760             :   // Note: Commutes FADD operands.
    9761        5397 :   if (N1.getOpcode() == ISD::FP_EXTEND) {
    9762          16 :     SDValue N10 = N1.getOperand(0);
    9763           4 :     if (isContractableFMUL(N10) &&
    9764           8 :         TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N10.getValueType())) {
    9765           3 :       return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9766           3 :                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
    9767             :                                      N10.getOperand(0)),
    9768             :                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
    9769           6 :                                      N10.getOperand(1)), N0, Flags);
    9770             :     }
    9771             :   }
    9772             : 
    9773             :   // More folding opportunities when target permits.
    9774        5394 :   if (Aggressive) {
    9775             :     // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, z))
    9776         436 :     if (CanFuse &&
    9777         455 :         N0.getOpcode() == PreferredFusedOpcode &&
    9778          19 :         N0.getOperand(2).getOpcode() == ISD::FMUL &&
    9779        2476 :         N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) {
    9780           7 :       return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9781             :                          N0.getOperand(0), N0.getOperand(1),
    9782             :                          DAG.getNode(PreferredFusedOpcode, SL, VT,
    9783             :                                      N0.getOperand(2).getOperand(0),
    9784             :                                      N0.getOperand(2).getOperand(1),
    9785          42 :                                      N1, Flags), Flags);
    9786             :     }
    9787             : 
    9788             :     // fold (fadd x, (fma y, z, (fmul u, v)) -> (fma y, z (fma u, v, x))
    9789         429 :     if (CanFuse &&
    9790         878 :         N1->getOpcode() == PreferredFusedOpcode &&
    9791          20 :         N1.getOperand(2).getOpcode() == ISD::FMUL &&
    9792        2469 :         N1->hasOneUse() && N1.getOperand(2)->hasOneUse()) {
    9793           2 :       return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9794             :                          N1.getOperand(0), N1.getOperand(1),
    9795             :                          DAG.getNode(PreferredFusedOpcode, SL, VT,
    9796             :                                      N1.getOperand(2).getOperand(0),
    9797             :                                      N1.getOperand(2).getOperand(1),
    9798           4 :                                      N0, Flags), Flags);
    9799             :     }
    9800             : 
    9801             : 
    9802             :     // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
    9803             :     //   -> (fma x, y, (fma (fpext u), (fpext v), z))
    9804             :     auto FoldFAddFMAFPExtFMul = [&] (
    9805             :       SDValue X, SDValue Y, SDValue U, SDValue V, SDValue Z,
    9806          11 :       SDNodeFlags Flags) {
    9807          66 :       return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
    9808             :                          DAG.getNode(PreferredFusedOpcode, SL, VT,
    9809          11 :                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
    9810             :                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
    9811          55 :                                      Z, Flags), Flags);
    9812        2489 :     };
    9813        2467 :     if (N0.getOpcode() == PreferredFusedOpcode) {
    9814          85 :       SDValue N02 = N0.getOperand(2);
    9815          85 :       if (N02.getOpcode() == ISD::FP_EXTEND) {
    9816           8 :         SDValue N020 = N02.getOperand(0);
    9817           8 :         if (isContractableFMUL(N020) &&
    9818          16 :             TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N020.getValueType())) {
    9819             :           return FoldFAddFMAFPExtFMul(N0.getOperand(0), N0.getOperand(1),
    9820             :                                       N020.getOperand(0), N020.getOperand(1),
    9821           6 :                                       N1, Flags);
    9822             :         }
    9823             :       }
    9824             :     }
    9825             : 
    9826             :     // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
    9827             :     //   -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
    9828             :     // FIXME: This turns two single-precision and one double-precision
    9829             :     // operation into two double-precision operations, which might not be
    9830             :     // interesting for all targets, especially GPUs.
    9831             :     auto FoldFAddFPExtFMAFMul = [&] (
    9832             :       SDValue X, SDValue Y, SDValue U, SDValue V, SDValue Z,
    9833           8 :       SDNodeFlags Flags) {
    9834          80 :       return DAG.getNode(PreferredFusedOpcode, SL, VT,
    9835           8 :                          DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
    9836           8 :                          DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
    9837             :                          DAG.getNode(PreferredFusedOpcode, SL, VT,
    9838           8 :                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
    9839             :                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
    9840          56 :                                      Z, Flags), Flags);
    9841        2477 :     };
    9842        2461 :     if (N0.getOpcode() == ISD::FP_EXTEND) {
    9843          14 :       SDValue N00 = N0.getOperand(0);
    9844          14 :       if (N00.getOpcode() == PreferredFusedOpcode) {
    9845           4 :         SDValue N002 = N00.getOperand(2);
    9846           4 :         if (isContractableFMUL(N002) &&
    9847           8 :             TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) {
    9848             :           return FoldFAddFPExtFMAFMul(N00.getOperand(0), N00.getOperand(1),
    9849             :                                       N002.getOperand(0), N002.getOperand(1),
    9850           4 :                                       N1, Flags);
    9851             :         }
    9852             :       }
    9853             :     }
    9854             : 
    9855             :     // fold (fadd x, (fma y, z, (fpext (fmul u, v)))
    9856             :     //   -> (fma y, z, (fma (fpext u), (fpext v), x))
    9857        2457 :     if (N1.getOpcode() == PreferredFusedOpcode) {
    9858          62 :       SDValue N12 = N1.getOperand(2);
    9859          62 :       if (N12.getOpcode() == ISD::FP_EXTEND) {
    9860           6 :         SDValue N120 = N12.getOperand(0);
    9861           6 :         if (isContractableFMUL(N120) &&
    9862          12 :             TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N120.getValueType())) {
    9863             :           return FoldFAddFMAFPExtFMul(N1.getOperand(0), N1.getOperand(1),
    9864             :                                       N120.getOperand(0), N120.getOperand(1),
    9865           5 :                                       N0, Flags);
    9866             :         }
    9867             :       }
    9868             :     }
    9869             : 
    9870             :     // fold (fadd x, (fpext (fma y, z, (fmul u, v)))
    9871             :     //   -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
    9872             :     // FIXME: This turns two single-precision and one double-precision
    9873             :     // operation into two double-precision operations, which might not be
    9874             :     // interesting for all targets, especially GPUs.
    9875        2452 :     if (N1.getOpcode() == ISD::FP_EXTEND) {
    9876          13 :       SDValue N10 = N1.getOperand(0);
    9877          13 :       if (N10.getOpcode() == PreferredFusedOpcode) {
    9878           4 :