LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - DAGCombiner.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 7059 8399 84.0 %
Date: 2018-09-23 13:06:45 Functions: 215 279 77.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
      11             : // both before and after the DAG is legalized.
      12             : //
      13             : // This pass is not a substitute for the LLVM IR instcombine pass. This pass is
      14             : // primarily intended to handle simplification opportunities that are implicit
      15             : // in the LLVM IR and exposed by the various codegen lowering phases.
      16             : //
      17             : //===----------------------------------------------------------------------===//
      18             : 
      19             : #include "llvm/ADT/APFloat.h"
      20             : #include "llvm/ADT/APInt.h"
      21             : #include "llvm/ADT/ArrayRef.h"
      22             : #include "llvm/ADT/DenseMap.h"
      23             : #include "llvm/ADT/None.h"
      24             : #include "llvm/ADT/Optional.h"
      25             : #include "llvm/ADT/STLExtras.h"
      26             : #include "llvm/ADT/SetVector.h"
      27             : #include "llvm/ADT/SmallBitVector.h"
      28             : #include "llvm/ADT/SmallPtrSet.h"
      29             : #include "llvm/ADT/SmallSet.h"
      30             : #include "llvm/ADT/SmallVector.h"
      31             : #include "llvm/ADT/Statistic.h"
      32             : #include "llvm/Analysis/AliasAnalysis.h"
      33             : #include "llvm/Analysis/MemoryLocation.h"
      34             : #include "llvm/CodeGen/DAGCombine.h"
      35             : #include "llvm/CodeGen/ISDOpcodes.h"
      36             : #include "llvm/CodeGen/MachineFrameInfo.h"
      37             : #include "llvm/CodeGen/MachineFunction.h"
      38             : #include "llvm/CodeGen/MachineMemOperand.h"
      39             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      40             : #include "llvm/CodeGen/SelectionDAG.h"
      41             : #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
      42             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      43             : #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
      44             : #include "llvm/CodeGen/TargetLowering.h"
      45             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      46             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      47             : #include "llvm/CodeGen/ValueTypes.h"
      48             : #include "llvm/IR/Attributes.h"
      49             : #include "llvm/IR/Constant.h"
      50             : #include "llvm/IR/DataLayout.h"
      51             : #include "llvm/IR/DerivedTypes.h"
      52             : #include "llvm/IR/Function.h"
      53             : #include "llvm/IR/LLVMContext.h"
      54             : #include "llvm/IR/Metadata.h"
      55             : #include "llvm/Support/Casting.h"
      56             : #include "llvm/Support/CodeGen.h"
      57             : #include "llvm/Support/CommandLine.h"
      58             : #include "llvm/Support/Compiler.h"
      59             : #include "llvm/Support/Debug.h"
      60             : #include "llvm/Support/ErrorHandling.h"
      61             : #include "llvm/Support/KnownBits.h"
      62             : #include "llvm/Support/MachineValueType.h"
      63             : #include "llvm/Support/MathExtras.h"
      64             : #include "llvm/Support/raw_ostream.h"
      65             : #include "llvm/Target/TargetMachine.h"
      66             : #include "llvm/Target/TargetOptions.h"
      67             : #include <algorithm>
      68             : #include <cassert>
      69             : #include <cstdint>
      70             : #include <functional>
      71             : #include <iterator>
      72             : #include <string>
      73             : #include <tuple>
      74             : #include <utility>
      75             : 
      76             : using namespace llvm;
      77             : 
      78             : #define DEBUG_TYPE "dagcombine"
      79             : 
      80             : STATISTIC(NodesCombined   , "Number of dag nodes combined");
      81             : STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
      82             : STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
      83             : STATISTIC(OpsNarrowed     , "Number of load/op/store narrowed");
      84             : STATISTIC(LdStFP2Int      , "Number of fp load/store pairs transformed to int");
      85             : STATISTIC(SlicedLoads, "Number of load sliced");
      86             : 
      87             : static cl::opt<bool>
      88             : CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
      89             :                  cl::desc("Enable DAG combiner's use of IR alias analysis"));
      90             : 
      91             : static cl::opt<bool>
      92             : UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
      93             :         cl::desc("Enable DAG combiner's use of TBAA"));
      94             : 
      95             : #ifndef NDEBUG
      96             : static cl::opt<std::string>
      97             : CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
      98             :                    cl::desc("Only use DAG-combiner alias analysis in this"
      99             :                             " function"));
     100             : #endif
     101             : 
     102             : /// Hidden option to stress test load slicing, i.e., when this option
     103             : /// is enabled, load slicing bypasses most of its profitability guards.
     104             : static cl::opt<bool>
     105             : StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
     106             :                   cl::desc("Bypass the profitability model of load slicing"),
     107             :                   cl::init(false));
     108             : 
     109             : static cl::opt<bool>
     110             :   MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
     111             :                     cl::desc("DAG combiner may split indexing from loads"));
     112             : 
     113             : namespace {
     114             : 
     115             :   class DAGCombiner {
     116             :     SelectionDAG &DAG;
     117             :     const TargetLowering &TLI;
     118             :     CombineLevel Level;
     119             :     CodeGenOpt::Level OptLevel;
     120             :     bool LegalOperations = false;
     121             :     bool LegalTypes = false;
     122             :     bool ForCodeSize;
     123             : 
     124             :     /// Worklist of all of the nodes that need to be simplified.
     125             :     ///
     126             :     /// This must behave as a stack -- new nodes to process are pushed onto the
     127             :     /// back and when processing we pop off of the back.
     128             :     ///
     129             :     /// The worklist will not contain duplicates but may contain null entries
     130             :     /// due to nodes being deleted from the underlying DAG.
     131             :     SmallVector<SDNode *, 64> Worklist;
     132             : 
     133             :     /// Mapping from an SDNode to its position on the worklist.
     134             :     ///
     135             :     /// This is used to find and remove nodes from the worklist (by nulling
     136             :     /// them) when they are deleted from the underlying DAG. It relies on
     137             :     /// stable indices of nodes within the worklist.
     138             :     DenseMap<SDNode *, unsigned> WorklistMap;
     139             : 
     140             :     /// Set of nodes which have been combined (at least once).
     141             :     ///
     142             :     /// This is used to allow us to reliably add any operands of a DAG node
     143             :     /// which have not yet been combined to the worklist.
     144             :     SmallPtrSet<SDNode *, 32> CombinedNodes;
     145             : 
     146             :     // AA - Used for DAG load/store alias analysis.
     147             :     AliasAnalysis *AA;
     148             : 
     149             :     /// When an instruction is simplified, add all users of the instruction to
     150             :     /// the work lists because they might get more simplified now.
     151             :     void AddUsersToWorklist(SDNode *N) {
     152     9505224 :       for (SDNode *Node : N->uses())
     153     6300972 :         AddToWorklist(Node);
     154             :     }
     155             : 
     156             :     /// Call the node-specific routine that folds each particular type of node.
     157             :     SDValue visit(SDNode *N);
     158             : 
     159             :   public:
     160     2800676 :     DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
     161     5601352 :         : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
     162     5601352 :           OptLevel(OL), AA(AA) {
     163     2800676 :       ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
     164             : 
     165     2800676 :       MaximumLegalStoreInBits = 0;
     166   319276812 :       for (MVT VT : MVT::all_valuetypes())
     167   316476136 :         if (EVT(VT).isSimple() && VT != MVT::Other &&
     168   313675460 :             TLI.isTypeLegal(EVT(VT)) &&
     169    42108384 :             VT.getSizeInBits() >= MaximumLegalStoreInBits)
     170    34761359 :           MaximumLegalStoreInBits = VT.getSizeInBits();
     171     2800676 :     }
     172             : 
     173             :     /// Add to the worklist making sure its instance is at the back (next to be
     174             :     /// processed.)
     175   253198332 :     void AddToWorklist(SDNode *N) {
     176             :       assert(N->getOpcode() != ISD::DELETED_NODE &&
     177             :              "Deleted Node added to Worklist");
     178             : 
     179             :       // Skip handle nodes as they can't usefully be combined and confuse the
     180             :       // zero-use deletion strategy.
     181   253198332 :       if (N->getOpcode() == ISD::HANDLENODE)
     182             :         return;
     183             : 
     184   253172357 :       if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
     185    94234119 :         Worklist.push_back(N);
     186             :     }
     187             : 
     188             :     /// Remove all instances of N from the worklist.
     189     7926126 :     void removeFromWorklist(SDNode *N) {
     190             :       CombinedNodes.erase(N);
     191             : 
     192     7926126 :       auto It = WorklistMap.find(N);
     193     7926126 :       if (It == WorklistMap.end())
     194     5447465 :         return; // Not in the worklist.
     195             : 
     196             :       // Null out the entry rather than erasing it to avoid a linear operation.
     197     4957322 :       Worklist[It->second] = nullptr;
     198             :       WorklistMap.erase(It);
     199             :     }
     200             : 
     201             :     void deleteAndRecombine(SDNode *N);
     202             :     bool recursivelyDeleteUnusedNodes(SDNode *N);
     203             : 
     204             :     /// Replaces all uses of the results of one DAG node with new values.
     205             :     SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
     206             :                       bool AddTo = true);
     207             : 
     208             :     /// Replaces all uses of the results of one DAG node with new values.
     209             :     SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
     210      471633 :       return CombineTo(N, &Res, 1, AddTo);
     211             :     }
     212             : 
     213             :     /// Replaces all uses of the results of one DAG node with new values.
     214             :     SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
     215             :                       bool AddTo = true) {
     216      237125 :       SDValue To[] = { Res0, Res1 };
     217        7140 :       return CombineTo(N, To, 2, AddTo);
     218             :     }
     219             : 
     220             :     void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
     221             : 
     222             :   private:
     223             :     unsigned MaximumLegalStoreInBits;
     224             : 
     225             :     /// Check the specified integer node value to see if it can be simplified or
     226             :     /// if things it uses can be simplified by bit propagation.
     227             :     /// If so, return true.
     228     4864187 :     bool SimplifyDemandedBits(SDValue Op) {
     229     4864187 :       unsigned BitWidth = Op.getScalarValueSizeInBits();
     230     4864187 :       APInt Demanded = APInt::getAllOnesValue(BitWidth);
     231     4864187 :       return SimplifyDemandedBits(Op, Demanded);
     232             :     }
     233             : 
     234             :     /// Check the specified vector node value to see if it can be simplified or
     235             :     /// if things it uses can be simplified as it only uses some of the
     236             :     /// elements. If so, return true.
     237      134887 :     bool SimplifyDemandedVectorElts(SDValue Op) {
     238      269774 :       unsigned NumElts = Op.getValueType().getVectorNumElements();
     239      134887 :       APInt Demanded = APInt::getAllOnesValue(NumElts);
     240      134887 :       return SimplifyDemandedVectorElts(Op, Demanded);
     241             :     }
     242             : 
     243             :     bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
     244             :     bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded,
     245             :                                     bool AssumeSingleUse = false);
     246             : 
     247             :     bool CombineToPreIndexedLoadStore(SDNode *N);
     248             :     bool CombineToPostIndexedLoadStore(SDNode *N);
     249             :     SDValue SplitIndexingFromLoad(LoadSDNode *LD);
     250             :     bool SliceUpLoad(SDNode *N);
     251             : 
     252             :     /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
     253             :     ///   load.
     254             :     ///
     255             :     /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
     256             :     /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
     257             :     /// \param EltNo index of the vector element to load.
     258             :     /// \param OriginalLoad load that EVE came from to be replaced.
     259             :     /// \returns EVE on success SDValue() on failure.
     260             :     SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
     261             :         SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
     262             :     void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
     263             :     SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
     264             :     SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
     265             :     SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
     266             :     SDValue PromoteIntBinOp(SDValue Op);
     267             :     SDValue PromoteIntShiftOp(SDValue Op);
     268             :     SDValue PromoteExtend(SDValue Op);
     269             :     bool PromoteLoad(SDValue Op);
     270             : 
     271             :     /// Call the node-specific routine that knows how to fold each
     272             :     /// particular type of node. If that doesn't do anything, try the
     273             :     /// target-specific DAG combines.
     274             :     SDValue combine(SDNode *N);
     275             : 
     276             :     // Visitation implementation - Implement dag node combining for different
     277             :     // node types.  The semantics are as follows:
     278             :     // Return Value:
     279             :     //   SDValue.getNode() == 0 - No change was made
     280             :     //   SDValue.getNode() == N - N was replaced, is dead and has been handled.
     281             :     //   otherwise              - N should be replaced by the returned Operand.
     282             :     //
     283             :     SDValue visitTokenFactor(SDNode *N);
     284             :     SDValue visitMERGE_VALUES(SDNode *N);
     285             :     SDValue visitADD(SDNode *N);
     286             :     SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
     287             :     SDValue visitSUB(SDNode *N);
     288             :     SDValue visitADDC(SDNode *N);
     289             :     SDValue visitUADDO(SDNode *N);
     290             :     SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
     291             :     SDValue visitSUBC(SDNode *N);
     292             :     SDValue visitUSUBO(SDNode *N);
     293             :     SDValue visitADDE(SDNode *N);
     294             :     SDValue visitADDCARRY(SDNode *N);
     295             :     SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
     296             :     SDValue visitSUBE(SDNode *N);
     297             :     SDValue visitSUBCARRY(SDNode *N);
     298             :     SDValue visitMUL(SDNode *N);
     299             :     SDValue useDivRem(SDNode *N);
     300             :     SDValue visitSDIV(SDNode *N);
     301             :     SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
     302             :     SDValue visitUDIV(SDNode *N);
     303             :     SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
     304             :     SDValue visitREM(SDNode *N);
     305             :     SDValue visitMULHU(SDNode *N);
     306             :     SDValue visitMULHS(SDNode *N);
     307             :     SDValue visitSMUL_LOHI(SDNode *N);
     308             :     SDValue visitUMUL_LOHI(SDNode *N);
     309             :     SDValue visitSMULO(SDNode *N);
     310             :     SDValue visitUMULO(SDNode *N);
     311             :     SDValue visitIMINMAX(SDNode *N);
     312             :     SDValue visitAND(SDNode *N);
     313             :     SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
     314             :     SDValue visitOR(SDNode *N);
     315             :     SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
     316             :     SDValue visitXOR(SDNode *N);
     317             :     SDValue SimplifyVBinOp(SDNode *N);
     318             :     SDValue visitSHL(SDNode *N);
     319             :     SDValue visitSRA(SDNode *N);
     320             :     SDValue visitSRL(SDNode *N);
     321             :     SDValue visitRotate(SDNode *N);
     322             :     SDValue visitABS(SDNode *N);
     323             :     SDValue visitBSWAP(SDNode *N);
     324             :     SDValue visitBITREVERSE(SDNode *N);
     325             :     SDValue visitCTLZ(SDNode *N);
     326             :     SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
     327             :     SDValue visitCTTZ(SDNode *N);
     328             :     SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
     329             :     SDValue visitCTPOP(SDNode *N);
     330             :     SDValue visitSELECT(SDNode *N);
     331             :     SDValue visitVSELECT(SDNode *N);
     332             :     SDValue visitSELECT_CC(SDNode *N);
     333             :     SDValue visitSETCC(SDNode *N);
     334             :     SDValue visitSETCCCARRY(SDNode *N);
     335             :     SDValue visitSIGN_EXTEND(SDNode *N);
     336             :     SDValue visitZERO_EXTEND(SDNode *N);
     337             :     SDValue visitANY_EXTEND(SDNode *N);
     338             :     SDValue visitAssertExt(SDNode *N);
     339             :     SDValue visitSIGN_EXTEND_INREG(SDNode *N);
     340             :     SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
     341             :     SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
     342             :     SDValue visitTRUNCATE(SDNode *N);
     343             :     SDValue visitBITCAST(SDNode *N);
     344             :     SDValue visitBUILD_PAIR(SDNode *N);
     345             :     SDValue visitFADD(SDNode *N);
     346             :     SDValue visitFSUB(SDNode *N);
     347             :     SDValue visitFMUL(SDNode *N);
     348             :     SDValue visitFMA(SDNode *N);
     349             :     SDValue visitFDIV(SDNode *N);
     350             :     SDValue visitFREM(SDNode *N);
     351             :     SDValue visitFSQRT(SDNode *N);
     352             :     SDValue visitFCOPYSIGN(SDNode *N);
     353             :     SDValue visitFPOW(SDNode *N);
     354             :     SDValue visitSINT_TO_FP(SDNode *N);
     355             :     SDValue visitUINT_TO_FP(SDNode *N);
     356             :     SDValue visitFP_TO_SINT(SDNode *N);
     357             :     SDValue visitFP_TO_UINT(SDNode *N);
     358             :     SDValue visitFP_ROUND(SDNode *N);
     359             :     SDValue visitFP_ROUND_INREG(SDNode *N);
     360             :     SDValue visitFP_EXTEND(SDNode *N);
     361             :     SDValue visitFNEG(SDNode *N);
     362             :     SDValue visitFABS(SDNode *N);
     363             :     SDValue visitFCEIL(SDNode *N);
     364             :     SDValue visitFTRUNC(SDNode *N);
     365             :     SDValue visitFFLOOR(SDNode *N);
     366             :     SDValue visitFMINNUM(SDNode *N);
     367             :     SDValue visitFMAXNUM(SDNode *N);
     368             :     SDValue visitBRCOND(SDNode *N);
     369             :     SDValue visitBR_CC(SDNode *N);
     370             :     SDValue visitLOAD(SDNode *N);
     371             : 
     372             :     SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
     373             :     SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
     374             : 
     375             :     SDValue visitSTORE(SDNode *N);
     376             :     SDValue visitINSERT_VECTOR_ELT(SDNode *N);
     377             :     SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
     378             :     SDValue visitBUILD_VECTOR(SDNode *N);
     379             :     SDValue visitCONCAT_VECTORS(SDNode *N);
     380             :     SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
     381             :     SDValue visitVECTOR_SHUFFLE(SDNode *N);
     382             :     SDValue visitSCALAR_TO_VECTOR(SDNode *N);
     383             :     SDValue visitINSERT_SUBVECTOR(SDNode *N);
     384             :     SDValue visitMLOAD(SDNode *N);
     385             :     SDValue visitMSTORE(SDNode *N);
     386             :     SDValue visitMGATHER(SDNode *N);
     387             :     SDValue visitMSCATTER(SDNode *N);
     388             :     SDValue visitFP_TO_FP16(SDNode *N);
     389             :     SDValue visitFP16_TO_FP(SDNode *N);
     390             : 
     391             :     SDValue visitFADDForFMACombine(SDNode *N);
     392             :     SDValue visitFSUBForFMACombine(SDNode *N);
     393             :     SDValue visitFMULForFMADistributiveCombine(SDNode *N);
     394             : 
     395             :     SDValue XformToShuffleWithZero(SDNode *N);
     396             :     SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
     397             :                            SDValue N1, SDNodeFlags Flags);
     398             : 
     399             :     SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
     400             : 
     401             :     SDValue foldSelectOfConstants(SDNode *N);
     402             :     SDValue foldVSelectOfConstants(SDNode *N);
     403             :     SDValue foldBinOpIntoSelect(SDNode *BO);
     404             :     bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
     405             :     SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
     406             :     SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
     407             :     SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
     408             :                              SDValue N2, SDValue N3, ISD::CondCode CC,
     409             :                              bool NotExtCompare = false);
     410             :     SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
     411             :                                    SDValue N2, SDValue N3, ISD::CondCode CC);
     412             :     SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
     413             :                               const SDLoc &DL);
     414             :     SDValue unfoldMaskedMerge(SDNode *N);
     415             :     SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
     416             :     SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
     417             :                           const SDLoc &DL, bool foldBooleans);
     418             :     SDValue rebuildSetCC(SDValue N);
     419             : 
     420             :     bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
     421             :                            SDValue &CC) const;
     422             :     bool isOneUseSetCC(SDValue N) const;
     423             : 
     424             :     SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
     425             :                                          unsigned HiOp);
     426             :     SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
     427             :     SDValue CombineExtLoad(SDNode *N);
     428             :     SDValue CombineZExtLogicopShiftLoad(SDNode *N);
     429             :     SDValue combineRepeatedFPDivisors(SDNode *N);
     430             :     SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
     431             :     SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
     432             :     SDValue BuildSDIV(SDNode *N);
     433             :     SDValue BuildSDIVPow2(SDNode *N);
     434             :     SDValue BuildUDIV(SDNode *N);
     435             :     SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
     436             :     SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
     437             :     SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
     438             :     SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
     439             :     SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
     440             :     SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
     441             :                                 SDNodeFlags Flags, bool Reciprocal);
     442             :     SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
     443             :                                 SDNodeFlags Flags, bool Reciprocal);
     444             :     SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
     445             :                                bool DemandHighBits = true);
     446             :     SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
     447             :     SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
     448             :                               SDValue InnerPos, SDValue InnerNeg,
     449             :                               unsigned PosOpcode, unsigned NegOpcode,
     450             :                               const SDLoc &DL);
     451             :     SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
     452             :     SDValue MatchLoadCombine(SDNode *N);
     453             :     SDValue ReduceLoadWidth(SDNode *N);
     454             :     SDValue ReduceLoadOpStoreWidth(SDNode *N);
     455             :     SDValue splitMergedValStore(StoreSDNode *ST);
     456             :     SDValue TransformFPLoadStorePair(SDNode *N);
     457             :     SDValue convertBuildVecZextToZext(SDNode *N);
     458             :     SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
     459             :     SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
     460             :     SDValue reduceBuildVecToShuffle(SDNode *N);
     461             :     SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
     462             :                                   ArrayRef<int> VectorMask, SDValue VecIn1,
     463             :                                   SDValue VecIn2, unsigned LeftIdx);
     464             :     SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
     465             : 
     466             :     /// Walk up chain skipping non-aliasing memory nodes,
     467             :     /// looking for aliasing nodes and adding them to the Aliases vector.
     468             :     void GatherAllAliases(SDNode *N, SDValue OriginalChain,
     469             :                           SmallVectorImpl<SDValue> &Aliases);
     470             : 
     471             :     /// Return true if there is any possibility that the two addresses overlap.
     472             :     bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
     473             : 
     474             :     /// Walk up chain skipping non-aliasing memory nodes, looking for a better
     475             :     /// chain (aliasing node.)
     476             :     SDValue FindBetterChain(SDNode *N, SDValue Chain);
     477             : 
     478             :     /// Try to replace a store and any possibly adjacent stores on
     479             :     /// consecutive chains with better chains. Return true only if St is
     480             :     /// replaced.
     481             :     ///
     482             :     /// Notice that other chains may still be replaced even if the function
     483             :     /// returns false.
     484             :     bool findBetterNeighborChains(StoreSDNode *St);
     485             : 
     486             :     /// Holds a pointer to an LSBaseSDNode as well as information on where it
     487             :     /// is located in a sequence of memory operations connected by a chain.
     488             :     struct MemOpLink {
     489             :       // Ptr to the mem node.
     490             :       LSBaseSDNode *MemNode;
     491             : 
     492             :       // Offset from the base ptr.
     493             :       int64_t OffsetFromBase;
     494             : 
     495             :       MemOpLink(LSBaseSDNode *N, int64_t Offset)
     496     1010019 :           : MemNode(N), OffsetFromBase(Offset) {}
     497             :     };
     498             : 
     499             :     /// This is a helper function for visitMUL to check the profitability
     500             :     /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
     501             :     /// MulNode is the original multiply, AddNode is (add x, c1),
     502             :     /// and ConstNode is c2.
     503             :     bool isMulAddWithConstProfitable(SDNode *MulNode,
     504             :                                      SDValue &AddNode,
     505             :                                      SDValue &ConstNode);
     506             : 
     507             :     /// This is a helper function for visitAND and visitZERO_EXTEND.  Returns
     508             :     /// true if the (and (load x) c) pattern matches an extload.  ExtVT returns
     509             :     /// the type of the loaded value to be extended.
     510             :     bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
     511             :                           EVT LoadResultTy, EVT &ExtVT);
     512             : 
     513             :     /// Helper function to calculate whether the given Load/Store can have its
     514             :     /// width reduced to ExtVT.
     515             :     bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
     516             :                            EVT &MemVT, unsigned ShAmt = 0);
     517             : 
     518             :     /// Used by BackwardsPropagateMask to find suitable loads.
     519             :     bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
     520             :                            SmallPtrSetImpl<SDNode*> &NodesWithConsts,
     521             :                            ConstantSDNode *Mask, SDNode *&NodeToMask);
     522             :     /// Attempt to propagate a given AND node back to load leaves so that they
     523             :     /// can be combined into narrow loads.
     524             :     bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
     525             : 
     526             :     /// Helper function for MergeConsecutiveStores which merges the
     527             :     /// component store chains.
     528             :     SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
     529             :                                 unsigned NumStores);
     530             : 
     531             :     /// This is a helper function for MergeConsecutiveStores. When the
     532             :     /// source elements of the consecutive stores are all constants or
     533             :     /// all extracted vector elements, try to merge them into one
     534             :     /// larger store introducing bitcasts if necessary.  \return True
     535             :     /// if a merged store was created.
     536             :     bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
     537             :                                          EVT MemVT, unsigned NumStores,
     538             :                                          bool IsConstantSrc, bool UseVector,
     539             :                                          bool UseTrunc);
     540             : 
     541             :     /// This is a helper function for MergeConsecutiveStores. Stores
     542             :     /// that potentially may be merged with St are placed in
     543             :     /// StoreNodes. RootNode is a chain predecessor to all store
     544             :     /// candidates.
     545             :     void getStoreMergeCandidates(StoreSDNode *St,
     546             :                                  SmallVectorImpl<MemOpLink> &StoreNodes,
     547             :                                  SDNode *&Root);
     548             : 
     549             :     /// Helper function for MergeConsecutiveStores. Checks if
     550             :     /// candidate stores have indirect dependency through their
     551             :     /// operands. RootNode is the predecessor to all stores calculated
     552             :     /// by getStoreMergeCandidates and is used to prune the dependency check.
     553             :     /// \return True if safe to merge.
     554             :     bool checkMergeStoreCandidatesForDependencies(
     555             :         SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
     556             :         SDNode *RootNode);
     557             : 
     558             :     /// Merge consecutive store operations into a wide store.
     559             :     /// This optimization uses wide integers or vectors when possible.
     560             :     /// \return number of stores that were merged into a merged store (the
     561             :     /// affected nodes are stored as a prefix in \p StoreNodes).
     562             :     bool MergeConsecutiveStores(StoreSDNode *St);
     563             : 
     564             :     /// Try to transform a truncation where C is a constant:
     565             :     ///     (trunc (and X, C)) -> (and (trunc X), (trunc C))
     566             :     ///
     567             :     /// \p N needs to be a truncation and its first operand an AND. Other
     568             :     /// requirements are checked by the function (e.g. that trunc is
     569             :     /// single-use) and if missed an empty SDValue is returned.
     570             :     SDValue distributeTruncateThroughAnd(SDNode *N);
     571             : 
     572             :     /// Helper function to determine whether the target supports operation
     573             :     /// given by \p Opcode for type \p VT, that is, whether the operation
     574             :     /// is legal or custom before legalizing operations, and whether is
     575             :     /// legal (but not custom) after legalization.
     576           0 :     bool hasOperation(unsigned Opcode, EVT VT) {
     577           0 :       if (LegalOperations)
     578           0 :         return TLI.isOperationLegal(Opcode, VT);
     579           0 :       return TLI.isOperationLegalOrCustom(Opcode, VT);
     580             :     }
     581             : 
     582             :   public:
     583             :     /// Runs the dag combiner on all nodes in the work list
     584             :     void Run(CombineLevel AtLevel);
     585             : 
     586           0 :     SelectionDAG &getDAG() const { return DAG; }
     587             : 
     588             :     /// Returns a type large enough to hold any valid shift amount - before type
     589             :     /// legalization these can be huge.
     590        8512 :     EVT getShiftAmountTy(EVT LHSTy) {
     591             :       assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
     592        8512 :       return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
     593             :     }
     594             : 
     595             :     /// This method returns true if we are running before type legalization or
     596             :     /// if the specified VT is legal.
     597           0 :     bool isTypeLegal(const EVT &VT) {
     598           0 :       if (!LegalTypes) return true;
     599        7190 :       return TLI.isTypeLegal(VT);
     600             :     }
     601             : 
     602             :     /// Convenience wrapper around TargetLowering::getSetCCResultType
     603           0 :     EVT getSetCCResultType(EVT VT) const {
     604           0 :       return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
     605             :     }
     606             : 
     607             :     void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
     608             :                          SDValue OrigLoad, SDValue ExtLoad,
     609             :                          ISD::NodeType ExtType);
     610             :   };
     611             : 
     612             : /// This class is a DAGUpdateListener that removes any deleted
     613             : /// nodes from the worklist.
     614        4664 : class WorklistRemover : public SelectionDAG::DAGUpdateListener {
     615             :   DAGCombiner &DC;
     616             : 
     617             : public:
     618             :   explicit WorklistRemover(DAGCombiner &dc)
     619    90249604 :     : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
     620             : 
     621       24710 :   void NodeDeleted(SDNode *N, SDNode *E) override {
     622       24710 :     DC.removeFromWorklist(N);
     623       24710 :   }
     624             : };
     625             : 
     626             : } // end anonymous namespace
     627             : 
     628             : //===----------------------------------------------------------------------===//
     629             : //  TargetLowering::DAGCombinerInfo implementation
     630             : //===----------------------------------------------------------------------===//
     631             : 
     632        9974 : void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
     633        9974 :   ((DAGCombiner*)DC)->AddToWorklist(N);
     634        9974 : }
     635             : 
     636         864 : SDValue TargetLowering::DAGCombinerInfo::
     637             : CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
     638         864 :   return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
     639             : }
     640             : 
     641        1078 : SDValue TargetLowering::DAGCombinerInfo::
     642             : CombineTo(SDNode *N, SDValue Res, bool AddTo) {
     643        1078 :   return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
     644             : }
     645             : 
     646        5683 : SDValue TargetLowering::DAGCombinerInfo::
     647             : CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
     648        5683 :   return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
     649             : }
     650             : 
     651        2267 : void TargetLowering::DAGCombinerInfo::
     652             : CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
     653        2267 :   return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
     654             : }
     655             : 
     656             : //===----------------------------------------------------------------------===//
     657             : // Helper Functions
     658             : //===----------------------------------------------------------------------===//
     659             : 
     660     1466976 : void DAGCombiner::deleteAndRecombine(SDNode *N) {
     661     1466976 :   removeFromWorklist(N);
     662             : 
     663             :   // If the operands of this node are only used by the node, they will now be
     664             :   // dead. Make sure to re-visit them and recursively delete dead nodes.
     665     5581139 :   for (const SDValue &Op : N->ops())
     666             :     // For an operand generating multiple values, one of the values may
     667             :     // become dead allowing further simplification (e.g. split index
     668             :     // arithmetic from an indexed load).
     669     8009641 :     if (Op->hasOneUse() || Op->getNumValues() > 1)
     670     1110291 :       AddToWorklist(Op.getNode());
     671             : 
     672     1466976 :   DAG.DeleteNode(N);
     673     1466976 : }
     674             : 
     675             : /// Return 1 if we can compute the negated form of the specified expression for
     676             : /// the same cost as the expression itself, or 2 if we can compute the negated
     677             : /// form more cheaply than the expression itself.
     678      212497 : static char isNegatibleForFree(SDValue Op, bool LegalOperations,
     679             :                                const TargetLowering &TLI,
     680             :                                const TargetOptions *Options,
     681             :                                unsigned Depth = 0) {
     682             :   // fneg is removable even if it has multiple uses.
     683      212497 :   if (Op.getOpcode() == ISD::FNEG) return 2;
     684             : 
     685             :   // Don't allow anything with multiple uses unless we know it is free.
     686      211548 :   EVT VT = Op.getValueType();
     687      211548 :   const SDNodeFlags Flags = Op->getFlags();
     688      211548 :   if (!Op.hasOneUse())
     689       59177 :     if (!(Op.getOpcode() == ISD::FP_EXTEND &&
     690         957 :           TLI.isFPExtFree(VT, Op.getOperand(0).getValueType())))
     691       58852 :       return 0;
     692             : 
     693             :   // Don't recurse exponentially.
     694      152696 :   if (Depth > 6) return 0;
     695             : 
     696      149651 :   switch (Op.getOpcode()) {
     697             :   default: return false;
     698        4560 :   case ISD::ConstantFP: {
     699        4560 :     if (!LegalOperations)
     700             :       return 1;
     701             : 
     702             :     // Don't invert constant FP values after legalization unless the target says
     703             :     // the negated constant is legal.
     704         204 :     return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
     705        1772 :       TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
     706             :   }
     707        8905 :   case ISD::FADD:
     708        8905 :     if (!Options->UnsafeFPMath && !Flags.hasNoSignedZeros())
     709             :       return 0;
     710             : 
     711             :     // After operation legalization, it might not be legal to create new FSUBs.
     712        1545 :     if (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
     713           0 :       return 0;
     714             : 
     715             :     // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     716        3090 :     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
     717             :                                     Options, Depth + 1))
     718        1523 :       return V;
     719             :     // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
     720        1523 :     return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
     721        1523 :                               Depth + 1);
     722        2571 :   case ISD::FSUB:
     723             :     // We can't turn -(A-B) into B-A when we honor signed zeros.
     724        2571 :     if (!Options->NoSignedZerosFPMath &&
     725             :         !Flags.hasNoSignedZeros())
     726        1706 :       return 0;
     727             : 
     728             :     // fold (fneg (fsub A, B)) -> (fsub B, A)
     729             :     return 1;
     730             : 
     731       35835 :   case ISD::FMUL:
     732             :   case ISD::FDIV:
     733             :     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
     734       71670 :     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
     735             :                                     Options, Depth + 1))
     736       34768 :       return V;
     737             : 
     738       34768 :     return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
     739       34768 :                               Depth + 1);
     740             : 
     741        2957 :   case ISD::FP_EXTEND:
     742             :   case ISD::FP_ROUND:
     743             :   case ISD::FSIN:
     744        5914 :     return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
     745        2957 :                               Depth + 1);
     746             :   }
     747             : }
     748             : 
     749             : /// If isNegatibleForFree returns true, return the newly negated expression.
     750        1030 : static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
     751             :                                     bool LegalOperations, unsigned Depth = 0) {
     752        1030 :   const TargetOptions &Options = DAG.getTarget().Options;
     753             :   // fneg is removable even if it has multiple uses.
     754        1030 :   if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
     755             : 
     756             :   assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
     757             : 
     758         537 :   const SDNodeFlags Flags = Op.getNode()->getFlags();
     759             : 
     760         537 :   switch (Op.getOpcode()) {
     761           0 :   default: llvm_unreachable("Unknown code");
     762             :   case ISD::ConstantFP: {
     763         211 :     APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
     764         211 :     V.changeSign();
     765         422 :     return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
     766             :   }
     767          19 :   case ISD::FADD:
     768             :     assert(Options.UnsafeFPMath || Flags.hasNoSignedZeros());
     769             : 
     770             :     // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     771          38 :     if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
     772             :                            DAG.getTargetLoweringInfo(), &Options, Depth+1))
     773          19 :       return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     774             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     775             :                                               LegalOperations, Depth+1),
     776          38 :                          Op.getOperand(1), Flags);
     777             :     // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
     778           0 :     return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     779             :                        GetNegatedExpression(Op.getOperand(1), DAG,
     780             :                                             LegalOperations, Depth+1),
     781           0 :                        Op.getOperand(0), Flags);
     782          21 :   case ISD::FSUB:
     783             :     // fold (fneg (fsub 0, B)) -> B
     784             :     if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
     785          10 :       if (N0CFP->isZero())
     786           5 :         return Op.getOperand(1);
     787             : 
     788             :     // fold (fneg (fsub A, B)) -> (fsub B, A)
     789          16 :     return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     790          16 :                        Op.getOperand(1), Op.getOperand(0), Flags);
     791             : 
     792         242 :   case ISD::FMUL:
     793             :   case ISD::FDIV:
     794             :     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
     795         484 :     if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
     796             :                            DAG.getTargetLoweringInfo(), &Options, Depth+1))
     797         115 :       return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     798             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     799             :                                               LegalOperations, Depth+1),
     800         230 :                          Op.getOperand(1), Flags);
     801             : 
     802             :     // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
     803         127 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     804             :                        Op.getOperand(0),
     805             :                        GetNegatedExpression(Op.getOperand(1), DAG,
     806         254 :                                             LegalOperations, Depth+1), Flags);
     807             : 
     808             :   case ISD::FP_EXTEND:
     809             :   case ISD::FSIN:
     810          20 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     811             :                        GetNegatedExpression(Op.getOperand(0), DAG,
     812          80 :                                             LegalOperations, Depth+1));
     813             :   case ISD::FP_ROUND:
     814          24 :       return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
     815             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     816             :                                               LegalOperations, Depth+1),
     817          96 :                          Op.getOperand(1));
     818             :   }
     819             : }
     820             : 
     821             : // APInts must be the same size for most operations, this helper
     822             : // function zero extends the shorter of the pair so that they match.
     823             : // We provide an Offset so that we can create bitwidths that won't overflow.
     824        9662 : static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
     825        9662 :   unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
     826        9662 :   LHS = LHS.zextOrSelf(Bits);
     827        9662 :   RHS = RHS.zextOrSelf(Bits);
     828        9662 : }
     829             : 
     830             : // Return true if this node is a setcc, or is a select_cc
     831             : // that selects between the target values used for true and false, making it
     832             : // equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
     833             : // the appropriate nodes based on the type of node we are checking. This
     834             : // simplifies life a bit for the callers.
     835           0 : bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
     836             :                                     SDValue &CC) const {
     837           0 :   if (N.getOpcode() == ISD::SETCC) {
     838           0 :     LHS = N.getOperand(0);
     839           0 :     RHS = N.getOperand(1);
     840           0 :     CC  = N.getOperand(2);
     841           0 :     return true;
     842             :   }
     843             : 
     844           0 :   if (N.getOpcode() != ISD::SELECT_CC ||
     845           0 :       !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
     846           0 :       !TLI.isConstFalseVal(N.getOperand(3).getNode()))
     847           0 :     return false;
     848             : 
     849           0 :   if (TLI.getBooleanContents(N.getValueType()) ==
     850             :       TargetLowering::UndefinedBooleanContent)
     851           0 :     return false;
     852             : 
     853           0 :   LHS = N.getOperand(0);
     854           0 :   RHS = N.getOperand(1);
     855           0 :   CC  = N.getOperand(4);
     856           0 :   return true;
     857             : }
     858             : 
     859             : /// Return true if this is a SetCC-equivalent operation with only one use.
     860             : /// If this is true, it allows the users to invert the operation for free when
     861             : /// it is profitable to do so.
     862           0 : bool DAGCombiner::isOneUseSetCC(SDValue N) const {
     863           0 :   SDValue N0, N1, N2;
     864           0 :   if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
     865           0 :     return true;
     866             :   return false;
     867             : }
     868             : 
     869             : // Returns the SDNode if it is a constant float BuildVector
     870             : // or constant float.
     871           0 : static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
     872             :   if (isa<ConstantFPSDNode>(N))
     873             :     return N.getNode();
     874      142387 :   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
     875           0 :     return N.getNode();
     876             :   return nullptr;
     877             : }
     878             : 
     879             : // Determines if it is a constant integer or a build vector of constant
     880             : // integers (and undefs).
     881             : // Do not permit build vector implicit truncation.
     882     3449938 : static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
     883             :   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
     884     2928236 :     return !(Const->isOpaque() && NoOpaques);
     885      521702 :   if (N.getOpcode() != ISD::BUILD_VECTOR)
     886             :     return false;
     887      105771 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     888      365274 :   for (const SDValue &Op : N->op_values()) {
     889      520334 :     if (Op.isUndef())
     890             :       continue;
     891             :     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
     892      258701 :     if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
     893      258597 :         (Const->isOpaque() && NoOpaques))
     894             :       return false;
     895             :   }
     896             :   return true;
     897             : }
     898             : 
     899             : // Determines if it is a constant null integer or a splatted vector of a
     900             : // constant null integer (with no undefs).
     901             : // Build vector implicit truncation is not an issue for null values.
     902      836232 : static bool isNullConstantOrNullSplatConstant(SDValue N) {
     903             :   // TODO: may want to use peekThroughBitcast() here.
     904      836232 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     905       27622 :     return Splat->isNullValue();
     906             :   return false;
     907             : }
     908             : 
     909             : // Determines if it is a constant integer of one or a splatted vector of a
     910             : // constant integer of one (with no undefs).
     911             : // Do not permit build vector implicit truncation.
     912        1021 : static bool isOneConstantOrOneSplatConstant(SDValue N) {
     913             :   // TODO: may want to use peekThroughBitcast() here.
     914        1021 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     915        1021 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     916        2010 :     return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
     917             :   return false;
     918             : }
     919             : 
     920             : // Determines if it is a constant integer of all ones or a splatted vector of a
     921             : // constant integer of all ones (with no undefs).
     922             : // Do not permit build vector implicit truncation.
     923      620734 : static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
     924      620734 :   N = peekThroughBitcasts(N);
     925      620734 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     926      620734 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     927      149042 :     return Splat->isAllOnesValue() &&
     928             :            Splat->getAPIntValue().getBitWidth() == BitWidth;
     929             :   return false;
     930             : }
     931             : 
     932             : // Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
     933             : // undef's.
     934       25282 : static bool isAnyConstantBuildVector(const SDNode *N) {
     935       47987 :   return ISD::isBuildVectorOfConstantSDNodes(N) ||
     936       22705 :          ISD::isBuildVectorOfConstantFPSDNodes(N);
     937             : }
     938             : 
     939     4045605 : SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
     940             :                                     SDValue N1, SDNodeFlags Flags) {
     941             :   // Don't reassociate reductions.
     942     4045605 :   if (Flags.hasVectorReduction())
     943         444 :     return SDValue();
     944             : 
     945     4045161 :   EVT VT = N0.getValueType();
     946     4045161 :   if (N0.getOpcode() == Opc && !N0->getFlags().hasVectorReduction()) {
     947     1044486 :     if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
     948      229211 :       if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
     949             :         // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
     950      226075 :         if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
     951      452108 :           return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
     952          21 :         return SDValue();
     953             :       }
     954        3136 :       if (N0.hasOneUse()) {
     955             :         // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
     956             :         // use
     957        3259 :         SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
     958        2000 :         if (!OpNode.getNode())
     959           0 :           return SDValue();
     960        2000 :         AddToWorklist(OpNode.getNode());
     961        4000 :         return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
     962             :       }
     963             :     }
     964             :   }
     965             : 
     966     3817086 :   if (N1.getOpcode() == Opc && !N1->getFlags().hasVectorReduction()) {
     967       14526 :     if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
     968        1643 :       if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
     969             :         // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
     970           0 :         if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
     971           0 :           return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
     972           0 :         return SDValue();
     973             :       }
     974        1643 :       if (N1.hasOneUse()) {
     975             :         // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
     976             :         // use
     977        1210 :         SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
     978        1025 :         if (!OpNode.getNode())
     979           0 :           return SDValue();
     980        1025 :         AddToWorklist(OpNode.getNode());
     981        2050 :         return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
     982             :       }
     983             :     }
     984             :   }
     985             : 
     986     3816061 :   return SDValue();
     987             : }
     988             : 
     989      748116 : SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
     990             :                                bool AddTo) {
     991             :   assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
     992             :   ++NodesCombined;
     993             :   LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
     994             :              To[0].getNode()->dump(&DAG);
     995             :              dbgs() << " and " << NumTo - 1 << " other values\n");
     996             :   for (unsigned i = 0, e = NumTo; i != e; ++i)
     997             :     assert((!To[i].getNode() ||
     998             :             N->getValueType(i) == To[i].getValueType()) &&
     999             :            "Cannot combine value to value of different type!");
    1000             : 
    1001             :   WorklistRemover DeadNodes(*this);
    1002      748116 :   DAG.ReplaceAllUsesWith(N, To);
    1003      748116 :   if (AddTo) {
    1004             :     // Push the new nodes and any users onto the worklist
    1005      871135 :     for (unsigned i = 0, e = NumTo; i != e; ++i) {
    1006      554803 :       if (To[i].getNode()) {
    1007      554803 :         AddToWorklist(To[i].getNode());
    1008      554803 :         AddUsersToWorklist(To[i].getNode());
    1009             :       }
    1010             :     }
    1011             :   }
    1012             : 
    1013             :   // Finally, if the node is now dead, remove it from the graph.  The node
    1014             :   // may not be dead if the replacement process recursively simplified to
    1015             :   // something else needing this node.
    1016      748116 :   if (N->use_empty())
    1017      748074 :     deleteAndRecombine(N);
    1018     1496232 :   return SDValue(N, 0);
    1019             : }
    1020             : 
    1021      156322 : void DAGCombiner::
    1022             : CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
    1023             :   // Replace all uses.  If any nodes become isomorphic to other nodes and
    1024             :   // are deleted, make sure to remove them from our worklist.
    1025             :   WorklistRemover DeadNodes(*this);
    1026      156322 :   DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
    1027             : 
    1028             :   // Push the new node and any (possibly new) users onto the worklist.
    1029      156322 :   AddToWorklist(TLO.New.getNode());
    1030      156322 :   AddUsersToWorklist(TLO.New.getNode());
    1031             : 
    1032             :   // Finally, if the node is now dead, remove it from the graph.  The node
    1033             :   // may not be dead if the replacement process recursively simplified to
    1034             :   // something else needing this node.
    1035      156322 :   if (TLO.Old.getNode()->use_empty())
    1036      155550 :     deleteAndRecombine(TLO.Old.getNode());
    1037      156322 : }
    1038             : 
    1039             : /// Check the specified integer node value to see if it can be simplified or if
    1040             : /// things it uses can be simplified by bit propagation. If so, return true.
    1041     4915337 : bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
    1042     4915337 :   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
    1043     4915337 :   KnownBits Known;
    1044     4915337 :   if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
    1045             :     return false;
    1046             : 
    1047             :   // Revisit the node.
    1048      151574 :   AddToWorklist(Op.getNode());
    1049             : 
    1050             :   // Replace the old value with the new one.
    1051             :   ++NodesCombined;
    1052             :   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
    1053             :              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
    1054             :              dbgs() << '\n');
    1055             : 
    1056      151574 :   CommitTargetLoweringOpt(TLO);
    1057      151574 :   return true;
    1058             : }
    1059             : 
    1060             : /// Check the specified vector node value to see if it can be simplified or
    1061             : /// if things it uses can be simplified as it only uses some of the elements.
    1062             : /// If so, return true.
    1063      400838 : bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded,
    1064             :                                              bool AssumeSingleUse) {
    1065      400838 :   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
    1066             :   APInt KnownUndef, KnownZero;
    1067      400838 :   if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO,
    1068             :                                       0, AssumeSingleUse))
    1069             :     return false;
    1070             : 
    1071             :   // Revisit the node.
    1072        2481 :   AddToWorklist(Op.getNode());
    1073             : 
    1074             :   // Replace the old value with the new one.
    1075             :   ++NodesCombined;
    1076             :   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
    1077             :              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
    1078             :              dbgs() << '\n');
    1079             : 
    1080        2481 :   CommitTargetLoweringOpt(TLO);
    1081        2481 :   return true;
    1082             : }
    1083             : 
    1084         313 : void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
    1085             :   SDLoc DL(Load);
    1086         313 :   EVT VT = Load->getValueType(0);
    1087         626 :   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
    1088             : 
    1089             :   LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
    1090             :              Trunc.getNode()->dump(&DAG); dbgs() << '\n');
    1091             :   WorklistRemover DeadNodes(*this);
    1092         626 :   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
    1093         626 :   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
    1094         313 :   deleteAndRecombine(Load);
    1095         313 :   AddToWorklist(Trunc.getNode());
    1096         313 : }
    1097             : 
    1098        8655 : SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
    1099        8655 :   Replace = false;
    1100             :   SDLoc DL(Op);
    1101             :   if (ISD::isUNINDEXEDLoad(Op.getNode())) {
    1102             :     LoadSDNode *LD = cast<LoadSDNode>(Op);
    1103         402 :     EVT MemVT = LD->getMemoryVT();
    1104             :     ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
    1105             :                                                       : LD->getExtensionType();
    1106         402 :     Replace = true;
    1107         402 :     return DAG.getExtLoad(ExtType, DL, PVT,
    1108             :                           LD->getChain(), LD->getBasePtr(),
    1109         804 :                           MemVT, LD->getMemOperand());
    1110             :   }
    1111             : 
    1112             :   unsigned Opc = Op.getOpcode();
    1113        8253 :   switch (Opc) {
    1114             :   default: break;
    1115           0 :   case ISD::AssertSext:
    1116           0 :     if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
    1117           0 :       return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
    1118           0 :     break;
    1119          34 :   case ISD::AssertZext:
    1120          34 :     if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
    1121          68 :       return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
    1122           0 :     break;
    1123        1696 :   case ISD::Constant: {
    1124             :     unsigned ExtOpc =
    1125        3392 :       Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    1126        3392 :     return DAG.getNode(ExtOpc, DL, PVT, Op);
    1127             :   }
    1128             :   }
    1129             : 
    1130        6523 :   if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
    1131           0 :     return SDValue();
    1132       13046 :   return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
    1133             : }
    1134             : 
    1135           0 : SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
    1136           0 :   if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
    1137           0 :     return SDValue();
    1138           0 :   EVT OldVT = Op.getValueType();
    1139             :   SDLoc DL(Op);
    1140           0 :   bool Replace = false;
    1141           0 :   SDValue NewOp = PromoteOperand(Op, PVT, Replace);
    1142           0 :   if (!NewOp.getNode())
    1143           0 :     return SDValue();
    1144           0 :   AddToWorklist(NewOp.getNode());
    1145             : 
    1146           0 :   if (Replace)
    1147           0 :     ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
    1148           0 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
    1149           0 :                      DAG.getValueType(OldVT));
    1150             : }
    1151             : 
    1152         524 : SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
    1153         524 :   EVT OldVT = Op.getValueType();
    1154             :   SDLoc DL(Op);
    1155         524 :   bool Replace = false;
    1156         524 :   SDValue NewOp = PromoteOperand(Op, PVT, Replace);
    1157         524 :   if (!NewOp.getNode())
    1158           0 :     return SDValue();
    1159         524 :   AddToWorklist(NewOp.getNode());
    1160             : 
    1161         524 :   if (Replace)
    1162          78 :     ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
    1163         524 :   return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
    1164             : }
    1165             : 
    1166             : /// Promote the specified integer binary operation if the target indicates it is
    1167             : /// beneficial. e.g. On x86, it's usually better to promote i16 operations to
    1168             : /// i32 since i16 instructions are longer.
    1169     4022167 : SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
    1170     4022167 :   if (!LegalOperations)
    1171     2281521 :     return SDValue();
    1172             : 
    1173     1740646 :   EVT VT = Op.getValueType();
    1174     1740646 :   if (VT.isVector() || !VT.isInteger())
    1175      188402 :     return SDValue();
    1176             : 
    1177             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1178             :   // promoting it.
    1179             :   unsigned Opc = Op.getOpcode();
    1180     1552244 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1181     1547813 :     return SDValue();
    1182             : 
    1183        4431 :   EVT PVT = VT;
    1184             :   // Consult target whether it is a good idea to promote this operation and
    1185             :   // what's the right type to promote it to.
    1186        4431 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1187             :     assert(PVT != VT && "Don't know what type to promote to!");
    1188             : 
    1189             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1190             : 
    1191        3506 :     bool Replace0 = false;
    1192        3506 :     SDValue N0 = Op.getOperand(0);
    1193        3506 :     SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
    1194             : 
    1195        3506 :     bool Replace1 = false;
    1196        3506 :     SDValue N1 = Op.getOperand(1);
    1197        3506 :     SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
    1198             :     SDLoc DL(Op);
    1199             : 
    1200             :     SDValue RV =
    1201        7012 :         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
    1202             : 
    1203             :     // We are always replacing N0/N1's use in N and only need
    1204             :     // additional replacements if there are additional uses.
    1205        3506 :     Replace0 &= !N0->hasOneUse();
    1206        3506 :     Replace1 &= (N0 != N1) && !N1->hasOneUse();
    1207             : 
    1208             :     // Combine Op here so it is preserved past replacements.
    1209        3506 :     CombineTo(Op.getNode(), RV);
    1210             : 
    1211             :     // If operands have a use ordering, make sure we deal with
    1212             :     // predecessor first.
    1213        3508 :     if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
    1214             :       std::swap(N0, N1);
    1215             :       std::swap(NN0, NN1);
    1216             :     }
    1217             : 
    1218        3506 :     if (Replace0) {
    1219          89 :       AddToWorklist(NN0.getNode());
    1220          89 :       ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
    1221             :     }
    1222        3506 :     if (Replace1) {
    1223          92 :       AddToWorklist(NN1.getNode());
    1224          92 :       ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
    1225             :     }
    1226        3506 :     return Op;
    1227             :   }
    1228         925 :   return SDValue();
    1229             : }
    1230             : 
    1231             : /// Promote the specified integer shift operation if the target indicates it is
    1232             : /// beneficial. e.g. On x86, it's usually better to promote i16 operations to
    1233             : /// i32 since i16 instructions are longer.
    1234      259714 : SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
    1235      259714 :   if (!LegalOperations)
    1236      113415 :     return SDValue();
    1237             : 
    1238      146299 :   EVT VT = Op.getValueType();
    1239      146299 :   if (VT.isVector() || !VT.isInteger())
    1240        2424 :     return SDValue();
    1241             : 
    1242             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1243             :   // promoting it.
    1244             :   unsigned Opc = Op.getOpcode();
    1245      143875 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1246      138686 :     return SDValue();
    1247             : 
    1248        5189 :   EVT PVT = VT;
    1249             :   // Consult target whether it is a good idea to promote this operation and
    1250             :   // what's the right type to promote it to.
    1251        5189 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1252             :     assert(PVT != VT && "Don't know what type to promote to!");
    1253             : 
    1254             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1255             : 
    1256        1609 :     bool Replace = false;
    1257        1609 :     SDValue N0 = Op.getOperand(0);
    1258        1609 :     SDValue N1 = Op.getOperand(1);
    1259        1609 :     if (Opc == ISD::SRA)
    1260           0 :       N0 = SExtPromoteOperand(N0, PVT);
    1261        1609 :     else if (Opc == ISD::SRL)
    1262         490 :       N0 = ZExtPromoteOperand(N0, PVT);
    1263             :     else
    1264        1119 :       N0 = PromoteOperand(N0, PVT, Replace);
    1265             : 
    1266        1609 :     if (!N0.getNode())
    1267           0 :       return SDValue();
    1268             : 
    1269             :     SDLoc DL(Op);
    1270             :     SDValue RV =
    1271        3218 :         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
    1272             : 
    1273        1609 :     AddToWorklist(N0.getNode());
    1274        1609 :     if (Replace)
    1275          54 :       ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
    1276             : 
    1277             :     // Deal with Op being deleted.
    1278        1609 :     if (Op && Op.getOpcode() != ISD::DELETED_NODE)
    1279        1609 :       return RV;
    1280             :   }
    1281        3580 :   return SDValue();
    1282             : }
    1283             : 
    1284      185782 : SDValue DAGCombiner::PromoteExtend(SDValue Op) {
    1285      185782 :   if (!LegalOperations)
    1286      115241 :     return SDValue();
    1287             : 
    1288       70541 :   EVT VT = Op.getValueType();
    1289       70541 :   if (VT.isVector() || !VT.isInteger())
    1290        1464 :     return SDValue();
    1291             : 
    1292             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1293             :   // promoting it.
    1294             :   unsigned Opc = Op.getOpcode();
    1295       69077 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1296       68820 :     return SDValue();
    1297             : 
    1298         257 :   EVT PVT = VT;
    1299             :   // Consult target whether it is a good idea to promote this operation and
    1300             :   // what's the right type to promote it to.
    1301         257 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1302             :     assert(PVT != VT && "Don't know what type to promote to!");
    1303             :     // fold (aext (aext x)) -> (aext x)
    1304             :     // fold (aext (zext x)) -> (zext x)
    1305             :     // fold (aext (sext x)) -> (sext x)
    1306             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1307         486 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
    1308             :   }
    1309          14 :   return SDValue();
    1310             : }
    1311             : 
    1312     6218337 : bool DAGCombiner::PromoteLoad(SDValue Op) {
    1313     6218337 :   if (!LegalOperations)
    1314             :     return false;
    1315             : 
    1316             :   if (!ISD::isUNINDEXEDLoad(Op.getNode()))
    1317             :     return false;
    1318             : 
    1319     2647329 :   EVT VT = Op.getValueType();
    1320     2647329 :   if (VT.isVector() || !VT.isInteger())
    1321             :     return false;
    1322             : 
    1323             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1324             :   // promoting it.
    1325             :   unsigned Opc = Op.getOpcode();
    1326     2365315 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1327             :     return false;
    1328             : 
    1329        4621 :   EVT PVT = VT;
    1330             :   // Consult target whether it is a good idea to promote this operation and
    1331             :   // what's the right type to promote it to.
    1332        4621 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1333             :     assert(PVT != VT && "Don't know what type to promote to!");
    1334             : 
    1335             :     SDLoc DL(Op);
    1336             :     SDNode *N = Op.getNode();
    1337             :     LoadSDNode *LD = cast<LoadSDNode>(N);
    1338           0 :     EVT MemVT = LD->getMemoryVT();
    1339             :     ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
    1340             :                                                       : LD->getExtensionType();
    1341           0 :     SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
    1342             :                                    LD->getChain(), LD->getBasePtr(),
    1343           0 :                                    MemVT, LD->getMemOperand());
    1344           0 :     SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
    1345             : 
    1346             :     LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
    1347             :                Result.getNode()->dump(&DAG); dbgs() << '\n');
    1348             :     WorklistRemover DeadNodes(*this);
    1349           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
    1350           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
    1351           0 :     deleteAndRecombine(N);
    1352           0 :     AddToWorklist(Result.getNode());
    1353             :     return true;
    1354             :   }
    1355             :   return false;
    1356             : }
    1357             : 
    1358             : /// Recursively delete a node which has no uses and any operands for
    1359             : /// which it is the only use.
    1360             : ///
    1361             : /// Note that this both deletes the nodes and removes them from the worklist.
    1362             : /// It also adds any nodes who have had a user deleted to the worklist as they
    1363             : /// may now have only one use and subject to other combines.
    1364    93663226 : bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
    1365    93663226 :   if (!N->use_empty())
    1366             :     return false;
    1367             : 
    1368             :   SmallSetVector<SDNode *, 16> Nodes;
    1369     3979008 :   Nodes.insert(N);
    1370             :   do {
    1371    12828311 :     N = Nodes.pop_back_val();
    1372    12828311 :     if (!N)
    1373             :       continue;
    1374             : 
    1375    12828311 :     if (N->use_empty()) {
    1376    16098426 :       for (const SDValue &ChildN : N->op_values())
    1377     9663986 :         Nodes.insert(ChildN.getNode());
    1378             : 
    1379     6434440 :       removeFromWorklist(N);
    1380     6434440 :       DAG.DeleteNode(N);
    1381             :     } else {
    1382     6393871 :       AddToWorklist(N);
    1383             :     }
    1384    12828311 :   } while (!Nodes.empty());
    1385             :   return true;
    1386             : }
    1387             : 
    1388             : //===----------------------------------------------------------------------===//
    1389             : //  Main DAG Combiner implementation
    1390             : //===----------------------------------------------------------------------===//
    1391             : 
    1392     2800676 : void DAGCombiner::Run(CombineLevel AtLevel) {
    1393             :   // set the instance variables, so that the various visit routines may use it.
    1394     2800676 :   Level = AtLevel;
    1395     2800676 :   LegalOperations = Level >= AfterLegalizeVectorOps;
    1396     2800676 :   LegalTypes = Level >= AfterLegalizeTypes;
    1397             : 
    1398             :   // Add all the dag nodes to the worklist.
    1399    86448936 :   for (SDNode &Node : DAG.allnodes())
    1400    83648260 :     AddToWorklist(&Node);
    1401             : 
    1402             :   // Create a dummy node (which is not added to allnodes), that adds a reference
    1403             :   // to the root node, preventing it from being deleted, and tracking any
    1404             :   // changes of the root.
    1405     8402028 :   HandleSDNode Dummy(DAG.getRoot());
    1406             : 
    1407             :   // While the worklist isn't empty, find a node and try to combine it.
    1408    94556132 :   while (!WorklistMap.empty()) {
    1409             :     SDNode *N;
    1410             :     // The Worklist holds the SDNodes in order, but it may contain null entries.
    1411             :     do {
    1412    94234117 :       N = Worklist.pop_back_val();
    1413    94234117 :     } while (!N);
    1414             : 
    1415    91755456 :     bool GoodWorklistEntry = WorklistMap.erase(N);
    1416             :     (void)GoodWorklistEntry;
    1417             :     assert(GoodWorklistEntry &&
    1418             :            "Found a worklist entry without a corresponding map entry!");
    1419             : 
    1420             :     // If N has no uses, it is dead.  Make sure to revisit all N's operands once
    1421             :     // N is deleted from the DAG, since they too may now be dead or may have a
    1422             :     // reduced number of uses, allowing other xforms.
    1423    91755456 :     if (recursivelyDeleteUnusedNodes(N))
    1424    89848357 :       continue;
    1425             : 
    1426             :     WorklistRemover DeadNodes(*this);
    1427             : 
    1428             :     // If this combine is running after legalizing the DAG, re-legalize any
    1429             :     // nodes pulled off the worklist.
    1430    89683459 :     if (Level == AfterLegalizeDAG) {
    1431             :       SmallSetVector<SDNode *, 16> UpdatedNodes;
    1432    39515675 :       bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
    1433             : 
    1434    39539884 :       for (SDNode *LN : UpdatedNodes) {
    1435       24212 :         AddToWorklist(LN);
    1436             :         AddUsersToWorklist(LN);
    1437             :       }
    1438    39515672 :       if (!NIsValid)
    1439             :         continue;
    1440             :     }
    1441             : 
    1442             :     LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
    1443             : 
    1444             :     // Add any operands of the new node which have not yet been combined to the
    1445             :     // worklist as well. Because the worklist uniques things already, this
    1446             :     // won't repeatedly process the same operand.
    1447    89672975 :     CombinedNodes.insert(N);
    1448   253341709 :     for (const SDValue &ChildN : N->op_values())
    1449   163668732 :       if (!CombinedNodes.count(ChildN.getNode()))
    1450   151614519 :         AddToWorklist(ChildN.getNode());
    1451             : 
    1452    89672977 :     SDValue RV = combine(N);
    1453             : 
    1454    89672978 :     if (!RV.getNode())
    1455             :       continue;
    1456             : 
    1457             :     ++NodesCombined;
    1458             : 
    1459             :     // If we get back the same node we passed in, rather than a new node or
    1460             :     // zero, we know that the node must have defined multiple values and
    1461             :     // CombineTo was used.  Since CombineTo takes care of the worklist
    1462             :     // mechanics for us, we have no work to do in this case.
    1463     3322735 :     if (RV.getNode() == N)
    1464             :       continue;
    1465             : 
    1466             :     assert(N->getOpcode() != ISD::DELETED_NODE &&
    1467             :            RV.getOpcode() != ISD::DELETED_NODE &&
    1468             :            "Node was deleted but visit returned new node!");
    1469             : 
    1470             :     LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG));
    1471             : 
    1472     1907099 :     if (N->getNumValues() == RV.getNode()->getNumValues())
    1473     1711878 :       DAG.ReplaceAllUsesWith(N, RV.getNode());
    1474             :     else {
    1475             :       assert(N->getValueType(0) == RV.getValueType() &&
    1476             :              N->getNumValues() == 1 && "Type mismatch");
    1477      195221 :       DAG.ReplaceAllUsesWith(N, &RV);
    1478             :     }
    1479             : 
    1480             :     // Push the new node and any users onto the worklist
    1481     1907099 :     AddToWorklist(RV.getNode());
    1482     1907099 :     AddUsersToWorklist(RV.getNode());
    1483             : 
    1484             :     // Finally, if the node is now dead, remove it from the graph.  The node
    1485             :     // may not be dead if the replacement process recursively simplified to
    1486             :     // something else needing this node. This will also take care of adding any
    1487             :     // operands which have lost a user to the worklist.
    1488     1907099 :     recursivelyDeleteUnusedNodes(N);
    1489             :   }
    1490             : 
    1491             :   // If the root changed (e.g. it was a dead load, update the root).
    1492     2800676 :   DAG.setRoot(Dummy.getValue());
    1493     2800676 :   DAG.RemoveDeadNodes();
    1494     2800676 : }
    1495             : 
    1496    89673703 : SDValue DAGCombiner::visit(SDNode *N) {
    1497   179347406 :   switch (N->getOpcode()) {
    1498             :   default: break;
    1499     4812108 :   case ISD::TokenFactor:        return visitTokenFactor(N);
    1500      441173 :   case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
    1501     3376644 :   case ISD::ADD:                return visitADD(N);
    1502      535947 :   case ISD::SUB:                return visitSUB(N);
    1503         522 :   case ISD::ADDC:               return visitADDC(N);
    1504      136837 :   case ISD::UADDO:              return visitUADDO(N);
    1505          89 :   case ISD::SUBC:               return visitSUBC(N);
    1506        1911 :   case ISD::USUBO:              return visitUSUBO(N);
    1507        2544 :   case ISD::ADDE:               return visitADDE(N);
    1508      162301 :   case ISD::ADDCARRY:           return visitADDCARRY(N);
    1509          79 :   case ISD::SUBE:               return visitSUBE(N);
    1510         838 :   case ISD::SUBCARRY:           return visitSUBCARRY(N);
    1511       39002 :   case ISD::MUL:                return visitMUL(N);
    1512        5989 :   case ISD::SDIV:               return visitSDIV(N);
    1513        4417 :   case ISD::UDIV:               return visitUDIV(N);
    1514        4731 :   case ISD::SREM:
    1515        4731 :   case ISD::UREM:               return visitREM(N);
    1516        6323 :   case ISD::MULHU:              return visitMULHU(N);
    1517        1447 :   case ISD::MULHS:              return visitMULHS(N);
    1518         726 :   case ISD::SMUL_LOHI:          return visitSMUL_LOHI(N);
    1519        5604 :   case ISD::UMUL_LOHI:          return visitUMUL_LOHI(N);
    1520          99 :   case ISD::SMULO:              return visitSMULO(N);
    1521         228 :   case ISD::UMULO:              return visitUMULO(N);
    1522       25227 :   case ISD::SMIN:
    1523             :   case ISD::SMAX:
    1524             :   case ISD::UMIN:
    1525       25227 :   case ISD::UMAX:               return visitIMINMAX(N);
    1526      389060 :   case ISD::AND:                return visitAND(N);
    1527      143210 :   case ISD::OR:                 return visitOR(N);
    1528       85823 :   case ISD::XOR:                return visitXOR(N);
    1529      126522 :   case ISD::SHL:                return visitSHL(N);
    1530       29698 :   case ISD::SRA:                return visitSRA(N);
    1531      161976 :   case ISD::SRL:                return visitSRL(N);
    1532        2760 :   case ISD::ROTR:
    1533        2760 :   case ISD::ROTL:               return visitRotate(N);
    1534        1188 :   case ISD::ABS:                return visitABS(N);
    1535        1396 :   case ISD::BSWAP:              return visitBSWAP(N);
    1536         531 :   case ISD::BITREVERSE:         return visitBITREVERSE(N);
    1537        1212 :   case ISD::CTLZ:               return visitCTLZ(N);
    1538        1569 :   case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
    1539         474 :   case ISD::CTTZ:               return visitCTTZ(N);
    1540         650 :   case ISD::CTTZ_ZERO_UNDEF:    return visitCTTZ_ZERO_UNDEF(N);
    1541        1995 :   case ISD::CTPOP:              return visitCTPOP(N);
    1542       32128 :   case ISD::SELECT:             return visitSELECT(N);
    1543       39285 :   case ISD::VSELECT:            return visitVSELECT(N);
    1544       16218 :   case ISD::SELECT_CC:          return visitSELECT_CC(N);
    1545      297657 :   case ISD::SETCC:              return visitSETCC(N);
    1546         285 :   case ISD::SETCCCARRY:         return visitSETCCCARRY(N);
    1547       52105 :   case ISD::SIGN_EXTEND:        return visitSIGN_EXTEND(N);
    1548      140434 :   case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
    1549       87023 :   case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
    1550      189122 :   case ISD::AssertSext:
    1551      189122 :   case ISD::AssertZext:         return visitAssertExt(N);
    1552       45301 :   case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
    1553        2920 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
    1554        5771 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
    1555      862466 :   case ISD::TRUNCATE:           return visitTRUNCATE(N);
    1556      817480 :   case ISD::BITCAST:            return visitBITCAST(N);
    1557             :   case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
    1558       42897 :   case ISD::FADD:               return visitFADD(N);
    1559       12552 :   case ISD::FSUB:               return visitFSUB(N);
    1560       27119 :   case ISD::FMUL:               return visitFMUL(N);
    1561        9619 :   case ISD::FMA:                return visitFMA(N);
    1562       10224 :   case ISD::FDIV:               return visitFDIV(N);
    1563         283 :   case ISD::FREM:               return visitFREM(N);
    1564        2072 :   case ISD::FSQRT:              return visitFSQRT(N);
    1565        1439 :   case ISD::FCOPYSIGN:          return visitFCOPYSIGN(N);
    1566         249 :   case ISD::FPOW:               return visitFPOW(N);
    1567       21250 :   case ISD::SINT_TO_FP:         return visitSINT_TO_FP(N);
    1568       12399 :   case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
    1569        8950 :   case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
    1570        6272 :   case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
    1571        4242 :   case ISD::FP_ROUND:           return visitFP_ROUND(N);
    1572           0 :   case ISD::FP_ROUND_INREG:     return visitFP_ROUND_INREG(N);
    1573        9310 :   case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
    1574        6661 :   case ISD::FNEG:               return visitFNEG(N);
    1575        5093 :   case ISD::FABS:               return visitFABS(N);
    1576        1640 :   case ISD::FFLOOR:             return visitFFLOOR(N);
    1577        3382 :   case ISD::FMINNUM:            return visitFMINNUM(N);
    1578        3287 :   case ISD::FMAXNUM:            return visitFMAXNUM(N);
    1579        4299 :   case ISD::FCEIL:              return visitFCEIL(N);
    1580        1546 :   case ISD::FTRUNC:             return visitFTRUNC(N);
    1581      258110 :   case ISD::BRCOND:             return visitBRCOND(N);
    1582        6329 :   case ISD::BR_CC:              return visitBR_CC(N);
    1583     6550937 :   case ISD::LOAD:               return visitLOAD(N);
    1584     8092954 :   case ISD::STORE:              return visitSTORE(N);
    1585       51586 :   case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
    1586      409341 :   case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
    1587      698745 :   case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
    1588       30553 :   case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
    1589       73209 :   case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
    1590       71081 :   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
    1591       22779 :   case ISD::SCALAR_TO_VECTOR:   return visitSCALAR_TO_VECTOR(N);
    1592       19127 :   case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
    1593        1004 :   case ISD::MGATHER:            return visitMGATHER(N);
    1594        1291 :   case ISD::MLOAD:              return visitMLOAD(N);
    1595         219 :   case ISD::MSCATTER:           return visitMSCATTER(N);
    1596         758 :   case ISD::MSTORE:             return visitMSTORE(N);
    1597             :   case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
    1598        6737 :   case ISD::FP16_TO_FP:         return visitFP16_TO_FP(N);
    1599             :   }
    1600    60028765 :   return SDValue();
    1601             : }
    1602             : 
    1603    89673705 : SDValue DAGCombiner::combine(SDNode *N) {
    1604    89673705 :   SDValue RV = visit(N);
    1605             : 
    1606             :   // If nothing happened, try a target-specific DAG combine.
    1607    89673704 :   if (!RV.getNode()) {
    1608             :     assert(N->getOpcode() != ISD::DELETED_NODE &&
    1609             :            "Node was deleted but visit returned NULL!");
    1610             : 
    1611   173289718 :     if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
    1612    81997330 :         TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
    1613             : 
    1614             :       // Expose the DAG combiner to the target combiner impls.
    1615             :       TargetLowering::DAGCombinerInfo
    1616    25174546 :         DagCombineInfo(DAG, Level, false, this);
    1617             : 
    1618    25174546 :       RV = TLI.PerformDAGCombine(N, DagCombineInfo);
    1619             :     }
    1620             :   }
    1621             : 
    1622             :   // If nothing happened still, try promoting the operation.
    1623    89673706 :   if (!RV.getNode()) {
    1624   172712688 :     switch (N->getOpcode()) {
    1625             :     default: break;
    1626             :     case ISD::ADD:
    1627             :     case ISD::SUB:
    1628             :     case ISD::MUL:
    1629             :     case ISD::AND:
    1630             :     case ISD::OR:
    1631             :     case ISD::XOR:
    1632     4022167 :       RV = PromoteIntBinOp(SDValue(N, 0));
    1633     4022167 :       break;
    1634             :     case ISD::SHL:
    1635             :     case ISD::SRA:
    1636             :     case ISD::SRL:
    1637      259714 :       RV = PromoteIntShiftOp(SDValue(N, 0));
    1638      259714 :       break;
    1639             :     case ISD::SIGN_EXTEND:
    1640             :     case ISD::ZERO_EXTEND:
    1641             :     case ISD::ANY_EXTEND:
    1642      185782 :       RV = PromoteExtend(SDValue(N, 0));
    1643      185782 :       break;
    1644             :     case ISD::LOAD:
    1645     6218337 :       if (PromoteLoad(SDValue(N, 0)))
    1646           0 :         RV = SDValue(N, 0);
    1647             :       break;
    1648             :     }
    1649             :   }
    1650             : 
    1651             :   // If N is a commutative binary node, try eliminate it if the commuted
    1652             :   // version is already present in the DAG.
    1653    89673706 :   if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
    1654     3741540 :       N->getNumValues() == 1) {
    1655     3596550 :     SDValue N0 = N->getOperand(0);
    1656     3596550 :     SDValue N1 = N->getOperand(1);
    1657             : 
    1658             :     // Constant operands are canonicalized to RHS.
    1659             :     if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
    1660      644790 :       SDValue Ops[] = {N1, N0};
    1661     2579160 :       SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
    1662             :                                             N->getFlags());
    1663      644790 :       if (CSENode)
    1664          19 :         return SDValue(CSENode, 0);
    1665             :     }
    1666             :   }
    1667             : 
    1668    89673687 :   return RV;
    1669             : }
    1670             : 
    1671             : /// Given a node, return its input chain if it has one, otherwise return a null
    1672             : /// sd operand.
    1673     7302459 : static SDValue getInputChainForNode(SDNode *N) {
    1674    14604918 :   if (unsigned NumOps = N->getNumOperands()) {
    1675     7281176 :     if (N->getOperand(0).getValueType() == MVT::Other)
    1676     7232114 :       return N->getOperand(0);
    1677       98124 :     if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
    1678       49062 :       return N->getOperand(NumOps-1);
    1679           0 :     for (unsigned i = 1; i < NumOps-1; ++i)
    1680           0 :       if (N->getOperand(i).getValueType() == MVT::Other)
    1681           0 :         return N->getOperand(i);
    1682             :   }
    1683       21283 :   return SDValue();
    1684             : }
    1685             : 
    1686     4812108 : SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
    1687             :   // If N has two operands, where one has an input chain equal to the other,
    1688             :   // the 'other' chain is redundant.
    1689     4812108 :   if (N->getNumOperands() == 2) {
    1690     3658931 :     if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
    1691       15403 :       return N->getOperand(0);
    1692     3643528 :     if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
    1693       90055 :       return N->getOperand(1);
    1694             :   }
    1695             : 
    1696             :   // Don't simplify token factors if optnone.
    1697     4706650 :   if (OptLevel == CodeGenOpt::None)
    1698     2548639 :     return SDValue();
    1699             : 
    1700             :   SmallVector<SDNode *, 8> TFs;     // List of token factors to visit.
    1701             :   SmallVector<SDValue, 8> Ops;      // Ops for replacing token factor.
    1702             :   SmallPtrSet<SDNode*, 16> SeenOps;
    1703     2158011 :   bool Changed = false;             // If we should replace this token factor.
    1704             : 
    1705             :   // Start out with this token factor.
    1706     2158011 :   TFs.push_back(N);
    1707             : 
    1708             :   // Iterate through token factors.  The TFs grows when new token factors are
    1709             :   // encountered.
    1710     5069268 :   for (unsigned i = 0; i < TFs.size(); ++i) {
    1711     2911257 :     SDNode *TF = TFs[i];
    1712             : 
    1713             :     // Check each of the operands.
    1714    11421763 :     for (const SDValue &Op : TF->op_values()) {
    1715    17021012 :       switch (Op.getOpcode()) {
    1716       31801 :       case ISD::EntryToken:
    1717             :         // Entry tokens don't need to be added to the list. They are
    1718             :         // redundant.
    1719       31801 :         Changed = true;
    1720       31801 :         break;
    1721             : 
    1722     1329925 :       case ISD::TokenFactor:
    1723     1329925 :         if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
    1724             :           // Queue up for processing.
    1725      753246 :           TFs.push_back(Op.getNode());
    1726             :           // Clean up in case the token factor is removed.
    1727      753246 :           AddToWorklist(Op.getNode());
    1728      753246 :           Changed = true;
    1729      753246 :           break;
    1730             :         }
    1731             :         LLVM_FALLTHROUGH;
    1732             : 
    1733             :       default:
    1734             :         // Only add if it isn't already in the list.
    1735     7725459 :         if (SeenOps.insert(Op.getNode()).second)
    1736     7597234 :           Ops.push_back(Op);
    1737             :         else
    1738      128225 :           Changed = true;
    1739             :         break;
    1740             :       }
    1741             :     }
    1742             :   }
    1743             : 
    1744             :   // Remove Nodes that are chained to another node in the list. Do so
    1745             :   // by walking up chains breath-first stopping when we've seen
    1746             :   // another operand. In general we must climb to the EntryNode, but we can exit
    1747             :   // early if we find all remaining work is associated with just one operand as
    1748             :   // no further pruning is possible.
    1749             : 
    1750             :   // List of nodes to search through and original Ops from which they originate.
    1751             :   SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
    1752             :   SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
    1753             :   SmallPtrSet<SDNode *, 16> SeenChains;
    1754     2158011 :   bool DidPruneOps = false;
    1755             : 
    1756     2158011 :   unsigned NumLeftToConsider = 0;
    1757     9755245 :   for (const SDValue &Op : Ops) {
    1758     7597234 :     Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
    1759     7597234 :     OpWorkCount.push_back(1);
    1760             :   }
    1761             : 
    1762             :   auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
    1763             :     // If this is an Op, we can remove the op from the list. Remark any
    1764             :     // search associated with it as from the current OpNumber.
    1765             :     if (SeenOps.count(Op) != 0) {
    1766             :       Changed = true;
    1767             :       DidPruneOps = true;
    1768             :       unsigned OrigOpNumber = 0;
    1769             :       while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
    1770             :         OrigOpNumber++;
    1771             :       assert((OrigOpNumber != Ops.size()) &&
    1772             :              "expected to find TokenFactor Operand");
    1773             :       // Re-mark worklist from OrigOpNumber to OpNumber
    1774             :       for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
    1775             :         if (Worklist[i].second == OrigOpNumber) {
    1776             :           Worklist[i].second = OpNumber;
    1777             :         }
    1778             :       }
    1779             :       OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
    1780             :       OpWorkCount[OrigOpNumber] = 0;
    1781             :       NumLeftToConsider--;
    1782             :     }
    1783             :     // Add if it's a new chain
    1784             :     if (SeenChains.insert(Op).second) {
    1785             :       OpWorkCount[OpNumber]++;
    1786             :       Worklist.push_back(std::make_pair(Op, OpNumber));
    1787             :     }
    1788     2158011 :   };
    1789             : 
    1790    14909549 :   for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
    1791             :     // We need at least be consider at least 2 Ops to prune.
    1792    14431495 :     if (NumLeftToConsider <= 1)
    1793             :       break;
    1794    12751538 :     auto CurNode = Worklist[i].first;
    1795    12751538 :     auto CurOpNumber = Worklist[i].second;
    1796             :     assert((OpWorkCount[CurOpNumber] > 0) &&
    1797             :            "Node should not appear in worklist");
    1798    25503076 :     switch (CurNode->getOpcode()) {
    1799      497703 :     case ISD::EntryToken:
    1800             :       // Hitting EntryToken is the only way for the search to terminate without
    1801             :       // hitting
    1802             :       // another operand's search. Prevent us from marking this operand
    1803             :       // considered.
    1804      497703 :       NumLeftToConsider++;
    1805      497703 :       break;
    1806             :     case ISD::TokenFactor:
    1807     4371209 :       for (const SDValue &Op : CurNode->op_values())
    1808     3123304 :         AddToWorklist(i, Op.getNode(), CurOpNumber);
    1809             :       break;
    1810      989472 :     case ISD::CopyFromReg:
    1811             :     case ISD::CopyToReg:
    1812      989472 :       AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
    1813      989472 :       break;
    1814             :     default:
    1815             :       if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
    1816     9242101 :         AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
    1817             :       break;
    1818             :     }
    1819    12751538 :     OpWorkCount[CurOpNumber]--;
    1820    12751538 :     if (OpWorkCount[CurOpNumber] == 0)
    1821     5384511 :       NumLeftToConsider--;
    1822             :   }
    1823             : 
    1824             :   // If we've changed things around then replace token factor.
    1825     2158011 :   if (Changed) {
    1826             :     SDValue Result;
    1827      592218 :     if (Ops.empty()) {
    1828             :       // The entry token is the only possible outcome.
    1829         319 :       Result = DAG.getEntryNode();
    1830             :     } else {
    1831      591899 :       if (DidPruneOps) {
    1832             :         SmallVector<SDValue, 8> PrunedOps;
    1833             :         //
    1834     1860520 :         for (const SDValue &Op : Ops) {
    1835     1514132 :           if (SeenChains.count(Op.getNode()) == 0)
    1836      936726 :             PrunedOps.push_back(Op);
    1837             :         }
    1838     1325100 :         Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
    1839             :       } else {
    1840      815252 :         Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
    1841             :       }
    1842             :     }
    1843      592218 :     return Result;
    1844             :   }
    1845     1565793 :   return SDValue();
    1846             : }
    1847             : 
    1848             : /// MERGE_VALUES can always be eliminated.
    1849      441173 : SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
    1850             :   WorklistRemover DeadNodes(*this);
    1851             :   // Replacing results may cause a different MERGE_VALUES to suddenly
    1852             :   // be CSE'd with N, and carry its uses with it. Iterate until no
    1853             :   // uses remain, to ensure that the node can be safely deleted.
    1854             :   // First add the users of this node to the work list so that they
    1855             :   // can be tried again once they have new operands.
    1856             :   AddUsersToWorklist(N);
    1857             :   do {
    1858             :     // Do as a single replacement to avoid rewalking use lists.
    1859             :     SmallVector<SDValue, 8> Ops;
    1860     1328933 :     for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
    1861     1775520 :       Ops.push_back(N->getOperand(i));
    1862      441173 :     DAG.ReplaceAllUsesWith(N, Ops.data());
    1863      441173 :   } while (!N->use_empty());
    1864      441173 :   deleteAndRecombine(N);
    1865      882346 :   return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    1866             : }
    1867             : 
    1868             : /// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
    1869             : /// ConstantSDNode pointer else nullptr.
    1870           0 : static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
    1871             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
    1872      111726 :   return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
    1873             : }
    1874             : 
    1875           0 : SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
    1876             :   assert(ISD::isBinaryOp(BO) && "Unexpected binary operator");
    1877             : 
    1878             :   // Don't do this unless the old select is going away. We want to eliminate the
    1879             :   // binary operator, not replace a binop with a select.
    1880             :   // TODO: Handle ISD::SELECT_CC.
    1881             :   unsigned SelOpNo = 0;
    1882           0 :   SDValue Sel = BO->getOperand(0);
    1883           0 :   if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
    1884             :     SelOpNo = 1;
    1885           0 :     Sel = BO->getOperand(1);
    1886             :   }
    1887             : 
    1888           0 :   if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
    1889           0 :     return SDValue();
    1890             : 
    1891           0 :   SDValue CT = Sel.getOperand(1);
    1892           0 :   if (!isConstantOrConstantVector(CT, true) &&
    1893             :       !isConstantFPBuildVectorOrConstantFP(CT))
    1894           0 :     return SDValue();
    1895             : 
    1896           0 :   SDValue CF = Sel.getOperand(2);
    1897           0 :   if (!isConstantOrConstantVector(CF, true) &&
    1898             :       !isConstantFPBuildVectorOrConstantFP(CF))
    1899           0 :     return SDValue();
    1900             : 
    1901             :   // Bail out if any constants are opaque because we can't constant fold those.
    1902             :   // The exception is "and" and "or" with either 0 or -1 in which case we can
    1903             :   // propagate non constant operands into select. I.e.:
    1904             :   // and (select Cond, 0, -1), X --> select Cond, 0, X
    1905             :   // or X, (select Cond, -1, 0) --> select Cond, -1, X
    1906           0 :   auto BinOpcode = BO->getOpcode();
    1907           0 :   bool CanFoldNonConst = (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
    1908           0 :                          (isNullConstantOrNullSplatConstant(CT) ||
    1909           0 :                           isAllOnesConstantOrAllOnesSplatConstant(CT)) &&
    1910           0 :                          (isNullConstantOrNullSplatConstant(CF) ||
    1911           0 :                           isAllOnesConstantOrAllOnesSplatConstant(CF));
    1912             : 
    1913           0 :   SDValue CBO = BO->getOperand(SelOpNo ^ 1);
    1914           0 :   if (!CanFoldNonConst &&
    1915           0 :       !isConstantOrConstantVector(CBO, true) &&
    1916             :       !isConstantFPBuildVectorOrConstantFP(CBO))
    1917           0 :     return SDValue();
    1918             : 
    1919           0 :   EVT VT = Sel.getValueType();
    1920             : 
    1921             :   // In case of shift value and shift amount may have different VT. For instance
    1922             :   // on x86 shift amount is i8 regardles of LHS type. Bail out if we have
    1923             :   // swapped operands and value types do not match. NB: x86 is fine if operands
    1924             :   // are not swapped with shift amount VT being not bigger than shifted value.
    1925             :   // TODO: that is possible to check for a shift operation, correct VTs and
    1926             :   // still perform optimization on x86 if needed.
    1927           0 :   if (SelOpNo && VT != CBO.getValueType())
    1928           0 :     return SDValue();
    1929             : 
    1930             :   // We have a select-of-constants followed by a binary operator with a
    1931             :   // constant. Eliminate the binop by pulling the constant math into the select.
    1932             :   // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
    1933           0 :   SDLoc DL(Sel);
    1934           0 :   SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
    1935           0 :                           : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
    1936           0 :   if (!CanFoldNonConst && !NewCT.isUndef() &&
    1937           0 :       !isConstantOrConstantVector(NewCT, true) &&
    1938             :       !isConstantFPBuildVectorOrConstantFP(NewCT))
    1939           0 :     return SDValue();
    1940             : 
    1941           0 :   SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
    1942           0 :                           : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
    1943           0 :   if (!CanFoldNonConst && !NewCF.isUndef() &&
    1944           0 :       !isConstantOrConstantVector(NewCF, true) &&
    1945             :       !isConstantFPBuildVectorOrConstantFP(NewCF))
    1946           0 :     return SDValue();
    1947             : 
    1948           0 :   return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
    1949             : }
    1950             : 
    1951     3671556 : static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
    1952             :   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
    1953             :          "Expecting add or sub");
    1954             : 
    1955             :   // Match a constant operand and a zext operand for the math instruction:
    1956             :   // add Z, C
    1957             :   // sub C, Z
    1958     3671556 :   bool IsAdd = N->getOpcode() == ISD::ADD;
    1959     3671556 :   SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
    1960     3671556 :   SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
    1961             :   auto *CN = dyn_cast<ConstantSDNode>(C);
    1962     2674175 :   if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
    1963     3670753 :     return SDValue();
    1964             : 
    1965             :   // Match the zext operand as a setcc of a boolean.
    1966        1606 :   if (Z.getOperand(0).getOpcode() != ISD::SETCC ||
    1967         143 :       Z.getOperand(0).getValueType() != MVT::i1)
    1968         679 :     return SDValue();
    1969             : 
    1970             :   // Match the compare as: setcc (X & 1), 0, eq.
    1971             :   SDValue SetCC = Z.getOperand(0);
    1972         124 :   ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
    1973          37 :   if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) ||
    1974         175 :       SetCC.getOperand(0).getOpcode() != ISD::AND ||
    1975          23 :       !isOneConstant(SetCC.getOperand(0).getOperand(1)))
    1976         102 :     return SDValue();
    1977             : 
    1978             :   // We are adding/subtracting a constant and an inverted low bit. Turn that
    1979             :   // into a subtract/add of the low bit with incremented/decremented constant:
    1980             :   // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
    1981             :   // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
    1982          22 :   EVT VT = C.getValueType();
    1983             :   SDLoc DL(N);
    1984          22 :   SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
    1985          61 :   SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
    1986          62 :                        DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
    1987          31 :   return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
    1988             : }
    1989             : 
    1990             : /// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
    1991             : /// a shift and add with a different constant.
    1992     3671534 : static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
    1993             :   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
    1994             :          "Expecting add or sub");
    1995             : 
    1996             :   // We need a constant operand for the add/sub, and the other operand is a
    1997             :   // logical shift right: add (srl), C or sub C, (srl).
    1998     3671534 :   bool IsAdd = N->getOpcode() == ISD::ADD;
    1999     3671534 :   SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
    2000     3671534 :   SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
    2001     3671534 :   ConstantSDNode *C = isConstOrConstSplat(ConstantOp);
    2002     3671534 :   if (!C || ShiftOp.getOpcode() != ISD::SRL)
    2003     3670174 :     return SDValue();
    2004             : 
    2005             :   // The shift must be of a 'not' value.
    2006        1360 :   SDValue Not = ShiftOp.getOperand(0);
    2007        1360 :   if (!Not.hasOneUse() || !isBitwiseNot(Not))
    2008        1341 :     return SDValue();
    2009             : 
    2010             :   // The shift must be moving the sign bit to the least-significant-bit.
    2011          19 :   EVT VT = ShiftOp.getValueType();
    2012          19 :   SDValue ShAmt = ShiftOp.getOperand(1);
    2013          19 :   ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
    2014          38 :   if (!ShAmtC || ShAmtC->getZExtValue() != VT.getScalarSizeInBits() - 1)
    2015           0 :     return SDValue();
    2016             : 
    2017             :   // Eliminate the 'not' by adjusting the shift and add/sub constant:
    2018             :   // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
    2019             :   // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
    2020             :   SDLoc DL(N);
    2021          19 :   auto ShOpcode = IsAdd ? ISD::SRA : ISD::SRL;
    2022          19 :   SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
    2023          48 :   APInt NewC = IsAdd ? C->getAPIntValue() + 1 : C->getAPIntValue() - 1;
    2024          19 :   return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT));
    2025             : }
    2026             : 
    2027     3376644 : SDValue DAGCombiner::visitADD(SDNode *N) {
    2028     3376644 :   SDValue N0 = N->getOperand(0);
    2029     3376644 :   SDValue N1 = N->getOperand(1);
    2030     3376644 :   EVT VT = N0.getValueType();
    2031             :   SDLoc DL(N);
    2032             : 
    2033             :   // fold vector ops
    2034     3376644 :   if (VT.isVector()) {
    2035      142394 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2036          17 :       return FoldedVOp;
    2037             : 
    2038             :     // fold (add x, 0) -> x, vector edition
    2039      142377 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    2040         137 :       return N0;
    2041      142240 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    2042         154 :       return N1;
    2043             :   }
    2044             : 
    2045             :   // fold (add x, undef) -> undef
    2046     3376336 :   if (N0.isUndef())
    2047           3 :     return N0;
    2048             : 
    2049     3376333 :   if (N1.isUndef())
    2050           9 :     return N1;
    2051             : 
    2052     3376324 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
    2053             :     // canonicalize constant to RHS
    2054        2086 :     if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
    2055        2916 :       return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
    2056             :     // fold (add c1, c2) -> c1+c2
    2057         628 :     return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
    2058         628 :                                       N1.getNode());
    2059             :   }
    2060             : 
    2061             :   // fold (add x, 0) -> x
    2062     3374238 :   if (isNullConstant(N1))
    2063         330 :     return N0;
    2064             : 
    2065     3373908 :   if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
    2066             :     // fold ((c1-A)+c2) -> (c1+c2)-A
    2067     2993507 :     if (N0.getOpcode() == ISD::SUB &&
    2068         637 :         isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
    2069             :       // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
    2070          31 :       return DAG.getNode(ISD::SUB, DL, VT,
    2071             :                          DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
    2072          31 :                          N0.getOperand(1));
    2073             :     }
    2074             : 
    2075             :     // add (sext i1 X), 1 -> zext (not i1 X)
    2076             :     // We don't transform this pattern:
    2077             :     //   add (zext i1 X), -1 -> sext (not i1 X)
    2078             :     // because most (?) targets generate better code for the zext form.
    2079     2994778 :     if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
    2080         932 :         isOneConstantOrOneSplatConstant(N1)) {
    2081          37 :       SDValue X = N0.getOperand(0);
    2082          37 :       if ((!LegalOperations ||
    2083           3 :            (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
    2084          73 :             TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
    2085          36 :           X.getScalarValueSizeInBits() == 1) {
    2086          34 :         SDValue Not = DAG.getNOT(DL, X, X.getValueType());
    2087          34 :         return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
    2088             :       }
    2089             :     }
    2090             : 
    2091             :     // Undo the add -> or combine to merge constant offsets from a frame index.
    2092             :     if (N0.getOpcode() == ISD::OR &&
    2093             :         isa<FrameIndexSDNode>(N0.getOperand(0)) &&
    2094     2995668 :         isa<ConstantSDNode>(N0.getOperand(1)) &&
    2095        5692 :         DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
    2096        5692 :       SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
    2097        5692 :       return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
    2098             :     }
    2099             :   }
    2100             : 
    2101     3371014 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2102           8 :     return NewSel;
    2103             : 
    2104             :   // reassociate add
    2105     3371006 :   if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
    2106      226276 :     return RADD;
    2107             : 
    2108             :   // fold ((0-A) + B) -> B-A
    2109     3147549 :   if (N0.getOpcode() == ISD::SUB &&
    2110        2819 :       isNullConstantOrNullSplatConstant(N0.getOperand(0)))
    2111          54 :     return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
    2112             : 
    2113             :   // fold (A + (0-B)) -> A-B
    2114     3150865 :   if (N1.getOpcode() == ISD::SUB &&
    2115        6162 :       isNullConstantOrNullSplatConstant(N1.getOperand(0)))
    2116         620 :     return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
    2117             : 
    2118             :   // fold (A+(B-A)) -> B
    2119     3144393 :   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
    2120           4 :     return N1.getOperand(0);
    2121             : 
    2122             :   // fold ((B-A)+A) -> B
    2123     3144389 :   if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
    2124          31 :     return N0.getOperand(0);
    2125             : 
    2126             :   // fold (A+(B-(A+C))) to (B-C)
    2127     3144358 :   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
    2128          63 :       N0 == N1.getOperand(1).getOperand(0))
    2129           2 :     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
    2130           2 :                        N1.getOperand(1).getOperand(1));
    2131             : 
    2132             :   // fold (A+(B-(C+A))) to (B-C)
    2133     3144356 :   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
    2134          61 :       N0 == N1.getOperand(1).getOperand(1))
    2135           3 :     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
    2136           3 :                        N1.getOperand(1).getOperand(0));
    2137             : 
    2138             :   // fold (A+((B-A)+or-C)) to (B+or-C)
    2139     3144353 :   if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
    2140     3148317 :       N1.getOperand(0).getOpcode() == ISD::SUB &&
    2141          90 :       N0 == N1.getOperand(0).getOperand(1))
    2142           5 :     return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
    2143           5 :                        N1.getOperand(1));
    2144             : 
    2145             :   // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
    2146     3144348 :   if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
    2147          18 :     SDValue N00 = N0.getOperand(0);
    2148          18 :     SDValue N01 = N0.getOperand(1);
    2149          18 :     SDValue N10 = N1.getOperand(0);
    2150          18 :     SDValue N11 = N1.getOperand(1);
    2151             : 
    2152          18 :     if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
    2153           5 :       return DAG.getNode(ISD::SUB, DL, VT,
    2154           5 :                          DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
    2155          15 :                          DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
    2156             :   }
    2157             : 
    2158     3144343 :   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
    2159          13 :     return V;
    2160             : 
    2161     3144330 :   if (SDValue V = foldAddSubOfSignBit(N, DAG))
    2162          10 :     return V;
    2163             : 
    2164     3144320 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    2165        4734 :     return SDValue(N, 0);
    2166             : 
    2167             :   // fold (a+b) -> (a|b) iff a and b share no bits.
    2168     6263036 :   if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
    2169     3123450 :       DAG.haveNoCommonBitsSet(N0, N1))
    2170       63824 :     return DAG.getNode(ISD::OR, DL, VT, N0, N1);
    2171             : 
    2172             :   // fold (add (xor a, -1), 1) -> (sub 0, a)
    2173     3107674 :   if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1))
    2174           9 :     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
    2175           9 :                        N0.getOperand(0));
    2176             : 
    2177     3107665 :   if (SDValue Combined = visitADDLike(N0, N1, N))
    2178         804 :     return Combined;
    2179             : 
    2180     3106861 :   if (SDValue Combined = visitADDLike(N1, N0, N))
    2181          73 :     return Combined;
    2182             : 
    2183     3106788 :   return SDValue();
    2184             : }
    2185             : 
    2186     5921795 : static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
    2187             :   bool Masked = false;
    2188             : 
    2189             :   // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
    2190             :   while (true) {
    2191     5932671 :     if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
    2192        8677 :       V = V.getOperand(0);
    2193        8677 :       continue;
    2194             :     }
    2195             : 
    2196     5923994 :     if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
    2197             :       Masked = true;
    2198        2199 :       V = V.getOperand(0);
    2199        2199 :       continue;
    2200             :     }
    2201             : 
    2202             :     break;
    2203             :   }
    2204             : 
    2205             :   // If this is not a carry, return.
    2206     5921795 :   if (V.getResNo() != 1)
    2207     5908877 :     return SDValue();
    2208             : 
    2209       12646 :   if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
    2210       24932 :       V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
    2211       12014 :     return SDValue();
    2212             : 
    2213             :   // If the result is masked, then no matter what kind of bool it is we can
    2214             :   // return. If it isn't, then we need to make sure the bool type is either 0 or
    2215             :   // 1 and not other values.
    2216         904 :   if (Masked ||
    2217         926 :       TLI.getBooleanContents(V.getValueType()) ==
    2218             :           TargetLoweringBase::ZeroOrOneBooleanContent)
    2219         904 :     return V;
    2220             : 
    2221           0 :   return SDValue();
    2222             : }
    2223             : 
    2224           0 : SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
    2225           0 :   EVT VT = N0.getValueType();
    2226             :   SDLoc DL(LocReference);
    2227             : 
    2228             :   // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
    2229           0 :   if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
    2230           0 :       isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
    2231           0 :     return DAG.getNode(ISD::SUB, DL, VT, N0,
    2232             :                        DAG.getNode(ISD::SHL, DL, VT,
    2233           0 :                                    N1.getOperand(0).getOperand(1),
    2234           0 :                                    N1.getOperand(1)));
    2235             : 
    2236           0 :   if (N1.getOpcode() == ISD::AND) {
    2237           0 :     SDValue AndOp0 = N1.getOperand(0);
    2238           0 :     unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
    2239             :     unsigned DestBits = VT.getScalarSizeInBits();
    2240             : 
    2241             :     // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
    2242             :     // and similar xforms where the inner op is either ~0 or 0.
    2243           0 :     if (NumSignBits == DestBits &&
    2244           0 :         isOneConstantOrOneSplatConstant(N1->getOperand(1)))
    2245           0 :       return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
    2246             :   }
    2247             : 
    2248             :   // add (sext i1), X -> sub X, (zext i1)
    2249             :   if (N0.getOpcode() == ISD::SIGN_EXTEND &&
    2250           0 :       N0.getOperand(0).getValueType() == MVT::i1 &&
    2251           0 :       !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
    2252           0 :     SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
    2253           0 :     return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
    2254             :   }
    2255             : 
    2256             :   // add X, (sextinreg Y i1) -> sub X, (and Y 1)
    2257           0 :   if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2258             :     VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    2259           0 :     if (TN->getVT() == MVT::i1) {
    2260           0 :       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
    2261           0 :                                  DAG.getConstant(1, DL, VT));
    2262           0 :       return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
    2263             :     }
    2264             :   }
    2265             : 
    2266             :   // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
    2267           0 :   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
    2268           0 :       N1.getResNo() == 0)
    2269           0 :     return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
    2270           0 :                        N0, N1.getOperand(0), N1.getOperand(2));
    2271             : 
    2272             :   // (add X, Carry) -> (addcarry X, 0, Carry)
    2273           0 :   if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
    2274           0 :     if (SDValue Carry = getAsCarry(TLI, N1))
    2275           0 :       return DAG.getNode(ISD::ADDCARRY, DL,
    2276           0 :                          DAG.getVTList(VT, Carry.getValueType()), N0,
    2277           0 :                          DAG.getConstant(0, DL, VT), Carry);
    2278             : 
    2279           0 :   return SDValue();
    2280             : }
    2281             : 
    2282         522 : SDValue DAGCombiner::visitADDC(SDNode *N) {
    2283         522 :   SDValue N0 = N->getOperand(0);
    2284         522 :   SDValue N1 = N->getOperand(1);
    2285         522 :   EVT VT = N0.getValueType();
    2286             :   SDLoc DL(N);
    2287             : 
    2288             :   // If the flag result is dead, turn this into an ADD.
    2289         522 :   if (!N->hasAnyUseOfValue(1))
    2290          15 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2291          45 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2292             : 
    2293             :   // canonicalize constant to RHS.
    2294             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2295             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2296         507 :   if (N0C && !N1C)
    2297           0 :     return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
    2298             : 
    2299             :   // fold (addc x, 0) -> x + no carry out
    2300         507 :   if (isNullConstant(N1))
    2301          11 :     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
    2302          22 :                                         DL, MVT::Glue));
    2303             : 
    2304             :   // If it cannot overflow, transform into an add.
    2305         496 :   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
    2306          11 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2307          33 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2308             : 
    2309         485 :   return SDValue();
    2310             : }
    2311             : 
    2312           7 : static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT,
    2313             :                            SelectionDAG &DAG, const TargetLowering &TLI) {
    2314           7 :   SDValue Cst;
    2315           7 :   switch (TLI.getBooleanContents(VT)) {
    2316           7 :   case TargetLowering::ZeroOrOneBooleanContent:
    2317             :   case TargetLowering::UndefinedBooleanContent:
    2318           7 :     Cst = DAG.getConstant(1, DL, VT);
    2319           7 :     break;
    2320           0 :   case TargetLowering::ZeroOrNegativeOneBooleanContent:
    2321           0 :     Cst = DAG.getConstant(-1, DL, VT);
    2322           0 :     break;
    2323             :   }
    2324             : 
    2325           7 :   return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
    2326             : }
    2327             : 
    2328           0 : static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) {
    2329           0 :   if (V.getOpcode() != ISD::XOR) return false;
    2330             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V.getOperand(1));
    2331           0 :   if (!Const) return false;
    2332             : 
    2333           0 :   switch(TLI.getBooleanContents(VT)) {
    2334           0 :     case TargetLowering::ZeroOrOneBooleanContent:
    2335           0 :       return Const->isOne();
    2336           0 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    2337           0 :       return Const->isAllOnesValue();
    2338           0 :     case TargetLowering::UndefinedBooleanContent:
    2339           0 :       return (Const->getAPIntValue() & 0x01) == 1;
    2340             :   }
    2341           0 :   llvm_unreachable("Unsupported boolean content");
    2342             : }
    2343             : 
    2344      136837 : SDValue DAGCombiner::visitUADDO(SDNode *N) {
    2345      136837 :   SDValue N0 = N->getOperand(0);
    2346      136837 :   SDValue N1 = N->getOperand(1);
    2347      136837 :   EVT VT = N0.getValueType();
    2348      136837 :   if (VT.isVector())
    2349           0 :     return SDValue();
    2350             : 
    2351      273674 :   EVT CarryVT = N->getValueType(1);
    2352             :   SDLoc DL(N);
    2353             : 
    2354             :   // If the flag result is dead, turn this into an ADD.
    2355      136837 :   if (!N->hasAnyUseOfValue(1))
    2356         403 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2357         806 :                      DAG.getUNDEF(CarryVT));
    2358             : 
    2359             :   // canonicalize constant to RHS.
    2360             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2361             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2362      136434 :   if (N0C && !N1C)
    2363          10 :     return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
    2364             : 
    2365             :   // fold (uaddo x, 0) -> x + no carry out
    2366      136429 :   if (isNullConstant(N1))
    2367         904 :     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
    2368             : 
    2369             :   // If it cannot overflow, transform into an add.
    2370      135977 :   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
    2371         103 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2372         206 :                      DAG.getConstant(0, DL, CarryVT));
    2373             : 
    2374             :   // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
    2375      135874 :   if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1)) {
    2376           4 :     SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
    2377             :                               DAG.getConstant(0, DL, VT),
    2378           4 :                               N0.getOperand(0));
    2379             :     return CombineTo(N, Sub,
    2380           4 :                      flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
    2381             :   }
    2382             : 
    2383      135870 :   if (SDValue Combined = visitUADDOLike(N0, N1, N))
    2384         231 :     return Combined;
    2385             : 
    2386      135639 :   if (SDValue Combined = visitUADDOLike(N1, N0, N))
    2387           9 :     return Combined;
    2388             : 
    2389      135630 :   return SDValue();
    2390             : }
    2391             : 
    2392           0 : SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
    2393           0 :   auto VT = N0.getValueType();
    2394             : 
    2395             :   // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
    2396             :   // If Y + 1 cannot overflow.
    2397           0 :   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
    2398           0 :     SDValue Y = N1.getOperand(0);
    2399           0 :     SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
    2400           0 :     if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
    2401           0 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
    2402           0 :                          N1.getOperand(2));
    2403             :   }
    2404             : 
    2405             :   // (uaddo X, Carry) -> (addcarry X, 0, Carry)
    2406           0 :   if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
    2407           0 :     if (SDValue Carry = getAsCarry(TLI, N1))
    2408           0 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
    2409           0 :                          DAG.getConstant(0, SDLoc(N), VT), Carry);
    2410             : 
    2411           0 :   return SDValue();
    2412             : }
    2413             : 
    2414           0 : SDValue DAGCombiner::visitADDE(SDNode *N) {
    2415           0 :   SDValue N0 = N->getOperand(0);
    2416           0 :   SDValue N1 = N->getOperand(1);
    2417           0 :   SDValue CarryIn = N->getOperand(2);
    2418             : 
    2419             :   // canonicalize constant to RHS
    2420             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2421             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2422           0 :   if (N0C && !N1C)
    2423           0 :     return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
    2424           0 :                        N1, N0, CarryIn);
    2425             : 
    2426             :   // fold (adde x, y, false) -> (addc x, y)
    2427           0 :   if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    2428           0 :     return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
    2429             : 
    2430           0 :   return SDValue();
    2431             : }
    2432             : 
    2433      162301 : SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
    2434      162301 :   SDValue N0 = N->getOperand(0);
    2435      162301 :   SDValue N1 = N->getOperand(1);
    2436      162301 :   SDValue CarryIn = N->getOperand(2);
    2437             :   SDLoc DL(N);
    2438             : 
    2439             :   // canonicalize constant to RHS
    2440             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2441             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2442      162301 :   if (N0C && !N1C)
    2443         720 :     return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
    2444             : 
    2445             :   // fold (addcarry x, y, false) -> (uaddo x, y)
    2446      161941 :   if (isNullConstant(CarryIn)) {
    2447         565 :     if (!LegalOperations ||
    2448          76 :         TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
    2449        1130 :       return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
    2450             :   }
    2451             : 
    2452      161376 :   EVT CarryVT = CarryIn.getValueType();
    2453             : 
    2454             :   // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
    2455      161376 :   if (isNullConstant(N0) && isNullConstant(N1)) {
    2456         510 :     EVT VT = N0.getValueType();
    2457         510 :     SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
    2458         510 :     AddToWorklist(CarryExt.getNode());
    2459         510 :     return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
    2460             :                                     DAG.getConstant(1, DL, VT)),
    2461        1020 :                      DAG.getConstant(0, DL, CarryVT));
    2462             :   }
    2463             : 
    2464             :   // fold (addcarry (xor a, -1), 0, !b) -> (subcarry 0, a, b) and flip carry.
    2465      160874 :   if (isBitwiseNot(N0) && isNullConstant(N1) &&
    2466           8 :       isBooleanFlip(CarryIn, CarryVT, TLI)) {
    2467           3 :     SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(),
    2468             :                               DAG.getConstant(0, DL, N0.getValueType()),
    2469           3 :                               N0.getOperand(0), CarryIn.getOperand(0));
    2470             :     return CombineTo(N, Sub,
    2471           3 :                      flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
    2472             :   }
    2473             : 
    2474      160863 :   if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
    2475         234 :     return Combined;
    2476             : 
    2477      160629 :   if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
    2478           0 :     return Combined;
    2479             : 
    2480      160629 :   return SDValue();
    2481             : }
    2482             : 
    2483      321492 : SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
    2484             :                                        SDNode *N) {
    2485             :   // Iff the flag result is dead:
    2486             :   // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
    2487      320048 :   if ((N0.getOpcode() == ISD::ADD ||
    2488       16535 :        (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
    2489      335310 :       isNullConstant(N1) && !N->hasAnyUseOfValue(1))
    2490         464 :     return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
    2491         232 :                        N0.getOperand(0), N0.getOperand(1), CarryIn);
    2492             : 
    2493             :   /**
    2494             :    * When one of the addcarry argument is itself a carry, we may be facing
    2495             :    * a diamond carry propagation. In which case we try to transform the DAG
    2496             :    * to ensure linear carry propagation if that is possible.
    2497             :    *
    2498             :    * We are trying to get:
    2499             :    *   (addcarry X, 0, (addcarry A, B, Z):Carry)
    2500             :    */
    2501      321260 :   if (auto Y = getAsCarry(TLI, N1)) {
    2502             :     /**
    2503             :      *            (uaddo A, B)
    2504             :      *             /       \
    2505             :      *          Carry      Sum
    2506             :      *            |          \
    2507             :      *            | (addcarry *, 0, Z)
    2508             :      *            |       /
    2509             :      *             \   Carry
    2510             :      *              |   /
    2511             :      * (addcarry X, *, *)
    2512             :      */
    2513         621 :     if (Y.getOpcode() == ISD::UADDO &&
    2514         621 :         CarryIn.getResNo() == 1 &&
    2515          12 :         CarryIn.getOpcode() == ISD::ADDCARRY &&
    2516         900 :         isNullConstant(CarryIn.getOperand(1)) &&
    2517           4 :         CarryIn.getOperand(0) == Y.getValue(0)) {
    2518           2 :       auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
    2519             :                               Y.getOperand(0), Y.getOperand(1),
    2520           2 :                               CarryIn.getOperand(2));
    2521           2 :       AddToWorklist(NewY.getNode());
    2522           4 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
    2523           2 :                          DAG.getConstant(0, SDLoc(N), N0.getValueType()),
    2524           2 :                          NewY.getValue(1));
    2525             :     }
    2526             :   }
    2527             : 
    2528      321258 :   return SDValue();
    2529             : }
    2530             : 
    2531             : // Since it may not be valid to emit a fold to zero for vector initializers
    2532             : // check if we can before folding.
    2533          57 : static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
    2534             :                              SelectionDAG &DAG, bool LegalOperations,
    2535             :                              bool LegalTypes) {
    2536          57 :   if (!VT.isVector())
    2537          41 :     return DAG.getConstant(0, DL, VT);
    2538          16 :   if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
    2539          16 :     return DAG.getConstant(0, DL, VT);
    2540           0 :   return SDValue();
    2541             : }
    2542             : 
    2543      535947 : SDValue DAGCombiner::visitSUB(SDNode *N) {
    2544      535947 :   SDValue N0 = N->getOperand(0);
    2545      535947 :   SDValue N1 = N->getOperand(1);
    2546      535947 :   EVT VT = N0.getValueType();
    2547             :   SDLoc DL(N);
    2548             : 
    2549             :   // fold vector ops
    2550      535947 :   if (VT.isVector()) {
    2551      487497 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2552           4 :       return FoldedVOp;
    2553             : 
    2554             :     // fold (sub x, 0) -> x, vector edition
    2555      487493 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    2556           7 :       return N0;
    2557             :   }
    2558             : 
    2559             :   // fold (sub x, x) -> 0
    2560             :   // FIXME: Refactor this and xor and other similar operations together.
    2561      535936 :   if (N0 == N1)
    2562          38 :     return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
    2563      546102 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    2564       10204 :       DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
    2565             :     // fold (sub c1, c2) -> c1-c2
    2566           3 :     return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
    2567           3 :                                       N1.getNode());
    2568             :   }
    2569             : 
    2570      535895 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2571          11 :     return NewSel;
    2572             : 
    2573      535884 :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    2574             : 
    2575             :   // fold (sub x, c) -> (add x, -c)
    2576             :   if (N1C) {
    2577        7821 :     return DAG.getNode(ISD::ADD, DL, VT, N0,
    2578       23463 :                        DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
    2579             :   }
    2580             : 
    2581      528063 :   if (isNullConstantOrNullSplatConstant(N0)) {
    2582             :     unsigned BitWidth = VT.getScalarSizeInBits();
    2583             :     // Right-shifting everything out but the sign bit followed by negation is
    2584             :     // the same as flipping arithmetic/logical shift type without the negation:
    2585             :     // -(X >>u 31) -> (X >>s 31)
    2586             :     // -(X >>s 31) -> (X >>u 31)
    2587       10438 :     if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
    2588         115 :       ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
    2589         227 :       if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
    2590           7 :         auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
    2591           7 :         if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
    2592          14 :           return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
    2593             :       }
    2594             :     }
    2595             : 
    2596             :     // 0 - X --> 0 if the sub is NUW.
    2597        5212 :     if (N->getFlags().hasNoUnsignedWrap())
    2598           2 :       return N0;
    2599             : 
    2600       15630 :     if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
    2601             :       // N1 is either 0 or the minimum signed value. If the sub is NSW, then
    2602             :       // N1 must be 0 because negating the minimum signed value is undefined.
    2603           6 :       if (N->getFlags().hasNoSignedWrap())
    2604           2 :         return N0;
    2605             : 
    2606             :       // 0 - X --> X if X is 0 or the minimum signed value.
    2607           4 :       return N1;
    2608             :     }
    2609             :   }
    2610             : 
    2611             :   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
    2612      528048 :   if (isAllOnesConstantOrAllOnesSplatConstant(N0))
    2613         136 :     return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
    2614             : 
    2615             :   // fold (A - (0-B)) -> A+B
    2616     1056704 :   if (N1.getOpcode() == ISD::SUB &&
    2617         744 :       isNullConstantOrNullSplatConstant(N1.getOperand(0)))
    2618          87 :     return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
    2619             : 
    2620             :   // fold A-(A-B) -> B
    2621     1055902 :   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
    2622         212 :     return N1.getOperand(1);
    2623             : 
    2624             :   // fold (A+B)-A -> B
    2625      527739 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
    2626          22 :     return N0.getOperand(1);
    2627             : 
    2628             :   // fold (A+B)-B -> A
    2629      527717 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
    2630           3 :     return N0.getOperand(0);
    2631             : 
    2632             :   // fold C2-(A+C1) -> (C2-C1)-A
    2633      527714 :   if (N1.getOpcode() == ISD::ADD) {
    2634         570 :     SDValue N11 = N1.getOperand(1);
    2635         742 :     if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
    2636         172 :         isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
    2637         288 :       SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
    2638         432 :       return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
    2639             :     }
    2640             :   }
    2641             : 
    2642             :   // fold ((A+(B+or-C))-B) -> A+or-C
    2643        1803 :   if (N0.getOpcode() == ISD::ADD &&
    2644        1803 :       (N0.getOperand(1).getOpcode() == ISD::SUB ||
    2645      527570 :        N0.getOperand(1).getOpcode() == ISD::ADD) &&
    2646          46 :       N0.getOperand(1).getOperand(0) == N1)
    2647           7 :     return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
    2648           7 :                        N0.getOperand(1).getOperand(1));
    2649             : 
    2650             :   // fold ((A+(C+B))-B) -> A+C
    2651      527563 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
    2652           7 :       N0.getOperand(1).getOperand(1) == N1)
    2653           1 :     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
    2654           1 :                        N0.getOperand(1).getOperand(0));
    2655             : 
    2656             :   // fold ((A-(B-C))-C) -> A-B
    2657      527562 :   if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
    2658           3 :       N0.getOperand(1).getOperand(1) == N1)
    2659           3 :     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
    2660           3 :                        N0.getOperand(1).getOperand(0));
    2661             : 
    2662             :   // fold (A-(B-C)) -> A+(C-B)
    2663     1055118 :   if (N1.getOpcode() == ISD::SUB && N1.hasOneUse())
    2664         337 :     return DAG.getNode(ISD::ADD, DL, VT, N0,
    2665             :                        DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
    2666         674 :                                    N1.getOperand(0)));
    2667             : 
    2668             :   // fold (X - (-Y * Z)) -> (X + (Y * Z))
    2669     1054444 :   if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) {
    2670        6612 :     if (N1.getOperand(0).getOpcode() == ISD::SUB &&
    2671           9 :         isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0))) {
    2672           9 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
    2673           9 :                                 N1.getOperand(0).getOperand(1),
    2674          18 :                                 N1.getOperand(1));
    2675          18 :       return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
    2676             :     }
    2677        6576 :     if (N1.getOperand(1).getOpcode() == ISD::SUB &&
    2678           0 :         isNullConstantOrNullSplatConstant(N1.getOperand(1).getOperand(0))) {
    2679           0 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
    2680             :                                 N1.getOperand(0),
    2681           0 :                                 N1.getOperand(1).getOperand(1));
    2682           0 :       return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
    2683             :     }
    2684             :   }
    2685             : 
    2686             :   // If either operand of a sub is undef, the result is undef
    2687      527213 :   if (N0.isUndef())
    2688           0 :     return N0;
    2689     1054426 :   if (N1.isUndef())
    2690           0 :     return N1;
    2691             : 
    2692      527213 :   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
    2693           9 :     return V;
    2694             : 
    2695      527204 :   if (SDValue V = foldAddSubOfSignBit(N, DAG))
    2696           9 :     return V;
    2697             : 
    2698             :   // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
    2699      527195 :   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
    2700       30460 :     if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
    2701           2 :       SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
    2702           2 :       SDValue S0 = N1.getOperand(0);
    2703             :       if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
    2704             :         unsigned OpSizeInBits = VT.getScalarSizeInBits();
    2705           4 :         if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
    2706           4 :           if (C->getAPIntValue() == (OpSizeInBits - 1))
    2707           4 :             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
    2708             :       }
    2709             :     }
    2710             :   }
    2711             : 
    2712             :   // If the relocation model supports it, consider symbol offsets.
    2713             :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
    2714          11 :     if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
    2715             :       // fold (sub Sym, c) -> Sym-c
    2716             :       if (N1C && GA->getOpcode() == ISD::GlobalAddress)
    2717             :         return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
    2718             :                                     GA->getOffset() -
    2719             :                                         (uint64_t)N1C->getSExtValue());
    2720             :       // fold (sub Sym+c1, Sym+c2) -> c1-c2
    2721             :       if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
    2722           0 :         if (GA->getGlobal() == GB->getGlobal())
    2723           0 :           return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
    2724           0 :                                  DL, VT);
    2725             :     }
    2726             : 
    2727             :   // sub X, (sextinreg Y i1) -> add X, (and Y 1)
    2728     1054386 :   if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2729             :     VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    2730             :     if (TN->getVT() == MVT::i1) {
    2731          58 :       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
    2732          58 :                                  DAG.getConstant(1, DL, VT));
    2733         116 :       return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
    2734             :     }
    2735             :   }
    2736             : 
    2737             :   // Prefer an add for more folding potential and possibly better codegen:
    2738             :   // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
    2739      527135 :   if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
    2740          67 :     SDValue ShAmt = N1.getOperand(1);
    2741          67 :     ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
    2742         134 :     if (ShAmtC && ShAmtC->getZExtValue() == N1.getScalarValueSizeInBits() - 1) {
    2743          45 :       SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
    2744          30 :       return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
    2745             :     }
    2746             :   }
    2747             : 
    2748      527120 :   return SDValue();
    2749             : }
    2750             : 
    2751          89 : SDValue DAGCombiner::visitSUBC(SDNode *N) {
    2752          89 :   SDValue N0 = N->getOperand(0);
    2753          89 :   SDValue N1 = N->getOperand(1);
    2754          89 :   EVT VT = N0.getValueType();
    2755             :   SDLoc DL(N);
    2756             : 
    2757             :   // If the flag result is dead, turn this into an SUB.
    2758          89 :   if (!N->hasAnyUseOfValue(1))
    2759          24 :     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
    2760          72 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2761             : 
    2762             :   // fold (subc x, x) -> 0 + no borrow
    2763             :   if (N0 == N1)
    2764           0 :     return CombineTo(N, DAG.getConstant(0, DL, VT),
    2765           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2766             : 
    2767             :   // fold (subc x, 0) -> x + no borrow
    2768          65 :   if (isNullConstant(N1))
    2769           0 :     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2770             : 
    2771             :   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
    2772          65 :   if (isAllOnesConstant(N0))
    2773           0 :     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
    2774           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2775             : 
    2776          65 :   return SDValue();
    2777             : }
    2778             : 
    2779        1911 : SDValue DAGCombiner::visitUSUBO(SDNode *N) {
    2780        1911 :   SDValue N0 = N->getOperand(0);
    2781        1911 :   SDValue N1 = N->getOperand(1);
    2782        1911 :   EVT VT = N0.getValueType();
    2783        1911 :   if (VT.isVector())
    2784           0 :     return SDValue();
    2785             : 
    2786        3822 :   EVT CarryVT = N->getValueType(1);
    2787             :   SDLoc DL(N);
    2788             : 
    2789             :   // If the flag result is dead, turn this into an SUB.
    2790        1911 :   if (!N->hasAnyUseOfValue(1))
    2791         106 :     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
    2792         212 :                      DAG.getUNDEF(CarryVT));
    2793             : 
    2794             :   // fold (usubo x, x) -> 0 + no borrow
    2795             :   if (N0 == N1)
    2796           7 :     return CombineTo(N, DAG.getConstant(0, DL, VT),
    2797          14 :                      DAG.getConstant(0, DL, CarryVT));
    2798             : 
    2799             :   // fold (usubo x, 0) -> x + no borrow
    2800        1798 :   if (isNullConstant(N1))
    2801          26 :     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
    2802             : 
    2803             :   // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
    2804        1785 :   if (isAllOnesConstant(N0))
    2805           4 :     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
    2806           8 :                      DAG.getConstant(0, DL, CarryVT));
    2807             : 
    2808        1781 :   return SDValue();
    2809             : }
    2810             : 
    2811           0 : SDValue DAGCombiner::visitSUBE(SDNode *N) {
    2812           0 :   SDValue N0 = N->getOperand(0);
    2813           0 :   SDValue N1 = N->getOperand(1);
    2814           0 :   SDValue CarryIn = N->getOperand(2);
    2815             : 
    2816             :   // fold (sube x, y, false) -> (subc x, y)
    2817           0 :   if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    2818           0 :     return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
    2819             : 
    2820           0 :   return SDValue();
    2821             : }
    2822             : 
    2823         838 : SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
    2824         838 :   SDValue N0 = N->getOperand(0);
    2825         838 :   SDValue N1 = N->getOperand(1);
    2826         838 :   SDValue CarryIn = N->getOperand(2);
    2827             : 
    2828             :   // fold (subcarry x, y, false) -> (usubo x, y)
    2829         838 :   if (isNullConstant(CarryIn)) {
    2830         120 :     if (!LegalOperations ||
    2831         109 :         TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
    2832         240 :       return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
    2833             :   }
    2834             : 
    2835         718 :   return SDValue();
    2836             : }
    2837             : 
    2838       39002 : SDValue DAGCombiner::visitMUL(SDNode *N) {
    2839       39002 :   SDValue N0 = N->getOperand(0);
    2840       39002 :   SDValue N1 = N->getOperand(1);
    2841       78004 :   EVT VT = N0.getValueType();
    2842             : 
    2843             :   // fold (mul x, undef) -> 0
    2844       39002 :   if (N0.isUndef() || N1.isUndef())
    2845          56 :     return DAG.getConstant(0, SDLoc(N), VT);
    2846             : 
    2847             :   bool N0IsConst = false;
    2848             :   bool N1IsConst = false;
    2849             :   bool N1IsOpaqueConst = false;
    2850             :   bool N0IsOpaqueConst = false;
    2851             :   APInt ConstValue0, ConstValue1;
    2852             :   // fold vector ops
    2853       38974 :   if (VT.isVector()) {
    2854        9633 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2855          12 :       return FoldedVOp;
    2856             : 
    2857        9621 :     N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
    2858        9621 :     N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
    2859             :     assert((!N0IsConst ||
    2860             :             ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&
    2861             :            "Splat APInt should be element width");
    2862             :     assert((!N1IsConst ||
    2863             :             ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&
    2864             :            "Splat APInt should be element width");
    2865             :   } else {
    2866             :     N0IsConst = isa<ConstantSDNode>(N0);
    2867             :     if (N0IsConst) {
    2868          18 :       ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
    2869           9 :       N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
    2870             :     }
    2871             :     N1IsConst = isa<ConstantSDNode>(N1);
    2872             :     if (N1IsConst) {
    2873       35006 :       ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
    2874       17503 :       N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
    2875             :     }
    2876             :   }
    2877             : 
    2878             :   // fold (mul c1, c2) -> c1*c2
    2879       38962 :   if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
    2880           6 :     return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
    2881           6 :                                       N0.getNode(), N1.getNode());
    2882             : 
    2883             :   // canonicalize constant to RHS (vector doesn't have to splat)
    2884       38985 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    2885          29 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    2886          58 :     return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
    2887             :   // fold (mul x, 0) -> 0
    2888       57645 :   if (N1IsConst && ConstValue1.isNullValue())
    2889          28 :     return N1;
    2890             :   // fold (mul x, 1) -> x
    2891       57589 :   if (N1IsConst && ConstValue1.isOneValue())
    2892        2299 :     return N0;
    2893             : 
    2894       36600 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2895           3 :     return NewSel;
    2896             : 
    2897             :   // fold (mul x, -1) -> 0-x
    2898       52985 :   if (N1IsConst && ConstValue1.isAllOnesValue()) {
    2899             :     SDLoc DL(N);
    2900          99 :     return DAG.getNode(ISD::SUB, DL, VT,
    2901          99 :                        DAG.getConstant(0, DL, VT), N0);
    2902             :   }
    2903             :   // fold (mul x, (1 << c)) -> x << c
    2904       53445 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    2905       40654 :       DAG.isKnownToBeAPowerOfTwo(N1) &&
    2906         286 :       (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
    2907             :     SDLoc DL(N);
    2908        4156 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    2909        8312 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    2910        4156 :     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
    2911        8312 :     return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
    2912             :   }
    2913             :   // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
    2914       80898 :   if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
    2915          84 :     unsigned Log2Val = (-ConstValue1).logBase2();
    2916             :     SDLoc DL(N);
    2917             :     // FIXME: If the input is something that is easily negated (e.g. a
    2918             :     // single-use add), we should put the negate there.
    2919          84 :     return DAG.getNode(ISD::SUB, DL, VT,
    2920          84 :                        DAG.getConstant(0, DL, VT),
    2921             :                        DAG.getNode(ISD::SHL, DL, VT, N0,
    2922             :                             DAG.getConstant(Log2Val, DL,
    2923         168 :                                       getShiftAmountTy(N0.getValueType()))));
    2924             :   }
    2925             : 
    2926             :   // Try to transform multiply-by-(power-of-2 +/- 1) into shift and add/sub.
    2927             :   // Examples: x * 33 --> (x << 5) + x
    2928             :   //           x * 15 --> (x << 4) - x
    2929       32258 :   if (N1IsConst && TLI.decomposeMulByConstant(VT, N1)) {
    2930             :     // TODO: Negative constants can be handled by negating the result.
    2931             :     // TODO: We could handle more general decomposition of any constant by
    2932             :     //       having the target set a limit on number of ops and making a
    2933             :     //       callback to determine that sequence (similar to sqrt expansion).
    2934             :     unsigned MathOp = ISD::DELETED_NODE;
    2935          50 :     if ((ConstValue1 - 1).isPowerOf2())
    2936             :       MathOp = ISD::ADD;
    2937          36 :     else if ((ConstValue1 + 1).isPowerOf2())
    2938             :       MathOp = ISD::SUB;
    2939             : 
    2940             :     if (MathOp != ISD::DELETED_NODE) {
    2941         114 :       unsigned ShAmt = MathOp == ISD::ADD ? (ConstValue1 - 1).logBase2()
    2942         122 :                                           : (ConstValue1 + 1).logBase2();
    2943             :       assert(ShAmt > 0 && ShAmt < VT.getScalarSizeInBits() &&
    2944             :              "Not expecting multiply-by-constant that could have simplified");
    2945             :       SDLoc DL(N);
    2946          50 :       SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, N0,
    2947          50 :                                 DAG.getConstant(ShAmt, DL, VT));
    2948         100 :       return DAG.getNode(MathOp, DL, VT, Shl, N0);
    2949             :     }
    2950             :   }
    2951             : 
    2952             :   // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
    2953       32230 :   if (N0.getOpcode() == ISD::SHL &&
    2954       32213 :       isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
    2955          10 :       isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
    2956          10 :     SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
    2957           5 :     if (isConstantOrConstantVector(C3))
    2958          15 :       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
    2959             :   }
    2960             : 
    2961             :   // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
    2962             :   // use.
    2963             :   {
    2964             :     SDValue Sh(nullptr, 0), Y(nullptr, 0);
    2965             : 
    2966             :     // Check for both (mul (shl X, C), Y)  and  (mul Y, (shl X, C)).
    2967       32220 :     if (N0.getOpcode() == ISD::SHL &&
    2968       32203 :         isConstantOrConstantVector(N0.getOperand(1)) &&
    2969          15 :         N0.getNode()->hasOneUse()) {
    2970          12 :       Sh = N0; Y = N1;
    2971       32200 :     } else if (N1.getOpcode() == ISD::SHL &&
    2972       32191 :                isConstantOrConstantVector(N1.getOperand(1)) &&
    2973           6 :                N1.getNode()->hasOneUse()) {
    2974           3 :       Sh = N1; Y = N0;
    2975             :     }
    2976             : 
    2977       32203 :     if (Sh.getNode()) {
    2978          16 :       SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
    2979          31 :       return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
    2980             :     }
    2981             :   }
    2982             : 
    2983             :   // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
    2984       45071 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
    2985       13382 :       N0.getOpcode() == ISD::ADD &&
    2986       33375 :       DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
    2987         189 :       isMulAddWithConstProfitable(N, N0, N1))
    2988         180 :       return DAG.getNode(ISD::ADD, SDLoc(N), VT,
    2989          90 :                          DAG.getNode(ISD::MUL, SDLoc(N0), VT,
    2990          90 :                                      N0.getOperand(0), N1),
    2991          90 :                          DAG.getNode(ISD::MUL, SDLoc(N1), VT,
    2992         360 :                                      N0.getOperand(1), N1));
    2993             : 
    2994             :   // reassociate mul
    2995       40548 :   if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags()))
    2996         541 :     return RMUL;
    2997             : 
    2998       31557 :   return SDValue();
    2999             : }
    3000             : 
    3001             : /// Return true if divmod libcall is available.
    3002        1423 : static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
    3003             :                                      const TargetLowering &TLI) {
    3004             :   RTLIB::Libcall LC;
    3005        1423 :   EVT NodeType = Node->getValueType(0);
    3006        1423 :   if (!NodeType.isSimple())
    3007             :     return false;
    3008        1423 :   switch (NodeType.getSimpleVT().SimpleTy) {
    3009             :   default: return false; // No libcall for vector types.
    3010           0 :   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
    3011           4 :   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
    3012        1224 :   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
    3013         188 :   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
    3014           7 :   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
    3015             :   }
    3016             : 
    3017        1423 :   return TLI.getLibcallName(LC) != nullptr;
    3018             : }
    3019             : 
    3020             : /// Issue divrem if both quotient and remainder are needed.
    3021        5553 : SDValue DAGCombiner::useDivRem(SDNode *Node) {
    3022        5553 :   if (Node->use_empty())
    3023           0 :     return SDValue(); // This is a dead node, leave it alone.
    3024             : 
    3025        5553 :   unsigned Opcode = Node->getOpcode();
    3026        5553 :   bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
    3027        5553 :   unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3028             : 
    3029             :   // DivMod lib calls can still work on non-legal types if using lib-calls.
    3030       11106 :   EVT VT = Node->getValueType(0);
    3031        5553 :   if (VT.isVector() || !VT.isInteger())
    3032         459 :     return SDValue();
    3033             : 
    3034        5908 :   if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
    3035         783 :     return SDValue();
    3036             : 
    3037             :   // If DIVREM is going to get expanded into a libcall,
    3038             :   // but there is no libcall available, then don't combine.
    3039        4311 :   if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
    3040        1423 :       !isDivRemLibcallAvailable(Node, isSigned, TLI))
    3041        1301 :     return SDValue();
    3042             : 
    3043             :   // If div is legal, it's better to do the normal expansion
    3044             :   unsigned OtherOpcode = 0;
    3045        3010 :   if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
    3046        1562 :     OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
    3047        1562 :     if (TLI.isOperationLegalOrCustom(Opcode, VT))
    3048         211 :       return SDValue();
    3049             :   } else {
    3050        1448 :     OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
    3051        1448 :     if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
    3052         176 :       return SDValue();
    3053             :   }
    3054             : 
    3055        2623 :   SDValue Op0 = Node->getOperand(0);
    3056        2623 :   SDValue Op1 = Node->getOperand(1);
    3057             :   SDValue combined;
    3058        2623 :   for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
    3059        6856 :          UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
    3060             :     SDNode *User = *UI;
    3061        4233 :     if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
    3062        1610 :         User->use_empty())
    3063             :       continue;
    3064             :     // Convert the other matching node(s), too;
    3065             :     // otherwise, the DIVREM may get target-legalized into something
    3066             :     // target-specific that we won't be able to recognize.
    3067             :     unsigned UserOpc = User->getOpcode();
    3068         634 :     if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
    3069        1629 :         User->getOperand(0) == Op0 &&
    3070         841 :         User->getOperand(1) == Op1) {
    3071         169 :       if (!combined) {
    3072         169 :         if (UserOpc == OtherOpcode) {
    3073         169 :           SDVTList VTs = DAG.getVTList(VT, VT);
    3074         185 :           combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
    3075           0 :         } else if (UserOpc == DivRemOpc) {
    3076             :           combined = SDValue(User, 0);
    3077             :         } else {
    3078             :           assert(UserOpc == Opcode);
    3079             :           continue;
    3080             :         }
    3081             :       }
    3082         169 :       if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
    3083         139 :         CombineTo(User, combined);
    3084          30 :       else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
    3085          30 :         CombineTo(User, combined.getValue(1));
    3086             :     }
    3087             :   }
    3088        2623 :   return combined;
    3089             : }
    3090             : 
    3091       15062 : static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
    3092       15062 :   SDValue N0 = N->getOperand(0);
    3093       15062 :   SDValue N1 = N->getOperand(1);
    3094       30124 :   EVT VT = N->getValueType(0);
    3095             :   SDLoc DL(N);
    3096             : 
    3097       15062 :   unsigned Opc = N->getOpcode();
    3098       15062 :   bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
    3099             : 
    3100             :   // X / undef -> undef
    3101             :   // X % undef -> undef
    3102             :   // X / 0 -> undef
    3103             :   // X % 0 -> undef
    3104             :   // NOTE: This includes vectors where any divisor element is zero/undef.
    3105       30124 :   if (DAG.isUndef(Opc, {N0, N1}))
    3106           1 :     return DAG.getUNDEF(VT);
    3107             : 
    3108             :   // undef / X -> 0
    3109             :   // undef % X -> 0
    3110       15061 :   if (N0.isUndef())
    3111           0 :     return DAG.getConstant(0, DL, VT);
    3112             : 
    3113             :   // TODO: 0 / X -> 0
    3114             :   // TODO: 0 % X -> 0
    3115             : 
    3116             :   // X / X -> 1
    3117             :   // X % X -> 0
    3118             :   if (N0 == N1)
    3119          62 :     return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
    3120             : 
    3121             :   // TODO: X / 1 -> X
    3122             :   // TODO: X % 1 -> 0
    3123             :   // If this is a boolean op (single-bit element type), we can't have
    3124             :   // division-by-zero or remainder-by-zero, so assume the divisor is 1.
    3125             :   // Similarly, if we're zero-extending a boolean divisor, then assume it's a 1.
    3126             : 
    3127       15017 :   return SDValue();
    3128             : }
    3129             : 
    3130        5989 : SDValue DAGCombiner::visitSDIV(SDNode *N) {
    3131        5989 :   SDValue N0 = N->getOperand(0);
    3132        5989 :   SDValue N1 = N->getOperand(1);
    3133        5989 :   EVT VT = N->getValueType(0);
    3134        5989 :   EVT CCVT = getSetCCResultType(VT);
    3135             : 
    3136             :   // fold vector ops
    3137        5989 :   if (VT.isVector())
    3138         413 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3139           1 :       return FoldedVOp;
    3140             : 
    3141             :   SDLoc DL(N);
    3142             : 
    3143             :   // fold (sdiv c1, c2) -> c1/c2
    3144        5988 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3145        5988 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3146        5988 :   if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
    3147           0 :     return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
    3148             :   // fold (sdiv X, 1) -> X
    3149       10354 :   if (N1C && N1C->isOne())
    3150          14 :     return N0;
    3151             :   // fold (sdiv X, -1) -> 0-X
    3152       10326 :   if (N1C && N1C->isAllOnesValue())
    3153          28 :     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
    3154             :   // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
    3155        5960 :   if (N1C && N1C->getAPIntValue().isMinSignedValue())
    3156          28 :     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
    3157          14 :                          DAG.getConstant(1, DL, VT),
    3158          14 :                          DAG.getConstant(0, DL, VT));
    3159             : 
    3160        5946 :   if (SDValue V = simplifyDivRem(N, DAG))
    3161          15 :     return V;
    3162             : 
    3163        5931 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3164           7 :     return NewSel;
    3165             : 
    3166             :   // If we know the sign bits of both operands are zero, strength reduce to a
    3167             :   // udiv instead.  Handles (X&15) /s 4 -> X&15 >> 2
    3168        5924 :   if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    3169          50 :     return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
    3170             : 
    3171        5899 :   if (SDValue V = visitSDIVLike(N0, N1, N))
    3172        4354 :     return V;
    3173             : 
    3174             :   // sdiv, srem -> sdivrem
    3175             :   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
    3176             :   // true.  Otherwise, we break the simplification logic in visitREM().
    3177        1545 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3178        1545 :   if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    3179        1473 :     if (SDValue DivRem = useDivRem(N))
    3180          14 :         return DivRem;
    3181             : 
    3182        1531 :   return SDValue();
    3183             : }
    3184             : 
    3185        6263 : SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
    3186             :   SDLoc DL(N);
    3187        6263 :   EVT VT = N->getValueType(0);
    3188        6263 :   EVT CCVT = getSetCCResultType(VT);
    3189             :   unsigned BitWidth = VT.getScalarSizeInBits();
    3190             : 
    3191             :   // Helper for determining whether a value is a power-2 constant scalar or a
    3192             :   // vector of such elements.
    3193             :   auto IsPowerOfTwo = [](ConstantSDNode *C) {
    3194             :     if (C->isNullValue() || C->isOpaque())
    3195             :       return false;
    3196             :     if (C->getAPIntValue().isPowerOf2())
    3197             :       return true;
    3198             :     if ((-C->getAPIntValue()).isPowerOf2())
    3199             :       return true;
    3200             :     return false;
    3201             :   };
    3202             : 
    3203             :   // fold (sdiv X, pow2) -> simple ops after legalize
    3204             :   // FIXME: We check for the exact bit here because the generic lowering gives
    3205             :   // better results in that case. The target-specific lowering should learn how
    3206             :   // to handle exact sdivs efficiently.
    3207        8874 :   if (!N->getFlags().hasExact() && ISD::matchUnaryPredicate(N1, IsPowerOfTwo)) {
    3208             :     // Target-specific implementation of sdiv x, pow2.
    3209         601 :     if (SDValue Res = BuildSDIVPow2(N))
    3210          32 :       return Res;
    3211             : 
    3212             :     // Create constants that are functions of the shift amount value.
    3213        1138 :     EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
    3214         569 :     SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
    3215        1138 :     SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
    3216         569 :     C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
    3217        1138 :     SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
    3218         569 :     if (!isConstantOrConstantVector(Inexact))
    3219           0 :       return SDValue();
    3220             : 
    3221             :     // Splat the sign bit into the register
    3222         569 :     SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
    3223         569 :                                DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
    3224         569 :     AddToWorklist(Sign.getNode());
    3225             : 
    3226             :     // Add (N0 < 0) ? abs2 - 1 : 0;
    3227        1138 :     SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
    3228         569 :     AddToWorklist(Srl.getNode());
    3229        1138 :     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
    3230         569 :     AddToWorklist(Add.getNode());
    3231        1138 :     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
    3232         569 :     AddToWorklist(Sra.getNode());
    3233             : 
    3234             :     // Special case: (sdiv X, 1) -> X
    3235             :     // Special Case: (sdiv X, -1) -> 0-X
    3236         569 :     SDValue One = DAG.getConstant(1, DL, VT);
    3237         569 :     SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
    3238         569 :     SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
    3239         569 :     SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
    3240        1138 :     SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
    3241         569 :     Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
    3242             : 
    3243             :     // If dividing by a positive value, we're done. Otherwise, the result must
    3244             :     // be negated.
    3245         569 :     SDValue Zero = DAG.getConstant(0, DL, VT);
    3246        1138 :     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
    3247             : 
    3248             :     // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
    3249         569 :     SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
    3250         569 :     SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
    3251         569 :     return Res;
    3252             :   }
    3253             : 
    3254             :   // If integer divide is expensive and we satisfy the requirements, emit an
    3255             :   // alternate sequence.  Targets may check function attributes for size/speed
    3256             :   // trade-offs.
    3257        5662 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3258        9895 :   if (isConstantOrConstantVector(N1) &&
    3259        8466 :       !TLI.isIntDivCheap(N->getValueType(0), Attr))
    3260        4189 :     if (SDValue Op = BuildSDIV(N))
    3261        4059 :       return Op;
    3262             : 
    3263        1603 :   return SDValue();
    3264             : }
    3265             : 
    3266        4417 : SDValue DAGCombiner::visitUDIV(SDNode *N) {
    3267        4417 :   SDValue N0 = N->getOperand(0);
    3268        4417 :   SDValue N1 = N->getOperand(1);
    3269        4417 :   EVT VT = N->getValueType(0);
    3270        4417 :   EVT CCVT = getSetCCResultType(VT);
    3271             : 
    3272             :   // fold vector ops
    3273        4417 :   if (VT.isVector())
    3274         258 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3275           0 :       return FoldedVOp;
    3276             : 
    3277             :   SDLoc DL(N);
    3278             : 
    3279             :   // fold (udiv c1, c2) -> c1/c2
    3280        4417 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3281        4417 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3282        4417 :   if (N0C && N1C)
    3283           0 :     if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
    3284           0 :                                                     N0C, N1C))
    3285           0 :       return Folded;
    3286             :   // fold (udiv X, 1) -> X
    3287        7238 :   if (N1C && N1C->isOne())
    3288          10 :     return N0;
    3289             :   // fold (udiv X, -1) -> select(X == -1, 1, 0)
    3290        7218 :   if (N1C && N1C->getAPIntValue().isAllOnesValue())
    3291          22 :     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
    3292          11 :                          DAG.getConstant(1, DL, VT),
    3293          11 :                          DAG.getConstant(0, DL, VT));
    3294             : 
    3295        4396 :   if (SDValue V = simplifyDivRem(N, DAG))
    3296          11 :     return V;
    3297             : 
    3298        4385 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3299           7 :     return NewSel;
    3300             : 
    3301        4378 :   if (SDValue V = visitUDIVLike(N0, N1, N))
    3302        2743 :     return V;
    3303             : 
    3304             :   // sdiv, srem -> sdivrem
    3305             :   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
    3306             :   // true.  Otherwise, we break the simplification logic in visitREM().
    3307        1635 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3308        1635 :   if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    3309        1571 :     if (SDValue DivRem = useDivRem(N))
    3310          16 :         return DivRem;
    3311             : 
    3312        1619 :   return SDValue();
    3313             : }
    3314             : 
    3315        4912 : SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
    3316             :   SDLoc DL(N);
    3317        4912 :   EVT VT = N->getValueType(0);
    3318             : 
    3319             :   // fold (udiv x, (1 << c)) -> x >>u c
    3320        8273 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    3321        3361 :       DAG.isKnownToBeAPowerOfTwo(N1)) {
    3322        2377 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    3323        2377 :     AddToWorklist(LogBase2.getNode());
    3324             : 
    3325        4754 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    3326        2377 :     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
    3327        2377 :     AddToWorklist(Trunc.getNode());
    3328        4754 :     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    3329             :   }
    3330             : 
    3331             :   // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
    3332        2535 :   if (N1.getOpcode() == ISD::SHL) {
    3333          15 :     SDValue N10 = N1.getOperand(0);
    3334          30 :     if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
    3335          15 :         DAG.isKnownToBeAPowerOfTwo(N10)) {
    3336          15 :       SDValue LogBase2 = BuildLogBase2(N10, DL);
    3337          15 :       AddToWorklist(LogBase2.getNode());
    3338             : 
    3339          15 :       EVT ADDVT = N1.getOperand(1).getValueType();
    3340          15 :       SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
    3341          15 :       AddToWorklist(Trunc.getNode());
    3342          30 :       SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
    3343          15 :       AddToWorklist(Add.getNode());
    3344          30 :       return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
    3345             :     }
    3346             :   }
    3347             : 
    3348             :   // fold (udiv x, c) -> alternate
    3349        2520 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3350        3512 :   if (isConstantOrConstantVector(N1) &&
    3351        1984 :       !TLI.isIntDivCheap(N->getValueType(0), Attr))
    3352         949 :     if (SDValue Op = BuildUDIV(N))
    3353         804 :       return Op;
    3354             : 
    3355        1716 :   return SDValue();
    3356             : }
    3357             : 
    3358             : // handles ISD::SREM and ISD::UREM
    3359        4731 : SDValue DAGCombiner::visitREM(SDNode *N) {
    3360        4731 :   unsigned Opcode = N->getOpcode();
    3361        4731 :   SDValue N0 = N->getOperand(0);
    3362        4731 :   SDValue N1 = N->getOperand(1);
    3363        4731 :   EVT VT = N->getValueType(0);
    3364        4731 :   EVT CCVT = getSetCCResultType(VT);
    3365             : 
    3366             :   bool isSigned = (Opcode == ISD::SREM);
    3367             :   SDLoc DL(N);
    3368             : 
    3369             :   // fold (rem c1, c2) -> c1%c2
    3370        4731 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3371        4731 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3372        4731 :   if (N0C && N1C)
    3373           2 :     if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
    3374           2 :       return Folded;
    3375             :   // fold (urem X, -1) -> select(X == -1, 0, x)
    3376        6655 :   if (!isSigned && N1C && N1C->getAPIntValue().isAllOnesValue())
    3377          18 :     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
    3378           9 :                          DAG.getConstant(0, DL, VT), N0);
    3379             : 
    3380        4720 :   if (SDValue V = simplifyDivRem(N, DAG))
    3381          19 :     return V;
    3382             : 
    3383        4701 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3384          14 :     return NewSel;
    3385             : 
    3386        4687 :   if (isSigned) {
    3387             :     // If we know the sign bits of both operands are zero, strength reduce to a
    3388             :     // urem instead.  Handles (X & 0x0FFFFFFF) %s 16 -> X&15
    3389        1553 :     if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    3390          40 :       return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
    3391             :   } else {
    3392        3134 :     SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
    3393        3134 :     if (DAG.isKnownToBeAPowerOfTwo(N1)) {
    3394             :       // fold (urem x, pow2) -> (and x, pow2-1)
    3395        2786 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
    3396        1393 :       AddToWorklist(Add.getNode());
    3397        2786 :       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    3398             :     }
    3399        1747 :     if (N1.getOpcode() == ISD::SHL &&
    3400          12 :         DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
    3401             :       // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
    3402          12 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
    3403           6 :       AddToWorklist(Add.getNode());
    3404          12 :       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    3405             :     }
    3406             :   }
    3407             : 
    3408        3268 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3409             : 
    3410             :   // If X/C can be simplified by the division-by-constant logic, lower
    3411             :   // X%C to the equivalent of X-X/C*C.
    3412             :   // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
    3413             :   // speculative DIV must not cause a DIVREM conversion.  We guard against this
    3414             :   // by skipping the simplification if isIntDivCheap().  When div is not cheap,
    3415             :   // combine will not return a DIVREM.  Regardless, checking cheapness here
    3416             :   // makes sense since the simplification results in fatter code.
    3417        3268 :   if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
    3418             :     SDValue OptimizedDiv =
    3419         898 :         isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
    3420         898 :     if (OptimizedDiv.getNode() && OptimizedDiv.getOpcode() != ISD::UDIVREM &&
    3421             :         OptimizedDiv.getOpcode() != ISD::SDIVREM) {
    3422        1518 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
    3423        1518 :       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
    3424         759 :       AddToWorklist(OptimizedDiv.getNode());
    3425         759 :       AddToWorklist(Mul.getNode());
    3426         759 :       return Sub;
    3427             :     }
    3428             :   }
    3429             : 
    3430             :   // sdiv, srem -> sdivrem
    3431        2509 :   if (SDValue DivRem = useDivRem(N))
    3432         139 :     return DivRem.getValue(1);
    3433             : 
    3434        2370 :   return SDValue();
    3435             : }
    3436             : 
    3437        1447 : SDValue DAGCombiner::visitMULHS(SDNode *N) {
    3438        1447 :   SDValue N0 = N->getOperand(0);
    3439        1447 :   SDValue N1 = N->getOperand(1);
    3440        2894 :   EVT VT = N->getValueType(0);
    3441             :   SDLoc DL(N);
    3442             : 
    3443        1447 :   if (VT.isVector()) {
    3444             :     // fold (mulhs x, 0) -> 0
    3445         742 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    3446           0 :       return N1;
    3447         742 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    3448           2 :       return N0;
    3449             :   }
    3450             : 
    3451             :   // fold (mulhs x, 0) -> 0
    3452        1445 :   if (isNullConstant(N1))
    3453           0 :     return N1;
    3454             :   // fold (mulhs x, 1) -> (sra x, size(x)-1)
    3455        1445 :   if (isOneConstant(N1))
    3456           0 :     return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
    3457           0 :                        DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
    3458           0 :                                        getShiftAmountTy(N0.getValueType())));
    3459             : 
    3460             :   // fold (mulhs x, undef) -> 0
    3461        2890 :   if (N0.isUndef() || N1.isUndef())
    3462           0 :     return DAG.getConstant(0, DL, VT);
    3463             : 
    3464             :   // If the type twice as wide is legal, transform the mulhs to a wider multiply
    3465             :   // plus a shift.
    3466        1445 :   if (VT.isSimple() && !VT.isVector()) {
    3467         705 :     MVT Simple = VT.getSimpleVT();
    3468         705 :     unsigned SimpleSize = Simple.getSizeInBits();
    3469         705 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3470         705 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3471         138 :       N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
    3472         138 :       N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
    3473         138 :       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
    3474          69 :       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
    3475             :             DAG.getConstant(SimpleSize, DL,
    3476          69 :                             getShiftAmountTy(N1.getValueType())));
    3477         138 :       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    3478             :     }
    3479             :   }
    3480             : 
    3481        1376 :   return SDValue();
    3482             : }
    3483             : 
    3484        6323 : SDValue DAGCombiner::visitMULHU(SDNode *N) {
    3485        6323 :   SDValue N0 = N->getOperand(0);
    3486        6323 :   SDValue N1 = N->getOperand(1);
    3487       12646 :   EVT VT = N->getValueType(0);
    3488             :   SDLoc DL(N);
    3489             : 
    3490        6323 :   if (VT.isVector()) {
    3491             :     // fold (mulhu x, 0) -> 0
    3492        1177 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    3493           0 :       return N1;
    3494        1177 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    3495           0 :       return N0;
    3496             :   }
    3497             : 
    3498             :   // fold (mulhu x, 0) -> 0
    3499        6323 :   if (isNullConstant(N1))
    3500           0 :     return N1;
    3501             :   // fold (mulhu x, 1) -> 0
    3502        6323 :   if (isOneConstant(N1))
    3503           0 :     return DAG.getConstant(0, DL, N0.getValueType());
    3504             :   // fold (mulhu x, undef) -> 0
    3505        6323 :   if (N0.isUndef() || N1.isUndef())
    3506           0 :     return DAG.getConstant(0, DL, VT);
    3507             : 
    3508             :   // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
    3509        8235 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    3510        6461 :       DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
    3511             :     SDLoc DL(N);
    3512             :     unsigned NumEltBits = VT.getScalarSizeInBits();
    3513          79 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    3514          79 :     SDValue SRLAmt = DAG.getNode(
    3515          79 :         ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
    3516          79 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    3517          79 :     SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
    3518         158 :     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    3519             :   }
    3520             : 
    3521             :   // If the type twice as wide is legal, transform the mulhu to a wider multiply
    3522             :   // plus a shift.
    3523        6244 :   if (VT.isSimple() && !VT.isVector()) {
    3524        5146 :     MVT Simple = VT.getSimpleVT();
    3525        5146 :     unsigned SimpleSize = Simple.getSizeInBits();
    3526        5146 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3527        5146 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3528         472 :       N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
    3529         472 :       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
    3530         472 :       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
    3531         236 :       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
    3532             :             DAG.getConstant(SimpleSize, DL,
    3533         236 :                             getShiftAmountTy(N1.getValueType())));
    3534         472 :       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    3535             :     }
    3536             :   }
    3537             : 
    3538        6008 :   return SDValue();
    3539             : }
    3540             : 
    3541             : /// Perform optimizations common to nodes that compute two values. LoOp and HiOp
    3542             : /// give the opcodes for the two computations that are being performed. Return
    3543             : /// true if a simplification was made.
    3544        6330 : SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
    3545             :                                                 unsigned HiOp) {
    3546             :   // If the high half is not needed, just compute the low half.
    3547        6330 :   bool HiExists = N->hasAnyUseOfValue(1);
    3548        6330 :   if (!HiExists &&
    3549          42 :       (!LegalOperations ||
    3550          26 :        TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
    3551          48 :     SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    3552             :     return CombineTo(N, Res, Res);
    3553             :   }
    3554             : 
    3555             :   // If the low half is not needed, just compute the high half.
    3556        6314 :   bool LoExists = N->hasAnyUseOfValue(0);
    3557        6314 :   if (!LoExists &&
    3558        1402 :       (!LegalOperations ||
    3559         715 :        TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
    3560        2250 :     SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    3561             :     return CombineTo(N, Res, Res);
    3562             :   }
    3563             : 
    3564             :   // If both halves are used, return as it is.
    3565        5627 :   if (LoExists && HiExists)
    3566        4899 :     return SDValue();
    3567             : 
    3568             :   // If the two computed results can be simplified separately, separate them.
    3569         728 :   if (LoExists) {
    3570          26 :     SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    3571          13 :     AddToWorklist(Lo.getNode());
    3572          13 :     SDValue LoOpt = combine(Lo.getNode());
    3573          13 :     if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
    3574           3 :         (!LegalOperations ||
    3575           3 :          TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
    3576           3 :       return CombineTo(N, LoOpt, LoOpt);
    3577             :   }
    3578             : 
    3579         725 :   if (HiExists) {
    3580        1577 :     SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    3581         715 :     AddToWorklist(Hi.getNode());
    3582         715 :     SDValue HiOpt = combine(Hi.getNode());
    3583         715 :     if (HiOpt.getNode() && HiOpt != Hi &&
    3584           0 :         (!LegalOperations ||
    3585           0 :          TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
    3586           0 :       return CombineTo(N, HiOpt, HiOpt);
    3587             :   }
    3588             : 
    3589         725 :   return SDValue();
    3590             : }
    3591             : 
    3592         726 : SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
    3593         726 :   if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
    3594         182 :     return Res;
    3595             : 
    3596        1088 :   EVT VT = N->getValueType(0);
    3597             :   SDLoc DL(N);
    3598             : 
    3599             :   // If the type is twice as wide is legal, transform the mulhu to a wider
    3600             :   // multiply plus a shift.
    3601         544 :   if (VT.isSimple() && !VT.isVector()) {
    3602         544 :     MVT Simple = VT.getSimpleVT();
    3603         544 :     unsigned SimpleSize = Simple.getSizeInBits();
    3604         544 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3605         544 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3606           0 :       SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
    3607           0 :       SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
    3608           0 :       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
    3609             :       // Compute the high part as N1.
    3610           0 :       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
    3611             :             DAG.getConstant(SimpleSize, DL,
    3612           0 :                             getShiftAmountTy(Lo.getValueType())));
    3613           0 :       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
    3614             :       // Compute the low part as N0.
    3615           0 :       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
    3616             :       return CombineTo(N, Lo, Hi);
    3617             :     }
    3618             :   }
    3619             : 
    3620         544 :   return SDValue();
    3621             : }
    3622             : 
    3623        5604 : SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
    3624        5604 :   if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
    3625         524 :     return Res;
    3626             : 
    3627       10160 :   EVT VT = N->getValueType(0);
    3628             :   SDLoc DL(N);
    3629             : 
    3630             :   // If the type is twice as wide is legal, transform the mulhu to a wider
    3631             :   // multiply plus a shift.
    3632        5080 :   if (VT.isSimple() && !VT.isVector()) {
    3633        5080 :     MVT Simple = VT.getSimpleVT();
    3634        5080 :     unsigned SimpleSize = Simple.getSizeInBits();
    3635        5080 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3636        5080 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3637           0 :       SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
    3638           0 :       SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
    3639           0 :       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
    3640             :       // Compute the high part as N1.
    3641           0 :       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
    3642             :             DAG.getConstant(SimpleSize, DL,
    3643           0 :                             getShiftAmountTy(Lo.getValueType())));
    3644           0 :       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
    3645             :       // Compute the low part as N0.
    3646           0 :       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
    3647             :       return CombineTo(N, Lo, Hi);
    3648             :     }
    3649             :   }
    3650             : 
    3651        5080 :   return SDValue();
    3652             : }
    3653             : 
    3654           0 : SDValue DAGCombiner::visitSMULO(SDNode *N) {
    3655             :   // (smulo x, 2) -> (saddo x, x)
    3656           0 :   if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    3657           0 :     if (C2->getAPIntValue() == 2)
    3658           0 :       return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
    3659           0 :                          N->getOperand(0), N->getOperand(0));
    3660             : 
    3661           0 :   return SDValue();
    3662             : }
    3663             : 
    3664           0 : SDValue DAGCombiner::visitUMULO(SDNode *N) {
    3665             :   // (umulo x, 2) -> (uaddo x, x)
    3666           0 :   if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    3667           0 :     if (C2->getAPIntValue() == 2)
    3668           0 :       return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
    3669           0 :                          N->getOperand(0), N->getOperand(0));
    3670             : 
    3671           0 :   return SDValue();
    3672             : }
    3673             : 
    3674       25227 : SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
    3675       25227 :   SDValue N0 = N->getOperand(0);
    3676       25227 :   SDValue N1 = N->getOperand(1);
    3677       25227 :   EVT VT = N0.getValueType();
    3678             : 
    3679             :   // fold vector ops
    3680       25227 :   if (VT.isVector())
    3681       20083 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3682         392 :       return FoldedVOp;
    3683             : 
    3684             :   // fold operation with constant operands.
    3685             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    3686             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    3687       24835 :   if (N0C && N1C)
    3688           0 :     return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
    3689             : 
    3690             :   // canonicalize constant to RHS
    3691       24845 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    3692          10 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    3693          20 :     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
    3694             : 
    3695             :   // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
    3696             :   // Only do this if the current op isn't legal and the flipped is.
    3697       24825 :   unsigned Opcode = N->getOpcode();
    3698       24825 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    3699        5455 :   if (!TLI.isOperationLegal(Opcode, VT) &&
    3700        5455 :       (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
    3701          52 :       (N1.isUndef() || DAG.SignBitIsZero(N1))) {
    3702             :     unsigned AltOpcode;
    3703             :     switch (Opcode) {
    3704             :     case ISD::SMIN: AltOpcode = ISD::UMIN; break;
    3705             :     case ISD::SMAX: AltOpcode = ISD::UMAX; break;
    3706             :     case ISD::UMIN: AltOpcode = ISD::SMIN; break;
    3707             :     case ISD::UMAX: AltOpcode = ISD::SMAX; break;
    3708           0 :     default: llvm_unreachable("Unknown MINMAX opcode");
    3709             :     }
    3710             :     if (TLI.isOperationLegal(AltOpcode, VT))
    3711          10 :       return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
    3712             :   }
    3713             : 
    3714       24820 :   return SDValue();
    3715             : }
    3716             : 
    3717             : /// If this is a binary operator with two operands of the same opcode, try to
    3718             : /// simplify it.
    3719       61780 : SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
    3720       61780 :   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
    3721       61780 :   EVT VT = N0.getValueType();
    3722             :   assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
    3723             : 
    3724             :   // Bail early if none of these transforms apply.
    3725       61780 :   if (N0.getNumOperands() == 0) return SDValue();
    3726             : 
    3727             :   // For each of OP in AND/OR/XOR:
    3728             :   // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
    3729             :   // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
    3730             :   // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
    3731             :   // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
    3732             :   // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
    3733             :   //
    3734             :   // do not sink logical op inside of a vector extend, since it may combine
    3735             :   // into a vsetcc.
    3736      123524 :   EVT Op0VT = N0.getOperand(0).getValueType();
    3737       61596 :   if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
    3738       61518 :        N0.getOpcode() == ISD::SIGN_EXTEND ||
    3739       61518 :        N0.getOpcode() == ISD::BSWAP ||
    3740             :        // Avoid infinite looping with PromoteIntBinOp.
    3741        1255 :        (N0.getOpcode() == ISD::ANY_EXTEND &&
    3742       61518 :         (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
    3743        1077 :        (N0.getOpcode() == ISD::TRUNCATE &&
    3744        1358 :         (!TLI.isZExtFree(VT, Op0VT) ||
    3745         281 :          !TLI.isTruncateFree(Op0VT, VT)) &&
    3746        1828 :         TLI.isTypeLegal(Op0VT))) &&
    3747           9 :       !VT.isVector() &&
    3748       62626 :       Op0VT == N1.getOperand(0).getValueType() &&
    3749         842 :       (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
    3750         842 :     SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
    3751             :                                  N0.getOperand(0).getValueType(),
    3752         860 :                                  N0.getOperand(0), N1.getOperand(0));
    3753         842 :     AddToWorklist(ORNode.getNode());
    3754        1697 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
    3755             :   }
    3756             : 
    3757             :   // For each of OP in SHL/SRL/SRA/AND...
    3758             :   //   fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
    3759             :   //   fold (or  (OP x, z), (OP y, z)) -> (OP (or  x, y), z)
    3760             :   //   fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
    3761       60670 :   if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
    3762      121305 :        N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
    3763        7734 :       N0.getOperand(1) == N1.getOperand(1)) {
    3764          98 :     SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
    3765             :                                  N0.getOperand(0).getValueType(),
    3766         102 :                                  N0.getOperand(0), N1.getOperand(0));
    3767          98 :     AddToWorklist(ORNode.getNode());
    3768          98 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
    3769         196 :                        ORNode, N0.getOperand(1));
    3770             :   }
    3771             : 
    3772             :   // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
    3773             :   // Only perform this optimization up until type legalization, before
    3774             :   // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
    3775             :   // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
    3776             :   // we don't want to undo this promotion.
    3777             :   // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
    3778             :   // on scalars.
    3779       33677 :   if ((N0.getOpcode() == ISD::BITCAST ||
    3780       60822 :        N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
    3781       27149 :        Level <= AfterLegalizeTypes) {
    3782         697 :     SDValue In0 = N0.getOperand(0);
    3783         697 :     SDValue In1 = N1.getOperand(0);
    3784         697 :     EVT In0Ty = In0.getValueType();
    3785         697 :     EVT In1Ty = In1.getValueType();
    3786             :     SDLoc DL(N);
    3787             :     // If both incoming values are integers, and the original types are the
    3788             :     // same.
    3789         727 :     if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
    3790         354 :       SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
    3791         236 :       SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
    3792         118 :       AddToWorklist(Op.getNode());
    3793         118 :       return BC;
    3794             :     }
    3795             :   }
    3796             : 
    3797             :   // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
    3798             :   // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
    3799             :   // If both shuffles use the same mask, and both shuffle within a single
    3800             :   // vector, then it is worthwhile to move the swizzle after the operation.
    3801             :   // The type-legalizer generates this pattern when loading illegal
    3802             :   // vector types from memory. In many cases this allows additional shuffle
    3803             :   // optimizations.
    3804             :   // There are other cases where moving the shuffle after the xor/and/or
    3805             :   // is profitable even if shuffles don't perform a swizzle.
    3806             :   // If both shuffles use the same mask, and both shuffles have the same first
    3807             :   // or second operand, then it might still be profitable to move the shuffle
    3808             :   // after the xor/and/or operation.
    3809       60704 :   if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
    3810             :     ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
    3811             :     ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
    3812             : 
    3813             :     assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
    3814             :            "Inputs to shuffles are not the same type");
    3815             : 
    3816             :     // Check that both shuffles use the same mask. The masks are known to be of
    3817             :     // the same length because the result vector type is the same.
    3818             :     // Check also that shuffles have only one use to avoid introducing extra
    3819             :     // instructions.
    3820         161 :     if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
    3821         216 :         SVN0->getMask().equals(SVN1->getMask())) {
    3822         143 :       SDValue ShOp = N0->getOperand(1);
    3823             : 
    3824             :       // Don't try to fold this node if it requires introducing a
    3825             :       // build vector of all zeros that might be illegal at this stage.
    3826         143 :       if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
    3827          26 :         if (!LegalTypes)
    3828          24 :           ShOp = DAG.getConstant(0, SDLoc(N), VT);
    3829             :         else
    3830           2 :           ShOp = SDValue();
    3831             :       }
    3832             : 
    3833             :       // (AND (shuf (A, C), shuf (B, C))) -> shuf (AND (A, B), C)
    3834             :       // (OR  (shuf (A, C), shuf (B, C))) -> shuf (OR  (A, B), C)
    3835             :       // (XOR (shuf (A, C), shuf (B, C))) -> shuf (XOR (A, B), V_0)
    3836         217 :       if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
    3837          74 :         SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
    3838         148 :                                       N0->getOperand(0), N1->getOperand(0));
    3839          74 :         AddToWorklist(NewNode.getNode());
    3840          74 :         return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
    3841         148 :                                     SVN0->getMask());
    3842             :       }
    3843             : 
    3844             :       // Don't try to fold this node if it requires introducing a
    3845             :       // build vector of all zeros that might be illegal at this stage.
    3846          69 :       ShOp = N0->getOperand(0);
    3847          69 :       if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
    3848          14 :         if (!LegalTypes)
    3849          12 :           ShOp = DAG.getConstant(0, SDLoc(N), VT);
    3850             :         else
    3851           2 :           ShOp = SDValue();
    3852             :       }
    3853             : 
    3854             :       // (AND (shuf (C, A), shuf (C, B))) -> shuf (C, AND (A, B))
    3855             :       // (OR  (shuf (C, A), shuf (C, B))) -> shuf (C, OR  (A, B))
    3856             :       // (XOR (shuf (C, A), shuf (C, B))) -> shuf (V_0, XOR (A, B))
    3857         105 :       if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
    3858          36 :         SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
    3859          72 :                                       N0->getOperand(1), N1->getOperand(1));
    3860          36 :         AddToWorklist(NewNode.getNode());
    3861          36 :         return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
    3862          72 :                                     SVN0->getMask());
    3863             :       }
    3864             :     }
    3865             :   }
    3866             : 
    3867       60594 :   return SDValue();
    3868             : }
    3869             : 
    3870             : /// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
    3871      507307 : SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
    3872             :                                        const SDLoc &DL) {
    3873      507307 :   SDValue LL, LR, RL, RR, N0CC, N1CC;
    3874      595533 :   if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
    3875       88226 :       !isSetCCEquivalent(N1, RL, RR, N1CC))
    3876      504359 :     return SDValue();
    3877             : 
    3878             :   assert(N0.getValueType() == N1.getValueType() &&
    3879             :          "Unexpected operand types for bitwise logic op");
    3880             :   assert(LL.getValueType() == LR.getValueType() &&
    3881             :          RL.getValueType() == RR.getValueType() &&
    3882             :          "Unexpected operand types for setcc");
    3883             : 
    3884             :   // If we're here post-legalization or the logic op type is not i1, the logic
    3885             :   // op type must match a setcc result type. Also, all folds require new
    3886             :   // operations on the left and right operands, so those types must match.
    3887        2948 :   EVT VT = N0.getValueType();
    3888        2948 :   EVT OpVT = LL.getValueType();
    3889        2948 :   if (LegalOperations || VT.getScalarType() != MVT::i1)
    3890        1495 :     if (VT != getSetCCResultType(OpVT))
    3891          24 :       return SDValue();
    3892        5856 :   if (OpVT != RL.getValueType())
    3893         109 :     return SDValue();
    3894             : 
    3895        2815 :   ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
    3896        2815 :   ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
    3897        2815 :   bool IsInteger = OpVT.isInteger();
    3898        3324 :   if (LR == RR && CC0 == CC1 && IsInteger) {
    3899         271 :     bool IsZero = isNullConstantOrNullSplatConstant(LR);
    3900         271 :     bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
    3901             : 
    3902             :     // All bits clear?
    3903         271 :     bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
    3904             :     // All sign bits clear?
    3905         271 :     bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
    3906             :     // Any bits set?
    3907         271 :     bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
    3908             :     // Any sign bits set?
    3909         271 :     bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
    3910             : 
    3911             :     // (and (seteq X,  0), (seteq Y,  0)) --> (seteq (or X, Y),  0)
    3912             :     // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
    3913             :     // (or  (setne X,  0), (setne Y,  0)) --> (setne (or X, Y),  0)
    3914             :     // (or  (setlt X,  0), (setlt Y,  0)) --> (setlt (or X, Y),  0)
    3915         271 :     if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
    3916          72 :       SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
    3917          69 :       AddToWorklist(Or.getNode());
    3918          69 :       return DAG.getSetCC(DL, VT, Or, LR, CC1);
    3919             :     }
    3920             : 
    3921             :     // All bits set?
    3922         202 :     bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
    3923             :     // All sign bits set?
    3924         202 :     bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
    3925             :     // Any bits clear?
    3926         202 :     bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
    3927             :     // Any sign bits clear?
    3928         202 :     bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
    3929             : 
    3930             :     // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
    3931             :     // (and (setlt X,  0), (setlt Y,  0)) --> (setlt (and X, Y),  0)
    3932             :     // (or  (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
    3933             :     // (or  (setgt X, -1), (setgt Y  -1)) --> (setgt (and X, Y), -1)
    3934         202 :     if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
    3935          22 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
    3936          22 :       AddToWorklist(And.getNode());
    3937          22 :       return DAG.getSetCC(DL, VT, And, LR, CC1);
    3938             :     }
    3939             :   }
    3940             : 
    3941             :   // TODO: What is the 'or' equivalent of this fold?
    3942             :   // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
    3943        2101 :   if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
    3944        2811 :       IsInteger && CC0 == ISD::SETNE &&
    3945          70 :       ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
    3946          38 :        (isAllOnesConstant(LR) && isNullConstant(RR)))) {
    3947           3 :     SDValue One = DAG.getConstant(1, DL, OpVT);
    3948           3 :     SDValue Two = DAG.getConstant(2, DL, OpVT);
    3949           3 :     SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
    3950           3 :     AddToWorklist(Add.getNode());
    3951           3 :     return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
    3952             :   }
    3953             : 
    3954             :   // Try more general transforms if the predicates match and the only user of
    3955             :   // the compares is the 'and' or 'or'.
    3956        2239 :   if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
    3957        2953 :       N0.hasOneUse() && N1.hasOneUse()) {
    3958             :     // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
    3959             :     // or  (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
    3960         232 :     if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
    3961          81 :       SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
    3962          81 :       SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
    3963         158 :       SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
    3964          79 :       SDValue Zero = DAG.getConstant(0, DL, OpVT);
    3965          79 :       return DAG.getSetCC(DL, VT, Or, Zero, CC1);
    3966             :     }
    3967             :   }
    3968             : 
    3969             :   // Canonicalize equivalent operands to LL == RL.
    3970        2682 :   if (LL == RR && LR == RL) {
    3971           0 :     CC1 = ISD::getSetCCSwappedOperands(CC1);
    3972             :     std::swap(RL, RR);
    3973             :   }
    3974             : 
    3975             :   // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
    3976             :   // (or  (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
    3977        3278 :   if (LL == RL && LR == RR) {
    3978          86 :     ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
    3979          43 :                                 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
    3980          86 :     if (NewCC != ISD::SETCC_INVALID &&
    3981          71 :         (!LegalOperations ||
    3982          39 :          (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
    3983             :           TLI.isOperationLegal(ISD::SETCC, OpVT))))
    3984          32 :       return DAG.getSetCC(DL, VT, LL, LR, NewCC);
    3985             :   }
    3986             : 
    3987        2610 :   return SDValue();
    3988             : }
    3989             : 
    3990             : /// This contains all DAGCombine rules which reduce two values combined by
    3991             : /// an And operation to a single value. This makes them reusable in the context
    3992             : /// of visitSELECT(). Rules involving constants are not included as
    3993             : /// visitSELECT() already handles those cases.
    3994      366881 : SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
    3995      733762 :   EVT VT = N1.getValueType();
    3996             :   SDLoc DL(N);
    3997             : 
    3998             :   // fold (and x, undef) -> 0
    3999      366881 :   if (N0.isUndef() || N1.isUndef())
    4000           2 :     return DAG.getConstant(0, DL, VT);
    4001             : 
    4002      366879 :   if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
    4003          93 :     return V;
    4004             : 
    4005      366786 :   if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
    4006         269 :       VT.getSizeInBits() <= 64) {
    4007             :     if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    4008             :       if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
    4009             :         // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
    4010             :         // immediate for an add, but it is legal if its top c2 bits are set,
    4011             :         // transform the ADD so the immediate doesn't need to be materialized
    4012             :         // in a register.
    4013          54 :         APInt ADDC = ADDI->getAPIntValue();
    4014          54 :         APInt SRLC = SRLI->getAPIntValue();
    4015         108 :         if (ADDC.getMinSignedBits() <= 64 &&
    4016         107 :             SRLC.ult(VT.getSizeInBits()) &&
    4017         106 :             !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
    4018             :           APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
    4019          12 :                                              SRLC.getZExtValue());
    4020          24 :           if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
    4021             :             ADDC |= Mask;
    4022          24 :             if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
    4023             :               SDLoc DL0(N0);
    4024             :               SDValue NewAdd =
    4025          12 :                 DAG.getNode(ISD::ADD, DL0, VT,
    4026          12 :                             N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
    4027          12 :               CombineTo(N0.getNode(), NewAdd);
    4028             :               // Return N so it doesn't get rechecked!
    4029          12 :               return SDValue(N, 0);
    4030             :             }
    4031             :           }
    4032             :         }
    4033             :       }
    4034             :     }
    4035             :   }
    4036             : 
    4037             :   // Reduce bit extract of low half of an integer to the narrower type.
    4038             :   // (and (srl i64:x, K), KMask) ->
    4039             :   //   (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
    4040      393335 :   if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    4041             :     if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
    4042             :       if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    4043       17931 :         unsigned Size = VT.getSizeInBits();
    4044       17931 :         const APInt &AndMask = CAnd->getAPIntValue();
    4045       17931 :         unsigned ShiftBits = CShift->getZExtValue();
    4046             : 
    4047             :         // Bail out, this node will probably disappear anyway.
    4048       17931 :         if (ShiftBits == 0)
    4049           2 :           return SDValue();
    4050             : 
    4051             :         unsigned MaskBits = AndMask.countTrailingOnes();
    4052       17929 :         EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
    4053             : 
    4054       33068 :         if (AndMask.isMask() &&
    4055             :             // Required bits must not span the two halves of the integer and
    4056             :             // must fit in the half size type.
    4057       23718 :             (ShiftBits + MaskBits <= Size / 2) &&
    4058        8694 :             TLI.isNarrowingProfitable(VT, HalfVT) &&
    4059         230 :             TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
    4060         230 :             TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
    4061       18159 :             TLI.isTruncateFree(VT, HalfVT) &&
    4062         115 :             TLI.isZExtFree(HalfVT, VT)) {
    4063             :           // The isNarrowingProfitable is to avoid regressions on PPC and
    4064             :           // AArch64 which match a few 64-bit bit insert / bit extract patterns
    4065             :           // on downstream users of this. Those patterns could probably be
    4066             :           // extended to handle extensions mixed in.
    4067             : 
    4068             :           SDValue SL(N0);
    4069             :           assert(MaskBits <= Size);
    4070             : 
    4071             :           // Extracting the highest bit of the low half.
    4072         108 :           EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
    4073         108 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
    4074         108 :                                       N0.getOperand(0));
    4075             : 
    4076         262 :           SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
    4077         154 :           SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
    4078         154 :           SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
    4079         154 :           SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
    4080         262 :           return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
    4081             :         }
    4082             :       }
    4083             :     }
    4084             :   }
    4085             : 
    4086      366664 :   return SDValue();
    4087             : }
    4088             : 
    4089           0 : bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
    4090             :                                    EVT LoadResultTy, EVT &ExtVT) {
    4091           0 :   if (!AndC->getAPIntValue().isMask())
    4092           0 :     return false;
    4093             : 
    4094             :   unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
    4095             : 
    4096           0 :   ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    4097           0 :   EVT LoadedVT = LoadN->getMemoryVT();
    4098             : 
    4099           0 :   if (ExtVT == LoadedVT &&
    4100           0 :       (!LegalOperations ||
    4101           0 :        TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
    4102             :     // ZEXTLOAD will match without needing to change the size of the value being
    4103             :     // loaded.
    4104           0 :     return true;
    4105             :   }
    4106             : 
    4107             :   // Do not change the width of a volatile load.
    4108           0 :   if (LoadN->isVolatile())
    4109           0 :     return false;
    4110             : 
    4111             :   // Do not generate loads of non-round integer types since these can
    4112             :   // be expensive (and would be wrong if the type is not byte sized).
    4113           0 :   if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
    4114           0 :     return false;
    4115             : 
    4116           0 :   if (LegalOperations &&
    4117           0 :       !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
    4118           0 :     return false;
    4119             : 
    4120           0 :   if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
    4121           0 :     return false;
    4122             : 
    4123             :   return true;
    4124             : }
    4125             : 
    4126      147171 : bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
    4127             :                                     ISD::LoadExtType ExtType, EVT &MemVT,
    4128             :                                     unsigned ShAmt) {
    4129      147171 :   if (!LDST)
    4130             :     return false;
    4131             :   // Only allow byte offsets.
    4132      147171 :   if (ShAmt % 8)
    4133             :     return false;
    4134             : 
    4135             :   // Do not generate loads of non-round integer types since these can
    4136             :   // be expensive (and would be wrong if the type is not byte sized).
    4137             :   if (!MemVT.isRound())
    4138             :     return false;
    4139             : 
    4140             :   // Don't change the width of a volatile load.
    4141       53534 :   if (LDST->isVolatile())
    4142             :     return false;
    4143             : 
    4144             :   // Verify that we are actually reducing a load width here.
    4145       52656 :   if (LDST->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits())
    4146             :     return false;
    4147             : 
    4148             :   // Ensure that this isn't going to produce an unsupported unaligned access.
    4149       63169 :   if (ShAmt &&
    4150       10972 :       !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
    4151             :                               LDST->getAddressSpace(), ShAmt / 8))
    4152             :     return false;
    4153             : 
    4154             :   // It's not possible to generate a constant of extended or untyped type.
    4155       52191 :   EVT PtrType = LDST->getBasePtr().getValueType();
    4156       52191 :   if (PtrType == MVT::Untyped || PtrType.isExtended())
    4157             :     return false;
    4158             : 
    4159       52191 :   if (isa<LoadSDNode>(LDST)) {
    4160             :     LoadSDNode *Load = cast<LoadSDNode>(LDST);
    4161             :     // Don't transform one with multiple uses, this would require adding a new
    4162             :     // load.
    4163       52191 :     if (!SDValue(Load, 0).hasOneUse())
    4164             :       return false;
    4165             : 
    4166       22744 :     if (LegalOperations &&
    4167       18446 :         !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
    4168             :       return false;
    4169             : 
    4170             :     // For the transform to be legal, the load must produce only two values
    4171             :     // (the value loaded and the chain).  Don't transform a pre-increment
    4172             :     // load, for example, which produces an extra value.  Otherwise the
    4173             :     // transformation is not equivalent, and the downstream logic to replace
    4174             :     // uses gets things wrong.
    4175        5138 :     if (Load->getNumValues() > 2)
    4176             :       return false;
    4177             : 
    4178             :     // If the load that we're shrinking is an extload and we're not just
    4179             :     // discarding the extension we can't simply shrink the load. Bail.
    4180             :     // TODO: It would be possible to merge the extensions in some cases.
    4181        5138 :     if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
    4182         846 :         Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
    4183           1 :       return false;
    4184             : 
    4185        5137 :     if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT))
    4186        1874 :       return false;
    4187             :   } else {
    4188             :     assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode");
    4189             :     StoreSDNode *Store = cast<StoreSDNode>(LDST);
    4190             :     // Can't write outside the original store
    4191           0 :     if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
    4192             :       return false;
    4193             : 
    4194           0 :     if (LegalOperations &&
    4195           0 :         !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT))
    4196           0 :       return false;
    4197             :   }
    4198             :   return true;
    4199             : }
    4200             : 
    4201      198295 : bool DAGCombiner::SearchForAndLoads(SDNode *N,
    4202             :                                     SmallPtrSetImpl<LoadSDNode*> &Loads,
    4203             :                                     SmallPtrSetImpl<SDNode*> &NodesWithConsts,
    4204             :                                     ConstantSDNode *Mask,
    4205             :                                     SDNode *&NodeToMask) {
    4206             :   // Recursively search for the operands, looking for loads which can be
    4207             :   // narrowed.
    4208      546799 :   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
    4209      745646 :     SDValue Op = N->getOperand(i);
    4210             : 
    4211      745646 :     if (Op.getValueType().isVector())
    4212             :       return false;
    4213             : 
    4214             :     // Some constants may need fixing up later if they are too large.
    4215             :     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
    4216      348542 :       if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
    4217      176924 :           (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
    4218          89 :         NodesWithConsts.insert(N);
    4219      173892 :       continue;
    4220             :     }
    4221             : 
    4222      198931 :     if (!Op.hasOneUse())
    4223             :       return false;
    4224             : 
    4225      180213 :     switch(Op.getOpcode()) {
    4226             :     case ISD::LOAD: {
    4227             :       auto *Load = cast<LoadSDNode>(Op);
    4228        1278 :       EVT ExtVT;
    4229        2726 :       if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
    4230         170 :           isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
    4231             : 
    4232             :         // ZEXTLOAD is already small enough.
    4233         159 :         if (Load->getExtensionType() == ISD::ZEXTLOAD &&
    4234          13 :             ExtVT.bitsGE(Load->getMemoryVT()))
    4235         159 :           continue;
    4236             : 
    4237             :         // Use LE to convert equal sized loads to zext.
    4238         150 :         if (ExtVT.bitsLE(Load->getMemoryVT()))
    4239         150 :           Loads.insert(Load);
    4240             : 
    4241         150 :         continue;
    4242             :       }
    4243        1119 :       return false;
    4244             :     }
    4245        1719 :     case ISD::ZERO_EXTEND:
    4246             :     case ISD::AssertZext: {
    4247        1719 :       unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
    4248        1719 :       EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    4249             :       EVT VT = Op.getOpcode() == ISD::AssertZext ?
    4250        1157 :         cast<VTSDNode>(Op.getOperand(1))->getVT() :
    4251        1719 :         Op.getOperand(0).getValueType();
    4252             : 
    4253             :       // We can accept extending nodes if the mask is wider or an equal
    4254             :       // width to the original type.
    4255        1719 :       if (ExtVT.bitsGE(VT))
    4256        1156 :         continue;
    4257         563 :       break;
    4258             :     }
    4259        4368 :     case ISD::OR:
    4260             :     case ISD::XOR:
    4261             :     case ISD::AND:
    4262        4368 :       if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
    4263             :                              NodeToMask))
    4264             :         return false;
    4265             :       continue;
    4266             :     }
    4267             : 
    4268             :     // Allow one node which will masked along with any loads found.
    4269      173411 :     if (NodeToMask)
    4270             :       return false;
    4271             : 
    4272             :     // Also ensure that the node to be masked only produces one data result.
    4273      173025 :     NodeToMask = Op.getNode();
    4274      346050 :     if (NodeToMask->getNumValues() > 1) {
    4275             :       bool HasValue = false;
    4276       20808 :       for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
    4277             :         MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
    4278       14661 :         if (VT != MVT::Glue && VT != MVT::Other) {
    4279        7681 :           if (HasValue) {
    4280         767 :             NodeToMask = nullptr;
    4281             :             return false;
    4282             :           }
    4283             :           HasValue = true;
    4284             :         }
    4285             :       }
    4286             :       assert(HasValue && "Node to be masked has no data result?");
    4287             :     }
    4288             :   }
    4289             :   return true;
    4290             : }
    4291             : 
    4292      297853 : bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
    4293      297853 :   auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
    4294             :   if (!Mask)
    4295             :     return false;
    4296             : 
    4297      497302 :   if (!Mask->getAPIntValue().isMask())
    4298             :     return false;
    4299             : 
    4300             :   // No need to do anything if the and directly uses a load.
    4301      225803 :   if (isa<LoadSDNode>(N->getOperand(0)))
    4302             :     return false;
    4303             : 
    4304             :   SmallPtrSet<LoadSDNode*, 8> Loads;
    4305             :   SmallPtrSet<SDNode*, 2> NodesWithConsts;
    4306      193927 :   SDNode *FixupNode = nullptr;
    4307      193927 :   if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
    4308      172937 :     if (Loads.size() == 0)
    4309             :       return false;
    4310             : 
    4311             :     LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump());
    4312          88 :     SDValue MaskOp = N->getOperand(1);
    4313             : 
    4314             :     // If it exists, fixup the single node we allow in the tree that needs
    4315             :     // masking.
    4316          88 :     if (FixupNode) {
    4317             :       LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump());
    4318          26 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
    4319             :                                 FixupNode->getValueType(0),
    4320          26 :                                 SDValue(FixupNode, 0), MaskOp);
    4321          52 :       DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
    4322          26 :       if (And.getOpcode() == ISD ::AND)
    4323          50 :         DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
    4324             :     }
    4325             : 
    4326             :     // Narrow any constants that need it.
    4327         102 :     for (auto *LogicN : NodesWithConsts) {
    4328          14 :       SDValue Op0 = LogicN->getOperand(0);
    4329          14 :       SDValue Op1 = LogicN->getOperand(1);
    4330             : 
    4331             :       if (isa<ConstantSDNode>(Op0))
    4332             :           std::swap(Op0, Op1);
    4333             : 
    4334          14 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
    4335          14 :                                 Op1, MaskOp);
    4336             : 
    4337          14 :       DAG.UpdateNodeOperands(LogicN, Op0, And);
    4338             :     }
    4339             : 
    4340             :     // Create narrow loads.
    4341         233 :     for (auto *Load : Loads) {
    4342             :       LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump());
    4343         145 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
    4344         145 :                                 SDValue(Load, 0), MaskOp);
    4345         145 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
    4346         145 :       if (And.getOpcode() == ISD ::AND)
    4347         145 :         And = SDValue(
    4348             :             DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
    4349         145 :       SDValue NewLoad = ReduceLoadWidth(And.getNode());
    4350             :       assert(NewLoad &&
    4351             :              "Shouldn't be masking the load if it can't be narrowed");
    4352             :       CombineTo(Load, NewLoad, NewLoad.getValue(1));
    4353             :     }
    4354          88 :     DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
    4355          88 :     return true;
    4356             :   }
    4357             :   return false;
    4358             : }
    4359             : 
    4360             : // Unfold
    4361             : //    x &  (-1 'logical shift' y)
    4362             : // To
    4363             : //    (x 'opposite logical shift' y) 'logical shift' y
    4364             : // if it is better for performance.
    4365           0 : SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
    4366             :   assert(N->getOpcode() == ISD::AND);
    4367             : 
    4368           0 :   SDValue N0 = N->getOperand(0);
    4369           0 :   SDValue N1 = N->getOperand(1);
    4370             : 
    4371             :   // Do we actually prefer shifts over mask?
    4372           0 :   if (!TLI.preferShiftsToClearExtremeBits(N0))
    4373           0 :     return SDValue();
    4374             : 
    4375             :   // Try to match  (-1 '[outer] logical shift' y)
    4376             :   unsigned OuterShift;
    4377             :   unsigned InnerShift; // The opposite direction to the OuterShift.
    4378           0 :   SDValue Y;           // Shift amount.
    4379             :   auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
    4380             :     if (!M.hasOneUse())
    4381             :       return false;
    4382             :     OuterShift = M->getOpcode();
    4383             :     if (OuterShift == ISD::SHL)
    4384             :       InnerShift = ISD::SRL;
    4385             :     else if (OuterShift == ISD::SRL)
    4386             :       InnerShift = ISD::SHL;
    4387             :     else
    4388             :       return false;
    4389             :     if (!isAllOnesConstant(M->getOperand(0)))
    4390             :       return false;
    4391             :     Y = M->getOperand(1);
    4392             :     return true;
    4393           0 :   };
    4394             : 
    4395           0 :   SDValue X;
    4396           0 :   if (matchMask(N1))
    4397           0 :     X = N0;
    4398           0 :   else if (matchMask(N0))
    4399           0 :     X = N1;
    4400             :   else
    4401           0 :     return SDValue();
    4402             : 
    4403             :   SDLoc DL(N);
    4404           0 :   EVT VT = N->getValueType(0);
    4405             : 
    4406             :   //     tmp = x   'opposite logical shift' y
    4407           0 :   SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
    4408             :   //     ret = tmp 'logical shift' y
    4409           0 :   SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
    4410             : 
    4411           0 :   return T1;
    4412             : }
    4413             : 
    4414      389060 : SDValue DAGCombiner::visitAND(SDNode *N) {
    4415      389060 :   SDValue N0 = N->getOperand(0);
    4416      389060 :   SDValue N1 = N->getOperand(1);
    4417      389060 :   EVT VT = N1.getValueType();
    4418             : 
    4419             :   // x & x --> x
    4420      389060 :   if (N0 == N1)
    4421          21 :     return N0;
    4422             : 
    4423             :   // fold vector ops
    4424      389039 :   if (VT.isVector()) {
    4425       46825 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    4426          10 :       return FoldedVOp;
    4427             : 
    4428             :     // fold (and x, 0) -> 0, vector edition
    4429       46815 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    4430             :       // do not return N0, because undef node may exist in N0
    4431         128 :       return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
    4432         192 :                              SDLoc(N), N0.getValueType());
    4433       46751 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    4434             :       // do not return N1, because undef node may exist in N1
    4435          22 :       return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
    4436          33 :                              SDLoc(N), N1.getValueType());
    4437             : 
    4438             :     // fold (and x, -1) -> x, vector edition
    4439       46740 :     if (ISD::isBuildVectorAllOnes(N0.getNode()))
    4440          55 :       return N1;
    4441       46685 :     if (ISD::isBuildVectorAllOnes(N1.getNode()))
    4442           7 :       return N0;
    4443             :   }
    4444             : 
    4445             :   // fold (and c1, c2) -> c1&c2
    4446      388892 :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    4447      388892 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    4448      388892 :   if (N0C && N1C && !N1C->isOpaque())
    4449          44 :     return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
    4450             :   // canonicalize constant to RHS
    4451      389418 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    4452         548 :       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    4453        1072 :     return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
    4454             :   // fold (and x, -1) -> x
    4455      388334 :   if (isAllOnesConstant(N1))
    4456          11 :     return N0;
    4457             :   // if (and x, c) is known to be zero, return 0
    4458             :   unsigned BitWidth = VT.getScalarSizeInBits();
    4459      716478 :   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
    4460     1044574 :                                    APInt::getAllOnesValue(BitWidth)))
    4461         118 :     return DAG.getConstant(0, SDLoc(N), VT);
    4462             : 
    4463      388264 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    4464         190 :     return NewSel;
    4465             : 
    4466             :   // reassociate and
    4467      536078 :   if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
    4468        2009 :     return RAND;
    4469             : 
    4470             :   // Try to convert a constant mask AND into a shuffle clear mask.
    4471      386065 :   if (VT.isVector())
    4472       46087 :     if (SDValue Shuffle = XformToShuffleWithZero(N))
    4473        1071 :       return Shuffle;
    4474             : 
    4475             :   // fold (and (or x, C), D) -> D if (C & D) == D
    4476             :   auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    4477             :     return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
    4478             :   };
    4479      780912 :   if (N0.getOpcode() == ISD::OR &&
    4480      404553 :       ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
    4481        2289 :     return N1;
    4482             :   // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
    4483      382705 :   if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    4484       67807 :     SDValue N0Op0 = N0.getOperand(0);
    4485       67807 :     APInt Mask = ~N1C->getAPIntValue();
    4486       67807 :     Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
    4487       67807 :     if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
    4488       13445 :       SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
    4489       27196 :                                  N0.getValueType(), N0Op0);
    4490             : 
    4491             :       // Replace uses of the AND with uses of the Zero extend node.
    4492       13445 :       CombineTo(N, Zext);
    4493             : 
    4494             :       // We actually want to replace all uses of the any_extend with the
    4495             :       // zero_extend, to avoid duplicating things.  This will later cause this
    4496             :       // AND to be folded.
    4497       13445 :       CombineTo(N0.getNode(), Zext);
    4498       13445 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4499             :     }
    4500             :   }
    4501             :   // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
    4502             :   // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
    4503             :   // already be zero by virtue of the width of the base type of the load.
    4504             :   //
    4505             :   // the 'X' node here can either be nothing or an extract_vector_elt to catch
    4506             :   // more cases.
    4507      378653 :   if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
    4508       17702 :        N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
    4509       16618 :        N0.getOperand(0).getOpcode() == ISD::LOAD &&
    4510      378653 :        N0.getOperand(0).getResNo() == 0) ||
    4511      365560 :       (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
    4512      159686 :     LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
    4513             :                                          N0 : N0.getOperand(0) );
    4514             : 
    4515             :     // Get the constant (if applicable) the zero'th operand is being ANDed with.
    4516             :     // This can be a pure constant or a vector splat, in which case we treat the
    4517             :     // vector as a scalar and use the splat value.
    4518             :     APInt Constant = APInt::getNullValue(1);
    4519             :     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
    4520      147150 :       Constant = C->getAPIntValue();
    4521             :     } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
    4522             :       APInt SplatValue, SplatUndef;
    4523             :       unsigned SplatBitSize;
    4524             :       bool HasAnyUndefs;
    4525        1098 :       bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
    4526             :                                              SplatBitSize, HasAnyUndefs);
    4527        1098 :       if (IsSplat) {
    4528             :         // Undef bits can contribute to a possible optimisation if set, so
    4529             :         // set them.
    4530             :         SplatValue |= SplatUndef;
    4531             : 
    4532             :         // The splat value may be something like "0x00FFFFFF", which means 0 for
    4533             :         // the first vector value and FF for the rest, repeating. We need a mask
    4534             :         // that will apply equally to all members of the vector, so AND all the
    4535             :         // lanes of the constant together.
    4536        2192 :         EVT VT = Vector->getValueType(0);
    4537             :         unsigned BitWidth = VT.getScalarSizeInBits();
    4538             : 
    4539             :         // If the splat value has been compressed to a bitlength lower
    4540             :         // than the size of the vector lane, we need to re-expand it to
    4541             :         // the lane size.
    4542        1096 :         if (BitWidth > SplatBitSize)
    4543           2 :           for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
    4544           2 :                SplatBitSize < BitWidth;
    4545           1 :                SplatBitSize = SplatBitSize * 2)
    4546           2 :             SplatValue |= SplatValue.shl(SplatBitSize);
    4547             : 
    4548             :         // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
    4549             :         // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
    4550        1096 :         if (SplatBitSize % BitWidth == 0) {
    4551        1095 :           Constant = APInt::getAllOnesValue(BitWidth);
    4552        2235 :           for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
    4553        2288 :             Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
    4554             :         }
    4555             :       }
    4556             :     }
    4557             : 
    4558             :     // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
    4559             :     // actually legal and isn't going to get expanded, else this is a false
    4560             :     // optimisation.
    4561       79843 :     bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
    4562             :                                                     Load->getValueType(0),
    4563             :                                                     Load->getMemoryVT());
    4564             : 
    4565             :     // Resize the constant to the same size as the original memory access before
    4566             :     // extension. If it is still the AllOnesValue then this AND is completely
    4567             :     // unneeded.
    4568      159686 :     Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
    4569             : 
    4570             :     bool B;
    4571       79843 :     switch (Load->getExtensionType()) {
    4572             :     default: B = false; break;
    4573             :     case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
    4574             :     case ISD::ZEXTLOAD:
    4575             :     case ISD::NON_EXTLOAD: B = true; break;
    4576             :     }
    4577             : 
    4578       92514 :     if (B && Constant.isAllOnesValue()) {
    4579             :       // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
    4580             :       // preserve semantics once we get rid of the AND.
    4581             :       SDValue NewLoad(Load, 0);
    4582             : 
    4583             :       // Fold the AND away. NewLoad may get replaced immediately.
    4584        4444 :       CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
    4585             : 
    4586        2222 :       if (Load->getExtensionType() == ISD::EXTLOAD) {
    4587        1268 :         NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
    4588        1268 :                               Load->getValueType(0), SDLoc(Load),
    4589             :                               Load->getChain(), Load->getBasePtr(),
    4590             :                               Load->getOffset(), Load->getMemoryVT(),
    4591        1276 :                               Load->getMemOperand());
    4592             :         // Replace uses of the EXTLOAD with the new ZEXTLOAD.
    4593        1268 :         if (Load->getNumValues() == 3) {
    4594             :           // PRE/POST_INC loads have 3 values.
    4595           0 :           SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
    4596           0 :                            NewLoad.getValue(2) };
    4597           0 :           CombineTo(Load, To, 3, true);
    4598             :         } else {
    4599        1268 :           CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
    4600             :         }
    4601             :       }
    4602             : 
    4603        2222 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    4604             :     }
    4605             :   }
    4606             : 
    4607             :   // fold (and (load x), 255) -> (zextload x, i8)
    4608             :   // fold (and (extload x, i16), 255) -> (zextload x, i8)
    4609             :   // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
    4610      367038 :   if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
    4611       54333 :                                 (N0.getOpcode() == ISD::ANY_EXTEND &&
    4612       54333 :                                  N0.getOperand(0).getOpcode() == ISD::LOAD))) {
    4613       79699 :     if (SDValue Res = ReduceLoadWidth(N)) {
    4614         137 :       LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
    4615         137 :         ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
    4616             : 
    4617         137 :       AddToWorklist(N);
    4618             :       CombineTo(LN0, Res, Res.getValue(1));
    4619         137 :       return SDValue(N, 0);
    4620             :     }
    4621             :   }
    4622             : 
    4623      366901 :   if (Level >= AfterLegalizeTypes) {
    4624             :     // Attempt to propagate the AND back up to the leaves which, if they're
    4625             :     // loads, can be combined to narrow loads and the AND node can be removed.
    4626             :     // Perform after legalization so that extend nodes will already be
    4627             :     // combined into the loads.
    4628      297853 :     if (BackwardsPropagateMask(N, DAG)) {
    4629          88 :       return SDValue(N, 0);
    4630             :     }
    4631             :   }
    4632             : 
    4633      366813 :   if (SDValue Combined = visitANDLike(N0, N1, N))
    4634         209 :     return Combined;
    4635             : 
    4636             :   // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
    4637     1099812 :   if (N0.getOpcode() == N1.getOpcode())
    4638       25753 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    4639         303 :       return Tmp;
    4640             : 
    4641             :   // Masking the negated extension of a boolean is just the zero-extended
    4642             :   // boolean:
    4643             :   // and (sub 0, zext(bool X)), 1 --> zext(bool X)
    4644             :   // and (sub 0, sext(bool X)), 1 --> zext(bool X)
    4645             :   //
    4646             :   // Note: the SimplifyDemandedBits fold below can make an information-losing
    4647             :   // transform, and then we have no way to find this better fold.
    4648      673248 :   if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
    4649          28 :     if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
    4650          12 :       SDValue SubRHS = N0.getOperand(1);
    4651           9 :       if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
    4652           3 :           SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
    4653           3 :         return SubRHS;
    4654           6 :       if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
    4655           3 :           SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
    4656           6 :         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
    4657             :     }
    4658             :   }
    4659             : 
    4660             :   // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
    4661             :   // fold (and (sra)) -> (and (srl)) when possible.
    4662      366295 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    4663      109940 :     return SDValue(N, 0);
    4664             : 
    4665             :   // fold (zext_inreg (extload x)) -> (zextload x)
    4666      256355 :   if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
    4667             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    4668       14248 :     EVT MemVT = LN0->getMemoryVT();
    4669             :     // If we zero all the possible extended bits, then we can turn this into
    4670             :     // a zextload if we are running before legalize or the operation is legal.
    4671       14248 :     unsigned BitWidth = N1.getScalarValueSizeInBits();
    4672       28496 :     if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
    4673       14248 :                            BitWidth - MemVT.getScalarSizeInBits())) &&
    4674       16737 :         ((!LegalOperations && !LN0->isVolatile()) ||
    4675        2562 :          TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
    4676       24562 :       SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
    4677             :                                        LN0->getChain(), LN0->getBasePtr(),
    4678       12325 :                                        MemVT, LN0->getMemOperand());
    4679       12281 :       AddToWorklist(N);
    4680       12281 :       CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    4681       12281 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4682             :     }
    4683             :   }
    4684             :   // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
    4685      244421 :   if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
    4686         347 :       N0.hasOneUse()) {
    4687             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    4688         302 :     EVT MemVT = LN0->getMemoryVT();
    4689             :     // If we zero all the possible extended bits, then we can turn this into
    4690             :     // a zextload if we are running before legalize or the operation is legal.
    4691         302 :     unsigned BitWidth = N1.getScalarValueSizeInBits();
    4692         604 :     if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
    4693         302 :                            BitWidth - MemVT.getScalarSizeInBits())) &&
    4694         490 :         ((!LegalOperations && !LN0->isVolatile()) ||
    4695         227 :          TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
    4696         494 :       SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
    4697             :                                        LN0->getChain(), LN0->getBasePtr(),
    4698         247 :                                        MemVT, LN0->getMemOperand());
    4699         247 :       AddToWorklist(N);
    4700         247 :       CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    4701         247 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4702             :     }
    4703             :   }
    4704             :   // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
    4705      243827 :   if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
    4706         264 :     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
    4707         264 :                                            N0.getOperand(1), false))
    4708           1 :       return BSwap;
    4709             :   }
    4710             : 
    4711      243826 :   if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
    4712         636 :     return Shifts;
    4713             : 
    4714      243190 :   return SDValue();
    4715             : }
    4716             : 
    4717             : /// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
    4718      140327 : SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
    4719             :                                         bool DemandHighBits) {
    4720      140327 :   if (!LegalOperations)
    4721       57599 :     return SDValue();
    4722             : 
    4723      165456 :   EVT VT = N->getValueType(0);
    4724             :   if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
    4725       16680 :     return SDValue();
    4726       66048 :   if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    4727       14676 :     return SDValue();
    4728             : 
    4729             :   // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
    4730             :   bool LookPassAnd0 = false;
    4731             :   bool LookPassAnd1 = false;
    4732       51372 :   if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
    4733             :       std::swap(N0, N1);
    4734       51372 :   if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
    4735             :       std::swap(N0, N1);
    4736       51372 :   if (N0.getOpcode() == ISD::AND) {
    4737             :     if (!N0.getNode()->hasOneUse())
    4738         202 :       return SDValue();
    4739             :     ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4740             :     // Also handle 0xffff since the LHS is guaranteed to have zeros there.
    4741             :     // This is needed for X86.
    4742       12722 :     if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
    4743             :                   N01C->getZExtValue() != 0xFFFF))
    4744        5575 :       return SDValue();
    4745         980 :     N0 = N0.getOperand(0);
    4746             :     LookPassAnd0 = true;
    4747             :   }
    4748             : 
    4749       45595 :   if (N1.getOpcode() == ISD::AND) {
    4750             :     if (!N1.getNode()->hasOneUse())
    4751          24 :       return SDValue();
    4752             :     ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    4753        2274 :     if (!N11C || N11C->getZExtValue() != 0xFF)
    4754        1026 :       return SDValue();
    4755         125 :     N1 = N1.getOperand(0);
    4756             :     LookPassAnd1 = true;
    4757             :   }
    4758             : 
    4759       44545 :   if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
    4760             :     std::swap(N0, N1);
    4761       44545 :   if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
    4762       43658 :     return SDValue();
    4763             :   if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
    4764          14 :     return SDValue();
    4765             : 
    4766             :   ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4767             :   ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    4768         873 :   if (!N01C || !N11C)
    4769         239 :     return SDValue();
    4770        1305 :   if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
    4771         614 :     return SDValue();
    4772             : 
    4773             :   // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
    4774          20 :   SDValue N00 = N0->getOperand(0);
    4775          20 :   if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
    4776             :     if (!N00.getNode()->hasOneUse())
    4777           0 :       return SDValue();
    4778             :     ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
    4779          12 :     if (!N001C || N001C->getZExtValue() != 0xFF)
    4780           2 :       return SDValue();
    4781           4 :     N00 = N00.getOperand(0);
    4782             :     LookPassAnd0 = true;
    4783             :   }
    4784             : 
    4785          18 :   SDValue N10 = N1->getOperand(0);
    4786          18 :   if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
    4787             :     if (!N10.getNode()->hasOneUse())
    4788           0 :       return SDValue();
    4789             :     ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
    4790             :     // Also allow 0xFFFF since the bits will be shifted out. This is needed
    4791             :     // for X86.
    4792          10 :     if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
    4793             :                    N101C->getZExtValue() != 0xFFFF))
    4794           0 :       return SDValue();
    4795           5 :     N10 = N10.getOperand(0);
    4796             :     LookPassAnd1 = true;
    4797             :   }
    4798             : 
    4799             :   if (N00 != N10)
    4800           0 :     return SDValue();
    4801             : 
    4802             :   // Make sure everything beyond the low halfword gets set to zero since the SRL
    4803             :   // 16 will clear the top bits.
    4804          18 :   unsigned OpSizeInBits = VT.getSizeInBits();
    4805          18 :   if (DemandHighBits && OpSizeInBits > 16) {
    4806             :     // If the left-shift isn't masked out then the only way this is a bswap is
    4807             :     // if all bits beyond the low 8 are 0. In that case the entire pattern
    4808             :     // reduces to a left shift anyway: leave it for other parts of the combiner.
    4809           9 :     if (!LookPassAnd0)
    4810           2 :       return SDValue();
    4811             : 
    4812             :     // However, if the right shift isn't masked out then it might be because
    4813             :     // it's not needed. See if we can spot that too.
    4814           9 :     if (!LookPassAnd1 &&
    4815           2 :         !DAG.MaskedValueIsZero(
    4816          11 :             N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
    4817           0 :       return SDValue();
    4818             :   }
    4819             : 
    4820          16 :   SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
    4821          16 :   if (OpSizeInBits > 16) {
    4822             :     SDLoc DL(N);
    4823          16 :     Res = DAG.getNode(ISD::SRL, DL, VT, Res,
    4824          16 :                       DAG.getConstant(OpSizeInBits - 16, DL,
    4825          16 :                                       getShiftAmountTy(VT)));
    4826             :   }
    4827          16 :   return Res;
    4828             : }
    4829             : 
    4830             : /// Return true if the specified node is an element that makes up a 32-bit
    4831             : /// packed halfword byteswap.
    4832             : /// ((x & 0x000000ff) << 8) |
    4833             : /// ((x & 0x0000ff00) >> 8) |
    4834             : /// ((x & 0x00ff0000) << 8) |
    4835             : /// ((x & 0xff000000) >> 8)
    4836           0 : static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
    4837             :   if (!N.getNode()->hasOneUse())
    4838           0 :     return false;
    4839             : 
    4840             :   unsigned Opc = N.getOpcode();
    4841           0 :   if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
    4842           0 :     return false;
    4843             : 
    4844           0 :   SDValue N0 = N.getOperand(0);
    4845             :   unsigned Opc0 = N0.getOpcode();
    4846           0 :   if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
    4847           0 :     return false;
    4848             : 
    4849             :   ConstantSDNode *N1C = nullptr;
    4850             :   // SHL or SRL: look upstream for AND mask operand
    4851           0 :   if (Opc == ISD::AND)
    4852             :     N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4853           0 :   else if (Opc0 == ISD::AND)
    4854             :     N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4855           0 :   if (!N1C)
    4856           0 :     return false;
    4857             : 
    4858             :   unsigned MaskByteOffset;
    4859           0 :   switch (N1C->getZExtValue()) {
    4860             :   default:
    4861             :     return false;
    4862             :   case 0xFF:       MaskByteOffset = 0; break;
    4863           0 :   case 0xFF00:     MaskByteOffset = 1; break;
    4864           0 :   case 0xFFFF:
    4865             :     // In case demanded bits didn't clear the bits that will be shifted out.
    4866             :     // This is needed for X86.
    4867           0 :     if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
    4868             :       MaskByteOffset = 1;
    4869             :       break;
    4870             :     }
    4871             :     return false;
    4872           0 :   case 0xFF0000:   MaskByteOffset = 2; break;
    4873           0 :   case 0xFF000000: MaskByteOffset = 3; break;
    4874             :   }
    4875             : 
    4876             :   // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
    4877           0 :   if (Opc == ISD::AND) {
    4878           0 :     if (MaskByteOffset == 0 || MaskByteOffset == 2) {
    4879             :       // (x >> 8) & 0xff
    4880             :       // (x >> 8) & 0xff0000
    4881           0 :       if (Opc0 != ISD::SRL)
    4882           0 :         return false;
    4883             :       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4884           0 :       if (!C || C->getZExtValue() != 8)
    4885           0 :         return false;
    4886             :     } else {
    4887             :       // (x << 8) & 0xff00
    4888             :       // (x << 8) & 0xff000000
    4889           0 :       if (Opc0 != ISD::SHL)
    4890           0 :         return false;
    4891             :       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4892           0 :       if (!C || C->getZExtValue() != 8)
    4893           0 :         return false;
    4894             :     }
    4895           0 :   } else if (Opc == ISD::SHL) {
    4896             :     // (x & 0xff) << 8
    4897             :     // (x & 0xff0000) << 8
    4898           0 :     if (MaskByteOffset != 0 && MaskByteOffset != 2)
    4899           0 :       return false;
    4900             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4901           0 :     if (!C || C->getZExtValue() != 8)
    4902           0 :       return false;
    4903             :   } else { // Opc == ISD::SRL
    4904             :     // (x & 0xff00) >> 8
    4905             :     // (x & 0xff000000) >> 8
    4906           0 :     if (MaskByteOffset != 1 && MaskByteOffset != 3)
    4907           0 :       return false;
    4908             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4909           0 :     if (!C || C->getZExtValue() != 8)
    4910           0 :       return false;
    4911             :   }
    4912             : 
    4913           0 :   if (Parts[MaskByteOffset])
    4914           0 :     return false;
    4915             : 
    4916           0 :   Parts[MaskByteOffset] = N0.getOperand(0).getNode();
    4917           0 :   return true;
    4918             : }
    4919             : 
    4920             : /// Match a 32-bit packed halfword bswap. That is
    4921             : /// ((x & 0x000000ff) << 8) |
    4922             : /// ((x & 0x0000ff00) >> 8) |
    4923             : /// ((x & 0x00ff0000) << 8) |
    4924             : /// ((x & 0xff000000) >> 8)
    4925             : /// => (rotl (bswap x), 16)
    4926           0 : SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
    4927           0 :   if (!LegalOperations)
    4928           0 :     return SDValue();
    4929             : 
    4930           0 :   EVT VT = N->getValueType(0);
    4931           0 :   if (VT != MVT::i32)
    4932           0 :     return SDValue();
    4933           0 :   if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    4934           0 :     return SDValue();
    4935             : 
    4936             :   // Look for either
    4937             :   // (or (or (and), (and)), (or (and), (and)))
    4938             :   // (or (or (or (and), (and)), (and)), (and))
    4939           0 :   if (N0.getOpcode() != ISD::OR)
    4940           0 :     return SDValue();
    4941           0 :   SDValue N00 = N0.getOperand(0);
    4942           0 :   SDValue N01 = N0.getOperand(1);
    4943           0 :   SDNode *Parts[4] = {};
    4944             : 
    4945           0 :   if (N1.getOpcode() == ISD::OR &&
    4946           0 :       N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
    4947             :     // (or (or (and), (and)), (or (and), (and)))
    4948           0 :     if (!isBSwapHWordElement(N00, Parts))
    4949           0 :       return SDValue();
    4950             : 
    4951           0 :     if (!isBSwapHWordElement(N01, Parts))
    4952           0 :       return SDValue();
    4953           0 :     SDValue N10 = N1.getOperand(0);
    4954           0 :     if (!isBSwapHWordElement(N10, Parts))
    4955           0 :       return SDValue();
    4956           0 :     SDValue N11 = N1.getOperand(1);
    4957           0 :     if (!isBSwapHWordElement(N11, Parts))
    4958           0 :       return SDValue();
    4959             :   } else {
    4960             :     // (or (or (or (and), (and)), (and)), (and))
    4961           0 :     if (!isBSwapHWordElement(N1, Parts))
    4962           0 :       return SDValue();
    4963           0 :     if (!isBSwapHWordElement(N01, Parts))
    4964           0 :       return SDValue();
    4965           0 :     if (N00.getOpcode() != ISD::OR)
    4966           0 :       return SDValue();
    4967           0 :     SDValue N000 = N00.getOperand(0);
    4968           0 :     if (!isBSwapHWordElement(N000, Parts))
    4969           0 :       return SDValue();
    4970           0 :     SDValue N001 = N00.getOperand(1);
    4971           0 :     if (!isBSwapHWordElement(N001, Parts))
    4972           0 :       return SDValue();
    4973             :   }
    4974             : 
    4975             :   // Make sure the parts are all coming from the same node.
    4976           0 :   if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
    4977           0 :     return SDValue();
    4978             : 
    4979             :   SDLoc DL(N);
    4980           0 :   SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
    4981           0 :                               SDValue(Parts[0], 0));
    4982             : 
    4983             :   // Result of the bswap should be rotated by 16. If it's not legal, then
    4984             :   // do  (x << 16) | (x >> 16).
    4985           0 :   SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
    4986           0 :   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
    4987           0 :     return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
    4988           0 :   if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
    4989           0 :     return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
    4990           0 :   return DAG.getNode(ISD::OR, DL, VT,
    4991           0 :                      DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
    4992           0 :                      DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
    4993             : }
    4994             : 
    4995             : /// This contains all DAGCombine rules which reduce two values combined by
    4996             : /// an Or operation to a single value \see visitANDLike().
    4997      140443 : SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
    4998      280886 :   EVT VT = N1.getValueType();
    4999             :   SDLoc DL(N);
    5000             : 
    5001             :   // fold (or x, undef) -> -1
    5002      140443 :   if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
    5003          15 :     return DAG.getAllOnesConstant(DL, VT);
    5004             : 
    5005      140428 :   if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
    5006         112 :     return V;
    5007             : 
    5008             :   // (or (and X, C1), (and Y, C2))  -> (and (or X, Y), C3) if possible.
    5009      140316 :   if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
    5010             :       // Don't increase # computations.
    5011             :       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
    5012             :     // We can only do this xform if we know that bits from X that are set in C2
    5013             :     // but not in C1 are already zero.  Likewise for Y.
    5014             :     if (const ConstantSDNode *N0O1C =
    5015        7184 :         getAsNonOpaqueConstant(N0.getOperand(1))) {
    5016             :       if (const ConstantSDNode *N1O1C =
    5017        2953 :           getAsNonOpaqueConstant(N1.getOperand(1))) {
    5018             :         // We can only do this xform if we know that bits from X that are set in
    5019             :         // C2 but not in C1 are already zero.  Likewise for Y.
    5020        2879 :         const APInt &LHSMask = N0O1C->getAPIntValue();
    5021        2879 :         const APInt &RHSMask = N1O1C->getAPIntValue();
    5022             : 
    5023        8851 :         if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
    5024        3949 :             DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
    5025          21 :           SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
    5026          21 :                                   N0.getOperand(0), N1.getOperand(0));
    5027          21 :           return DAG.getNode(ISD::AND, DL, VT, X,
    5028          42 :                              DAG.getConstant(LHSMask | RHSMask, DL, VT));
    5029             :         }
    5030             :       }
    5031             :     }
    5032             :   }
    5033             : 
    5034             :   // (or (and X, M), (and X, N)) -> (and X, (or M, N))
    5035       24393 :   if (N0.getOpcode() == ISD::AND &&
    5036             :       N1.getOpcode() == ISD::AND &&
    5037      147470 :       N0.getOperand(0) == N1.getOperand(0) &&
    5038             :       // Don't increase # computations.
    5039             :       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
    5040         188 :     SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
    5041         188 :                             N0.getOperand(1), N1.getOperand(1));
    5042         376 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
    5043             :   }
    5044             : 
    5045      140107 :   return SDValue();
    5046             : }
    5047             : 
    5048      143210 : SDValue DAGCombiner::visitOR(SDNode *N) {
    5049      143210 :   SDValue N0 = N->getOperand(0);
    5050      143210 :   SDValue N1 = N->getOperand(1);
    5051      143210 :   EVT VT = N1.getValueType();
    5052             : 
    5053             :   // x | x --> x
    5054             :   if (N0 == N1)
    5055          17 :     return N0;
    5056             : 
    5057             :   // fold vector ops
    5058      143193 :   if (VT.isVector()) {
    5059       20144 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    5060           4 :       return FoldedVOp;
    5061             : 
    5062             :     // fold (or x, 0) -> x, vector edition
    5063       20140 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    5064          21 :       return N1;
    5065       20119 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    5066         127 :       return N0;
    5067             : 
    5068             :     // fold (or x, -1) -> -1, vector edition
    5069       19992 :     if (ISD::isBuildVectorAllOnes(N0.getNode()))
    5070             :       // do not return N0, because undef node may exist in N0
    5071          10 :       return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
    5072       19987 :     if (ISD::isBuildVectorAllOnes(N1.getNode()))
    5073             :       // do not return N1, because undef node may exist in N1
    5074           2 :       return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
    5075             : 
    5076             :     // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
    5077             :     // Do this only if the resulting shuffle is legal.
    5078         149 :     if (isa<ShuffleVectorSDNode>(N0) &&
    5079       19986 :         isa<ShuffleVectorSDNode>(N1) &&
    5080             :         // Avoid folding a node with illegal type.
    5081         124 :         TLI.isTypeLegal(VT)) {
    5082         123 :       bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
    5083         123 :       bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
    5084         123 :       bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
    5085         123 :       bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
    5086             :       // Ensure both shuffles have a zero input.
    5087         123 :       if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
    5088             :         assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!");
    5089             :         assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!");
    5090             :         const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
    5091             :         const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
    5092             :         bool CanFold = true;
    5093          66 :         int NumElts = VT.getVectorNumElements();
    5094          66 :         SmallVector<int, 4> Mask(NumElts);
    5095             : 
    5096         300 :         for (int i = 0; i != NumElts; ++i) {
    5097         239 :           int M0 = SV0->getMaskElt(i);
    5098         239 :           int M1 = SV1->getMaskElt(i);
    5099             : 
    5100             :           // Determine if either index is pointing to a zero vector.
    5101         239 :           bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
    5102         239 :           bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
    5103             : 
    5104             :           // If one element is zero and the otherside is undef, keep undef.
    5105             :           // This also handles the case that both are undef.
    5106         239 :           if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
    5107           1 :             Mask[i] = -1;
    5108           1 :             continue;
    5109             :           }
    5110             : 
    5111             :           // Make sure only one of the elements is zero.
    5112         238 :           if (M0Zero == M1Zero) {
    5113             :             CanFold = false;
    5114             :             break;
    5115             :           }
    5116             : 
    5117             :           assert((M0 >= 0 || M1 >= 0) && "Undef index!");
    5118             : 
    5119             :           // We have a zero and non-zero element. If the non-zero came from
    5120             :           // SV0 make the index a LHS index. If it came from SV1, make it
    5121             :           // a RHS index. We need to mod by NumElts because we don't care
    5122             :           // which operand it came from in the original shuffles.
    5123         233 :           Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
    5124             :         }
    5125             : 
    5126          66 :         if (CanFold) {
    5127          61 :           SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
    5128          61 :           SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
    5129             : 
    5130         122 :           bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
    5131          61 :           if (!LegalMask) {
    5132             :             std::swap(NewLHS, NewRHS);
    5133             :             ShuffleVectorSDNode::commuteMask(Mask);
    5134           0 :             LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
    5135             :           }
    5136             : 
    5137          61 :           if (LegalMask)
    5138         122 :             return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
    5139             :         }
    5140             :       }
    5141             :     }
    5142             :   }
    5143             : 
    5144             :   // fold (or c1, c2) -> c1|c2
    5145             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    5146             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    5147      142974 :   if (N0C && N1C && !N1C->isOpaque())
    5148         182 :     return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
    5149             :   // canonicalize constant to RHS
    5150      144457 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    5151        1574 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    5152        3156 :     return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
    5153             :   // fold (or x, 0) -> x
    5154      141315 :   if (isNullConstant(N1))
    5155         831 :     return N0;
    5156             :   // fold (or x, -1) -> -1
    5157      140484 :   if (isAllOnesConstant(N1))
    5158          85 :     return N1;
    5159             : 
    5160      140399 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    5161          54 :     return NewSel;
    5162             : 
    5163             :   // fold (or x, c) -> c iff (x & ~c) == 0
    5164      408731 :   if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
    5165           2 :     return N1;
    5166             : 
    5167      140343 :   if (SDValue Combined = visitORLike(N0, N1, N))
    5168         330 :     return Combined;
    5169             : 
    5170             :   // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
    5171      140013 :   if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
    5172           8 :     return BSwap;
    5173      140005 :   if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
    5174           7 :     return BSwap;
    5175             : 
    5176             :   // reassociate or
    5177      183692 :   if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags()))
    5178          67 :     return ROR;
    5179             : 
    5180             :   // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
    5181             :   // iff (c1 & c2) != 0.
    5182             :   auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    5183             :     return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
    5184             :   };
    5185      163823 :   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
    5186      187690 :       ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
    5187          25 :     if (SDValue COR = DAG.FoldConstantArithmetic(
    5188          50 :             ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
    5189          25 :       SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
    5190          25 :       AddToWorklist(IOR.getNode());
    5191          50 :       return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
    5192             :     }
    5193             :   }
    5194             : 
    5195             :   // Simplify: (or (op x...), (op y...))  -> (op (or x, y))
    5196      139906 :   if (N0.getOpcode() == N1.getOpcode())
    5197       21193 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    5198         430 :       return Tmp;
    5199             : 
    5200             :   // See if this is some rotate idiom.
    5201      278952 :   if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
    5202        1112 :     return SDValue(Rot, 0);
    5203             : 
    5204      138364 :   if (SDValue Load = MatchLoadCombine(N))
    5205         184 :     return Load;
    5206             : 
    5207             :   // Simplify the operands using demanded-bits information.
    5208      138180 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    5209        6401 :     return SDValue(N, 0);
    5210             : 
    5211      131779 :   return SDValue();
    5212             : }
    5213             : 
    5214      267987 : static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
    5215      299677 :   if (Op.getOpcode() == ISD::AND &&
    5216       31690 :       DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
    5217       27458 :     Mask = Op.getOperand(1);
    5218       27458 :     return Op.getOperand(0);
    5219             :   }
    5220      240529 :   return Op;
    5221             : }
    5222             : 
    5223             : /// Match "(X shl/srl V1) & V2" where V2 may not be present.
    5224      229164 : static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
    5225             :                             SDValue &Mask) {
    5226      229164 :   Op = stripConstantMask(DAG, Op, Mask);
    5227      229164 :   if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
    5228       38815 :     Shift = Op;
    5229       38815 :     return true;
    5230             :   }
    5231             :   return false;
    5232             : }
    5233             : 
    5234             : /// Helper function for visitOR to extract the needed side of a rotate idiom
    5235             : /// from a shl/srl/mul/udiv.  This is meant to handle cases where
    5236             : /// InstCombine merged some outside op with one of the shifts from
    5237             : /// the rotate pattern.
    5238             : /// \returns An empty \c SDValue if the needed shift couldn't be extracted.
    5239             : /// Otherwise, returns an expansion of \p ExtractFrom based on the following
    5240             : /// patterns:
    5241             : ///
    5242             : ///   (or (mul v c0) (shrl (mul v c1) c2)):
    5243             : ///     expands (mul v c0) -> (shl (mul v c1) c3)
    5244             : ///
    5245             : ///   (or (udiv v c0) (shl (udiv v c1) c2)):
    5246             : ///     expands (udiv v c0) -> (shrl (udiv v c1) c3)
    5247             : ///
    5248             : ///   (or (shl v c0) (shrl (shl v c1) c2)):
    5249             : ///     expands (shl v c0) -> (shl (shl v c1) c3)
    5250             : ///
    5251             : ///   (or (shrl v c0) (shl (shrl v c1) c2)):
    5252             : ///     expands (shrl v c0) -> (shrl (shrl v c1) c3)
    5253             : ///
    5254             : /// Such that in all cases, c3+c2==bitwidth(op v c1).
    5255           0 : static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
    5256             :                                      SDValue ExtractFrom, SDValue &Mask,
    5257             :                                      const SDLoc &DL) {
    5258             :   assert(OppShift && ExtractFrom && "Empty SDValue");
    5259             :   assert(
    5260             :       (OppShift.getOpcode() == ISD::SHL || OppShift.getOpcode() == ISD::SRL) &&
    5261             :       "Existing shift must be valid as a rotate half");
    5262             : 
    5263           0 :   ExtractFrom = stripConstantMask(DAG, ExtractFrom, Mask);
    5264             :   // Preconditions:
    5265             :   //    (or (op0 v c0) (shiftl/r (op0 v c1) c2))
    5266             :   //
    5267             :   // Find opcode of the needed shift to be extracted from (op0 v c0).
    5268           0 :   unsigned Opcode = ISD::DELETED_NODE;
    5269           0 :   bool IsMulOrDiv = false;
    5270             :   // Set Opcode and IsMulOrDiv if the extract opcode matches the needed shift
    5271             :   // opcode or its arithmetic (mul or udiv) variant.
    5272             :   auto SelectOpcode = [&](unsigned NeededShift, unsigned MulOrDivVariant) {
    5273           0 :     IsMulOrDiv = ExtractFrom.getOpcode() == MulOrDivVariant;
    5274           0 :     if (!IsMulOrDiv && ExtractFrom.getOpcode() != NeededShift)
    5275             :       return false;
    5276           0 :     Opcode = NeededShift;
    5277             :     return true;
    5278             :   };
    5279             :   // op0 must be either the needed shift opcode or the mul/udiv equivalent
    5280             :   // that the needed shift can be extracted from.
    5281           0 :   if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) &&
    5282             :       (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV)))
    5283           0 :     return SDValue();
    5284             : 
    5285             :   // op0 must be the same opcode on both sides, have the same LHS argument,
    5286             :   // and produce the same value type.
    5287           0 :   SDValue OppShiftLHS = OppShift.getOperand(0);
    5288           0 :   EVT ShiftedVT = OppShiftLHS.getValueType();
    5289           0 :   if (OppShiftLHS.getOpcode() != ExtractFrom.getOpcode() ||
    5290           0 :       OppShiftLHS.getOperand(0) != ExtractFrom.getOperand(0) ||
    5291           0 :       ShiftedVT != ExtractFrom.getValueType())
    5292           0 :     return SDValue();
    5293             : 
    5294             :   // Amount of the existing shift.
    5295           0 :   ConstantSDNode *OppShiftCst = isConstOrConstSplat(OppShift.getOperand(1));
    5296             :   // Constant mul/udiv/shift amount from the RHS of the shift's LHS op.
    5297           0 :   ConstantSDNode *OppLHSCst = isConstOrConstSplat(OppShiftLHS.getOperand(1));
    5298             :   // Constant mul/udiv/shift amount from the RHS of the ExtractFrom op.
    5299             :   ConstantSDNode *ExtractFromCst =
    5300           0 :       isConstOrConstSplat(ExtractFrom.getOperand(1));
    5301             :   // TODO: We should be able to handle non-uniform constant vectors for these values
    5302             :   // Check that we have constant values.
    5303           0 :   if (!OppShiftCst || !OppShiftCst->getAPIntValue() ||
    5304           0 :       !OppLHSCst || !OppLHSCst->getAPIntValue() ||
    5305           0 :       !ExtractFromCst || !ExtractFromCst->getAPIntValue())
    5306           0 :     return SDValue();
    5307             : 
    5308             :   // Compute the shift amount we need to extract to complete the rotate.
    5309             :   const unsigned VTWidth = ShiftedVT.getScalarSizeInBits();
    5310           0 :   if (OppShiftCst->getAPIntValue().ugt(VTWidth))
    5311           0 :     return SDValue();
    5312           0 :   APInt NeededShiftAmt = VTWidth - OppShiftCst->getAPIntValue();
    5313             :   // Normalize the bitwidth of the two mul/udiv/shift constant operands.
    5314           0 :   APInt ExtractFromAmt = ExtractFromCst->getAPIntValue();
    5315           0 :   APInt OppLHSAmt = OppLHSCst->getAPIntValue();
    5316           0 :   zeroExtendToMatch(ExtractFromAmt, OppLHSAmt);
    5317             : 
    5318             :   // Now try extract the needed shift from the ExtractFrom op and see if the
    5319             :   // result matches up with the existing shift's LHS op.
    5320           0 :   if (IsMulOrDiv) {
    5321             :     // Op to extract from is a mul or udiv by a constant.
    5322             :     // Check:
    5323             :     //     c2 / (1 << (bitwidth(op0 v c0) - c1)) == c0
    5324             :     //     c2 % (1 << (bitwidth(op0 v c0) - c1)) == 0
    5325             :     const APInt ExtractDiv = APInt::getOneBitSet(ExtractFromAmt.getBitWidth(),
    5326           0 :                                                  NeededShiftAmt.getZExtValue());
    5327             :     APInt ResultAmt;
    5328             :     APInt Rem;
    5329           0 :     APInt::udivrem(ExtractFromAmt, ExtractDiv, ResultAmt, Rem);
    5330           0 :     if (Rem != 0 || ResultAmt != OppLHSAmt)
    5331           0 :       return SDValue();
    5332             :   } else {
    5333             :     // Op to extract from is a shift by a constant.
    5334             :     // Check:
    5335             :     //      c2 - (bitwidth(op0 v c0) - c1) == c0
    5336           0 :     if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.zextOrTrunc(
    5337             :                                           ExtractFromAmt.getBitWidth()))
    5338           0 :       return SDValue();
    5339             :   }
    5340             : 
    5341             :   // Return the expanded shift op that should allow a rotate to be formed.
    5342           0 :   EVT ShiftVT = OppShift.getOperand(1).getValueType();
    5343           0 :   EVT ResVT = ExtractFrom.getValueType();
    5344           0 :   SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
    5345           0 :   return DAG.getNode(Opcode, DL, ResVT, OppShiftLHS, NewShiftNode);
    5346             : }
    5347             : 
    5348             : // Return true if we can prove that, whenever Neg and Pos are both in the
    5349             : // range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos).  This means that
    5350             : // for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
    5351             : //
    5352             : //     (or (shift1 X, Neg), (shift2 X, Pos))
    5353             : //
    5354             : // reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
    5355             : // in direction shift1 by Neg.  The range [0, EltSize) means that we only need
    5356             : // to consider shift amounts with defined behavior.
    5357         546 : static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
    5358             :                            SelectionDAG &DAG) {
    5359             :   // If EltSize is a power of 2 then:
    5360             :   //
    5361             :   //  (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
    5362             :   //  (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
    5363             :   //
    5364             :   // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
    5365             :   // for the stronger condition:
    5366             :   //
    5367             :   //     Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1)    [A]
    5368             :   //
    5369             :   // for all Neg and Pos.  Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
    5370             :   // we can just replace Neg with Neg' for the rest of the function.
    5371             :   //
    5372             :   // In other cases we check for the even stronger condition:
    5373             :   //
    5374             :   //     Neg == EltSize - Pos                                    [B]
    5375             :   //
    5376             :   // for all Neg and Pos.  Note that the (or ...) then invokes undefined
    5377             :   // behavior if Pos == 0 (and consequently Neg == EltSize).
    5378             :   //
    5379             :   // We could actually use [A] whenever EltSize is a power of 2, but the
    5380             :   // only extra cases that it would match are those uninteresting ones
    5381             :   // where Neg and Pos are never in range at the same time.  E.g. for
    5382             :   // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
    5383             :   // as well as (sub 32, Pos), but:
    5384             :   //
    5385             :   //     (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
    5386             :   //
    5387             :   // always invokes undefined behavior for 32-bit X.
    5388             :   //
    5389             :   // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
    5390             :   unsigned MaskLoBits = 0;
    5391         546 :   if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
    5392          83 :     if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
    5393          83 :       KnownBits Known;
    5394          83 :       DAG.computeKnownBits(Neg.getOperand(0), Known);
    5395             :       unsigned Bits = Log2_64(EltSize);
    5396         246 :       if (NegC->getAPIntValue().getActiveBits() <= Bits &&
    5397         243 :           ((NegC->getAPIntValue() | Known.Zero).countTrailingOnes() >= Bits)) {
    5398          78 :         Neg = Neg.getOperand(0);
    5399             :         MaskLoBits = Bits;
    5400             :       }
    5401             :     }
    5402             :   }
    5403             : 
    5404             :   // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
    5405         546 :   if (Neg.getOpcode() != ISD::SUB)
    5406             :     return false;
    5407         400 :   ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
    5408         400 :   if (!NegC)
    5409             :     return false;
    5410         400 :   SDValue NegOp1 = Neg.getOperand(1);
    5411             : 
    5412             :   // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
    5413             :   // Pos'.  The truncation is redundant for the purpose of the equality.
    5414         400 :   if (MaskLoBits && Pos.getOpcode() == ISD::AND) {
    5415          54 :     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1))) {
    5416          54 :       KnownBits Known;
    5417          54 :       DAG.computeKnownBits(Pos.getOperand(0), Known);
    5418         162 :       if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
    5419         162 :           ((PosC->getAPIntValue() | Known.Zero).countTrailingOnes() >=
    5420             :            MaskLoBits))
    5421          42 :         Pos = Pos.getOperand(0);
    5422             :     }
    5423             :   }
    5424             : 
    5425             :   // The condition we need is now:
    5426             :   //
    5427             :   //     (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
    5428             :   //
    5429             :   // If NegOp1 == Pos then we need:
    5430             :   //
    5431             :   //              EltSize & Mask == NegC & Mask
    5432             :   //
    5433             :   // (because "x & Mask" is a truncation and distributes through subtraction).
    5434             :   APInt Width;
    5435             :   if (Pos == NegOp1)
    5436         780 :     Width = NegC->getAPIntValue();
    5437             : 
    5438             :   // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
    5439             :   // Then the condition we want to prove becomes:
    5440             :   //
    5441             :   //     (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
    5442             :   //
    5443             :   // which, again because "x & Mask" is a truncation, becomes:
    5444             :   //
    5445             :   //                NegC & Mask == (EltSize - PosC) & Mask
    5446             :   //             EltSize & Mask == (NegC + PosC) & Mask
    5447          10 :   else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
    5448           3 :     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
    5449           9 :       Width = PosC->getAPIntValue() + NegC->getAPIntValue();
    5450             :     else
    5451             :       return false;
    5452             :   } else
    5453             :     return false;
    5454             : 
    5455             :   // Now we just need to check that EltSize & Mask == Width & Mask.
    5456         393 :   if (MaskLoBits)
    5457             :     // EltSize & Mask is 0 since Mask is EltSize - 1.
    5458          60 :     return Width.getLoBits(MaskLoBits) == 0;
    5459         333 :   return Width == EltSize;
    5460             : }
    5461             : 
    5462             : // A subroutine of MatchRotate used once we have found an OR of two opposite
    5463             : // shifts of Shifted.  If Neg == <operand size> - Pos then the OR reduces
    5464             : // to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
    5465             : // former being preferred if supported.  InnerPos and InnerNeg are Pos and
    5466             : // Neg with outer conversions stripped away.
    5467           0 : SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
    5468             :                                        SDValue Neg, SDValue InnerPos,
    5469             :                                        SDValue InnerNeg, unsigned PosOpcode,
    5470             :                                        unsigned NegOpcode, const SDLoc &DL) {
    5471             :   // fold (or (shl x, (*ext y)),
    5472             :   //          (srl x, (*ext (sub 32, y)))) ->
    5473             :   //   (rotl x, y) or (rotr x, (sub 32, y))
    5474             :   //
    5475             :   // fold (or (shl x, (*ext (sub 32, y))),
    5476             :   //          (srl x, (*ext y))) ->
    5477             :   //   (rotr x, y) or (rotl x, (sub 32, y))
    5478           0 :   EVT VT = Shifted.getValueType();
    5479           0 :   if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
    5480           0 :     bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
    5481           0 :     return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
    5482           0 :                        HasPos ? Pos : Neg).getNode();
    5483             :   }
    5484             : 
    5485             :   return nullptr;
    5486             : }
    5487             : 
    5488             : // MatchRotate - Handle an 'or' of two operands.  If this is one of the many
    5489             : // idioms for rotate, and if the target supports rotation instructions, generate
    5490             : // a rot[lr].
    5491      139633 : SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
    5492             :   // Must be a legal type.  Expanded 'n promoted things won't work with rotates.
    5493      139633 :   EVT VT = LHS.getValueType();
    5494      139633 :   if (!TLI.isTypeLegal(VT)) return nullptr;
    5495             : 
    5496             :   // The target must have at least one rotate flavor.
    5497      136993 :   bool HasROTL = hasOperation(ISD::ROTL, VT);
    5498      136993 :   bool HasROTR = hasOperation(ISD::ROTR, VT);
    5499      136993 :   if (!HasROTL && !HasROTR) return nullptr;
    5500             : 
    5501             :   // Check for truncated rotate.
    5502      114583 :   if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
    5503         314 :       LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
    5504             :     assert(LHS.getValueType() == RHS.getValueType());
    5505         157 :     if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
    5506           1 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
    5507           1 :                          SDValue(Rot, 0)).getNode();
    5508             :     }
    5509             :   }
    5510             : 
    5511             :   // Match "(X shl/srl V1) & V2" where V2 may not be present.
    5512      114582 :   SDValue LHSShift;   // The shift.
    5513      114582 :   SDValue LHSMask;    // AND value if any.
    5514      114582 :   matchRotateHalf(DAG, LHS, LHSShift, LHSMask);
    5515             : 
    5516      114582 :   SDValue RHSShift;   // The shift.
    5517      114582 :   SDValue RHSMask;    // AND value if any.
    5518      114582 :   matchRotateHalf(DAG, RHS, RHSShift, RHSMask);
    5519             : 
    5520             :   // If neither side matched a rotate half, bail
    5521      114582 :   if (!LHSShift && !RHSShift)
    5522             :     return nullptr;
    5523             : 
    5524             :   // InstCombine may have combined a constant shl, srl, mul, or udiv with one
    5525             :   // side of the rotate, so try to handle that here. In all cases we need to
    5526             :   // pass the matched shift from the opposite side to compute the opcode and
    5527             :   // needed shift amount to extract.  We still want to do this if both sides
    5528             :   // matched a rotate half because one half may be a potential overshift that
    5529             :   // can be broken down (ie if InstCombine merged two shl or srl ops into a
    5530             :   // single one).
    5531             : 
    5532             :   // Have LHS side of the rotate, try to extract the needed shift from the RHS.
    5533       33089 :   if (LHSShift)
    5534        9980 :     if (SDValue NewRHSShift =
    5535        9980 :             extractShiftForRotate(DAG, LHSShift, RHS, RHSMask, DL))
    5536          12 :       RHSShift = NewRHSShift;
    5537             :   // Have RHS side of the rotate, try to extract the needed shift from the LHS.
    5538       33089 :   if (RHSShift)
    5539       28843 :     if (SDValue NewLHSShift =
    5540       28843 :             extractShiftForRotate(DAG, RHSShift, LHS, LHSMask, DL))
    5541          11 :       LHSShift = NewLHSShift;
    5542             : 
    5543             :   // If a side is still missing, nothing else we can do.
    5544       33089 :   if (!RHSShift || !LHSShift)
    5545             :     return nullptr;
    5546             : 
    5547             :   // At this point we've matched or extracted a shift op on each side.
    5548             : 
    5549             :   if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
    5550             :     return nullptr;   // Not shifting the same value.
    5551             : 
    5552        1340 :   if (LHSShift.getOpcode() == RHSShift.getOpcode())
    5553             :     return nullptr;   // Shifts must disagree.
    5554             : 
    5555             :   // Canonicalize shl to left side in a shl/srl pair.
    5556        1186 :   if (RHSShift.getOpcode() == ISD::SHL) {
    5557             :     std::swap(LHS, RHS);
    5558             :     std::swap(LHSShift, RHSShift);
    5559             :     std::swap(LHSMask, RHSMask);
    5560             :   }
    5561             : 
    5562             :   unsigned EltSizeInBits = VT.getScalarSizeInBits();
    5563        1186 :   SDValue LHSShiftArg = LHSShift.getOperand(0);
    5564        1186 :   SDValue LHSShiftAmt = LHSShift.getOperand(1);
    5565        1186 :   SDValue RHSShiftArg = RHSShift.getOperand(0);
    5566        1186 :   SDValue RHSShiftAmt = RHSShift.getOperand(1);
    5567             : 
    5568             :   // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
    5569             :   // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
    5570             :   auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
    5571             :                                         ConstantSDNode *RHS) {
    5572             :     return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
    5573             :   };
    5574        2372 :   if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
    5575         723 :     SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
    5576        1033 :                               LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
    5577             : 
    5578             :     // If there is an AND of either shifted operand, apply it to the result.
    5579         723 :     if (LHSMask.getNode() || RHSMask.getNode()) {
    5580         104 :       SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
    5581         104 :       SDValue Mask = AllOnes;
    5582             : 
    5583         104 :       if (LHSMask.getNode()) {
    5584         208 :         SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
    5585         104 :         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
    5586         104 :                            DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
    5587             :       }
    5588         104 :       if (RHSMask.getNode()) {
    5589         174 :         SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
    5590          87 :         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
    5591          87 :                            DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
    5592             :       }
    5593             : 
    5594         208 :       Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
    5595             :     }
    5596             : 
    5597             :     return Rot.getNode();
    5598             :   }
    5599             : 
    5600             :   // If there is a mask here, and we have a variable shift, we can't be sure
    5601             :   // that we're masking out the right stuff.
    5602         463 :   if (LHSMask.getNode() || RHSMask.getNode())
    5603             :     return nullptr;
    5604             : 
    5605             :   // If the shift amount is sign/zext/any-extended just peel it off.
    5606         424 :   SDValue LExtOp0 = LHSShiftAmt;
    5607         424 :   SDValue RExtOp0 = RHSShiftAmt;
    5608         424 :   if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
    5609         413 :        LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
    5610         413 :        LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
    5611         511 :        LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
    5612          87 :       (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
    5613          76 :        RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
    5614          76 :        RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
    5615             :        RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
    5616          79 :     LExtOp0 = LHSShiftAmt.getOperand(0);
    5617          79 :     RExtOp0 = RHSShiftAmt.getOperand(0);
    5618             :   }
    5619             : 
    5620         424 :   SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
    5621             :                                    LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
    5622         424 :   if (TryL)
    5623             :     return TryL;
    5624             : 
    5625         122 :   SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
    5626             :                                    RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
    5627         122 :   if (TryR)
    5628          87 :     return TryR;
    5629             : 
    5630             :   return nullptr;
    5631             : }
    5632             : 
    5633             : namespace {
    5634             : 
    5635             : /// Represents known origin of an individual byte in load combine pattern. The
    5636             : /// value of the byte is either constant zero or comes from memory.
    5637             : struct ByteProvider {
    5638             :   // For constant zero providers Load is set to nullptr. For memory providers
    5639             :   // Load represents the node which loads the byte from memory.
    5640             :   // ByteOffset is the offset of the byte in the value produced by the load.
    5641             :   LoadSDNode *Load = nullptr;
    5642             :   unsigned ByteOffset = 0;
    5643             : 
    5644             :   ByteProvider() = default;
    5645             : 
    5646             :   static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
    5647             :     return ByteProvider(Load, ByteOffset);
    5648             :   }
    5649             : 
    5650             :   static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
    5651             : 
    5652           0 :   bool isConstantZero() const { return !Load; }
    5653           0 :   bool isMemory() const { return Load; }
    5654             : 
    5655             :   bool operator==(const ByteProvider &Other) const {
    5656             :     return Other.Load == Load && Other.ByteOffset == ByteOffset;
    5657             :   }
    5658             : 
    5659             : private:
    5660             :   ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
    5661             :       : Load(Load), ByteOffset(ByteOffset) {}
    5662             : };
    5663             : 
    5664             : } // end anonymous namespace
    5665             : 
    5666             : /// Recursively traverses the expression calculating the origin of the requested
    5667             : /// byte of the given value. Returns None if the provider can't be calculated.
    5668             : ///
    5669             : /// For all the values except the root of the expression verifies that the value
    5670             : /// has exactly one use and if it's not true return None. This way if the origin
    5671             : /// of the byte is returned it's guaranteed that the values which contribute to
    5672             : /// the byte are not used outside of this expression.
    5673             : ///
    5674             : /// Because the parts of the expression are not allowed to have more than one
    5675             : /// use this function iterates over trees, not DAGs. So it never visits the same
    5676             : /// node more than once.
    5677             : static const Optional<ByteProvider>
    5678      295983 : calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
    5679             :                       bool Root = false) {
    5680             :   // Typical i64 by i8 pattern requires recursion up to 8 calls depth
    5681      295983 :   if (Depth == 10)
    5682             :     return None;
    5683             : 
    5684      292870 :   if (!Root && !Op.hasOneUse())
    5685             :     return None;
    5686             : 
    5687             :   assert(Op.getValueType().isScalarInteger() && "can't handle other types");
    5688      246423 :   unsigned BitWidth = Op.getValueSizeInBits();
    5689      246423 :   if (BitWidth % 8 != 0)
    5690             :     return None;
    5691      246423 :   unsigned ByteWidth = BitWidth / 8;
    5692             :   assert(Index < ByteWidth && "invalid index requested");
    5693             :   (void) ByteWidth;
    5694             : 
    5695      492846 :   switch (Op.getOpcode()) {
    5696      155883 :   case ISD::OR: {
    5697      311766 :     auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
    5698      155883 :     if (!LHS)
    5699             :       return None;
    5700       32530 :     auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
    5701       16265 :     if (!RHS)
    5702             :       return None;
    5703             : 
    5704       12484 :     if (LHS->isConstantZero())
    5705        6197 :       return RHS;
    5706        6287 :     if (RHS->isConstantZero())
    5707        4904 :       return LHS;
    5708             :     return None;
    5709             :   }
    5710       15815 :   case ISD::SHL: {
    5711       15815 :     auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
    5712             :     if (!ShiftOp)
    5713             :       return None;
    5714             : 
    5715       15318 :     uint64_t BitShift = ShiftOp->getZExtValue();
    5716       15318 :     if (BitShift % 8 != 0)
    5717             :       return None;
    5718       12477 :     uint64_t ByteShift = BitShift / 8;
    5719             : 
    5720       12477 :     return Index < ByteShift
    5721        6866 :                ? ByteProvider::getConstantZero()
    5722             :                : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
    5723       12477 :                                        Depth + 1);
    5724             :   }
    5725       12199 :   case ISD::ANY_EXTEND:
    5726             :   case ISD::SIGN_EXTEND:
    5727             :   case ISD::ZERO_EXTEND: {
    5728       12199 :     SDValue NarrowOp = Op->getOperand(0);
    5729       12199 :     unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
    5730       12199 :     if (NarrowBitWidth % 8 != 0)
    5731             :       return None;
    5732       12129 :     uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    5733             : 
    5734       12129 :     if (Index >= NarrowByteWidth)
    5735        1581 :       return Op.getOpcode() == ISD::ZERO_EXTEND
    5736        1581 :                  ? Optional<ByteProvider>(ByteProvider::getConstantZero())
    5737        1581 :                  : None;
    5738       10548 :     return calculateByteProvider(NarrowOp, Index, Depth + 1);
    5739             :   }
    5740          42 :   case ISD::BSWAP:
    5741          84 :     return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
    5742          84 :                                  Depth + 1);
    5743       16895 :   case ISD::LOAD: {
    5744             :     auto L = cast<LoadSDNode>(Op.getNode());
    5745       16895 :     if (L->isVolatile() || L->isIndexed())
    5746             :       return None;
    5747             : 
    5748       16809 :     unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
    5749       16809 :     if (NarrowBitWidth % 8 != 0)
    5750             :       return None;
    5751       16809 :     uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    5752             : 
    5753       16809 :     if (Index >= NarrowByteWidth)
    5754             :       return L->getExtensionType() == ISD::ZEXTLOAD
    5755        4431 :                  ? Optional<ByteProvider>(ByteProvider::getConstantZero())
    5756        4549 :                  : None;
    5757       12260 :     return ByteProvider::getMemory(L, Index);
    5758             :   }
    5759             :   }
    5760             : 
    5761             :   return None;
    5762             : }
    5763             : 
    5764             : /// Match a pattern where a wide type scalar value is loaded by several narrow
    5765             : /// loads and combined by shifts and ors. Fold it into a single load or a load
    5766             : /// and a BSWAP if the targets supports it.
    5767             : ///
    5768             : /// Assuming little endian target:
    5769             : ///  i8 *a = ...
    5770             : ///  i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
    5771             : /// =>
    5772             : ///  i32 val = *((i32)a)
    5773             : ///
    5774             : ///  i8 *a = ...
    5775             : ///  i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
    5776             : /// =>
    5777             : ///  i32 val = BSWAP(*((i32)a))
    5778             : ///
    5779             : /// TODO: This rule matches complex patterns with OR node roots and doesn't
    5780             : /// interact well with the worklist mechanism. When a part of the pattern is
    5781             : /// updated (e.g. one of the loads) its direct users are put into the worklist,
    5782             : /// but the root node of the pattern which triggers the load combine is not
    5783             : /// necessarily a direct user of the changed node. For example, once the address
    5784             : /// of t28 load is reassociated load combine won't be triggered:
    5785             : ///             t25: i32 = add t4, Constant:i32<2>
    5786             : ///           t26: i64 = sign_extend t25
    5787             : ///        t27: i64 = add t2, t26
    5788             : ///       t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
    5789             : ///     t29: i32 = zero_extend t28
    5790             : ///   t32: i32 = shl t29, Constant:i8<8>
    5791             : /// t33: i32 = or t23, t32
    5792             : /// As a possible fix visitLoad can check if the load can be a part of a load
    5793             : /// combine pattern and add corresponding OR roots to the worklist.
    5794           0 : SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
    5795             :   assert(N->getOpcode() == ISD::OR &&
    5796             :          "Can only match load combining against OR nodes");
    5797             : 
    5798             :   // Handles simple types only
    5799           0 :   EVT VT = N->getValueType(0);
    5800           0 :   if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
    5801           0 :     return SDValue();
    5802           0 :   unsigned ByteWidth = VT.getSizeInBits() / 8;
    5803             : 
    5804           0 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5805             :   // Before legalize we can introduce too wide illegal loads which will be later
    5806             :   // split into legal sized loads. This enables us to combine i64 load by i8
    5807             :   // patterns to a couple of i32 loads on 32 bit targets.
    5808           0 :   if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
    5809           0 :     return SDValue();
    5810             : 
    5811             :   std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
    5812             :     unsigned BW, unsigned i) { return i; };
    5813             :   std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
    5814        4466 :     unsigned BW, unsigned i) { return BW - i - 1; };
    5815             : 
    5816           0 :   bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
    5817             :   auto MemoryByteOffset = [&] (ByteProvider P) {
    5818             :     assert(P.isMemory() && "Must be a memory byte provider");
    5819             :     unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
    5820             :     assert(LoadBitWidth % 8 == 0 &&
    5821             :            "can only analyze providers for individual bytes not bit");
    5822             :     unsigned LoadByteWidth = LoadBitWidth / 8;
    5823             :     return IsBigEndianTarget
    5824             :             ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
    5825             :             : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
    5826           0 :   };
    5827             : 
    5828             :   Optional<BaseIndexOffset> Base;
    5829             :   SDValue Chain;
    5830             : 
    5831             :   SmallPtrSet<LoadSDNode *, 8> Loads;
    5832             :   Optional<ByteProvider> FirstByteProvider;
    5833             :   int64_t FirstOffset = INT64_MAX;
    5834             : 
    5835             :   // Check if all the bytes of the OR we are looking at are loaded from the same
    5836             :   // base address. Collect bytes offsets from Base address in ByteOffsets.
    5837           0 :   SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
    5838           0 :   for (unsigned i = 0; i < ByteWidth; i++) {
    5839           0 :     auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
    5840           0 :     if (!P || !P->isMemory()) // All the bytes must be loaded from memory
    5841           0 :       return SDValue();
    5842             : 
    5843             :     LoadSDNode *L = P->Load;
    5844             :     assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&
    5845             :            "Must be enforced by calculateByteProvider");
    5846             :     assert(L->getOffset().isUndef() && "Unindexed load must have undef offset");
    5847             : 
    5848             :     // All loads must share the same chain
    5849           0 :     SDValue LChain = L->getChain();
    5850           0 :     if (!Chain)
    5851           0 :       Chain = LChain;
    5852             :     else if (Chain != LChain)
    5853           0 :       return SDValue();
    5854             : 
    5855             :     // Loads must share the same base address
    5856           0 :     BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
    5857           0 :     int64_t ByteOffsetFromBase = 0;
    5858           0 :     if (!Base)
    5859             :       Base = Ptr;
    5860           0 :     else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
    5861           0 :       return SDValue();
    5862             : 
    5863             :     // Calculate the offset of the current byte from the base address
    5864           0 :     ByteOffsetFromBase += MemoryByteOffset(*P);
    5865           0 :     ByteOffsets[i] = ByteOffsetFromBase;
    5866             : 
    5867             :     // Remember the first byte load
    5868           0 :     if (ByteOffsetFromBase < FirstOffset) {
    5869             :       FirstByteProvider = P;
    5870             :       FirstOffset = ByteOffsetFromBase;
    5871             :     }
    5872             : 
    5873           0 :     Loads.insert(L);
    5874             :   }
    5875             :   assert(!Loads.empty() && "All the bytes of the value must be loaded from "
    5876             :          "memory, so there must be at least one load which produces the value");
    5877             :   assert(Base && "Base address of the accessed memory location must be set");
    5878             :   assert(FirstOffset != INT64_MAX && "First byte offset must be set");
    5879             : 
    5880             :   // Check if the bytes of the OR we are looking at match with either big or
    5881             :   // little endian value load
    5882             :   bool BigEndian = true, LittleEndian = true;
    5883           0 :   for (unsigned i = 0; i < ByteWidth; i++) {
    5884           0 :     int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
    5885           0 :     LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
    5886           0 :     BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
    5887           0 :     if (!BigEndian && !LittleEndian)
    5888           0 :       return SDValue();
    5889             :   }
    5890             :   assert((BigEndian != LittleEndian) && "should be either or");
    5891             :   assert(FirstByteProvider && "must be set");
    5892             : 
    5893             :   // Ensure that the first byte is loaded from zero offset of the first load.
    5894             :   // So the combined value can be loaded from the first load address.
    5895           0 :   if (MemoryByteOffset(*FirstByteProvider) != 0)
    5896           0 :     return SDValue();
    5897             :   LoadSDNode *FirstLoad = FirstByteProvider->Load;
    5898             : 
    5899             :   // The node we are looking at matches with the pattern, check if we can
    5900             :   // replace it with a single load and bswap if needed.
    5901             : 
    5902             :   // If the load needs byte swap check if the target supports it
    5903           0 :   bool NeedsBswap = IsBigEndianTarget != BigEndian;
    5904             : 
    5905             :   // Before legalize we can introduce illegal bswaps which will be later
    5906             :   // converted to an explicit bswap sequence. This way we end up with a single
    5907             :   // load and byte shuffling instead of several loads and byte shuffling.
    5908           0 :   if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
    5909           0 :     return SDValue();
    5910             : 
    5911             :   // Check that a load of the wide type is both allowed and fast on the target
    5912           0 :   bool Fast = false;
    5913           0 :   bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
    5914             :                                         VT, FirstLoad->getAddressSpace(),
    5915             :                                         FirstLoad->getAlignment(), &Fast);
    5916           0 :   if (!Allowed || !Fast)
    5917           0 :     return SDValue();
    5918             : 
    5919             :   SDValue NewLoad =
    5920           0 :       DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
    5921           0 :                   FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
    5922             : 
    5923             :   // Transfer chain users from old loads to the new load.
    5924           0 :   for (LoadSDNode *L : Loads)
    5925           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
    5926             : 
    5927           0 :   return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
    5928             : }
    5929             : 
    5930             : // If the target has andn, bsl, or a similar bit-select instruction,
    5931             : // we want to unfold masked merge, with canonical pattern of:
    5932             : //   |        A  |  |B|
    5933             : //   ((x ^ y) & m) ^ y
    5934             : //    |  D  |
    5935             : // Into:
    5936             : //   (x & m) | (y & ~m)
    5937             : // If y is a constant, and the 'andn' does not work with immediates,
    5938             : // we unfold into a different pattern:
    5939             : //   ~(~x & m) & (m | y)
    5940             : // NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
    5941             : //       the very least that breaks andnpd / andnps patterns, and because those
    5942             : //       patterns are simplified in IR and shouldn't be created in the DAG
    5943           0 : SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
    5944             :   assert(N->getOpcode() == ISD::XOR);
    5945             : 
    5946             :   // Don't touch 'not' (i.e. where y = -1).
    5947           0 :   if (isAllOnesConstantOrAllOnesSplatConstant(N->getOperand(1)))
    5948           0 :     return SDValue();
    5949             : 
    5950           0 :   EVT VT = N->getValueType(0);
    5951             : 
    5952             :   // There are 3 commutable operators in the pattern,
    5953             :   // so we have to deal with 8 possible variants of the basic pattern.
    5954           0 :   SDValue X, Y, M;
    5955             :   auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
    5956             :     if (And.getOpcode() != ISD::AND || !And.hasOneUse())
    5957             :       return false;
    5958             :     SDValue Xor = And.getOperand(XorIdx);
    5959             :     if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
    5960             :       return false;
    5961             :     SDValue Xor0 = Xor.getOperand(0);
    5962             :     SDValue Xor1 = Xor.getOperand(1);
    5963             :     // Don't touch 'not' (i.e. where y = -1).
    5964             :     if (isAllOnesConstantOrAllOnesSplatConstant(Xor1))
    5965             :       return false;
    5966             :     if (Other == Xor0)
    5967             :       std::swap(Xor0, Xor1);
    5968             :     if (Other != Xor1)
    5969             :       return false;
    5970             :     X = Xor0;
    5971             :     Y = Xor1;
    5972             :     M = And.getOperand(XorIdx ? 0 : 1);
    5973             :     return true;
    5974           0 :   };
    5975             : 
    5976           0 :   SDValue N0 = N->getOperand(0);
    5977           0 :   SDValue N1 = N->getOperand(1);
    5978           0 :   if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
    5979           0 :       !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
    5980           0 :     return SDValue();
    5981             : 
    5982             :   // Don't do anything if the mask is constant. This should not be reachable.
    5983             :   // InstCombine should have already unfolded this pattern, and DAGCombiner
    5984             :   // probably shouldn't produce it, too.
    5985           0 :   if (isa<ConstantSDNode>(M.getNode()))
    5986           0 :     return SDValue();
    5987             : 
    5988             :   // We can transform if the target has AndNot
    5989           0 :   if (!TLI.hasAndNot(M))
    5990           0 :     return SDValue();
    5991             : 
    5992             :   SDLoc DL(N);
    5993             : 
    5994             :   // If Y is a constant, check that 'andn' works with immediates.
    5995           0 :   if (!TLI.hasAndNot(Y)) {
    5996             :     assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.");
    5997             :     // If not, we need to do a bit more work to make sure andn is still used.
    5998           0 :     SDValue NotX = DAG.getNOT(DL, X, VT);
    5999           0 :     SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
    6000           0 :     SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
    6001           0 :     SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
    6002           0 :     return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
    6003             :   }
    6004             : 
    6005           0 :   SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
    6006           0 :   SDValue NotM = DAG.getNOT(DL, M, VT);
    6007           0 :   SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
    6008             : 
    6009           0 :   return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
    6010             : }
    6011             : 
    6012      115968 : SDValue DAGCombiner::visitXOR(SDNode *N) {
    6013      115968 :   SDValue N0 = N->getOperand(0);
    6014      115968 :   SDValue N1 = N->getOperand(1);
    6015      231936 :   EVT VT = N0.getValueType();
    6016             : 
    6017             :   // fold vector ops
    6018      115968 :   if (VT.isVector()) {
    6019       23689 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6020           0 :       return FoldedVOp;
    6021             : 
    6022             :     // fold (xor x, 0) -> x, vector edition
    6023       23689 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    6024          22 :       return N1;
    6025       23667 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    6026          14 :       return N0;
    6027             :   }
    6028             : 
    6029             :   // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
    6030      115932 :   if (N0.isUndef() && N1.isUndef())
    6031           0 :     return DAG.getConstant(0, SDLoc(N), VT);
    6032             :   // fold (xor x, undef) -> undef
    6033      115932 :   if (N0.isUndef())
    6034          14 :     return N0;
    6035      231836 :   if (N1.isUndef())
    6036           0 :     return N1;
    6037             :   // fold (xor c1, c2) -> c1^c2
    6038             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6039             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    6040      115918 :   if (N0C && N1C)
    6041        1808 :     return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
    6042             :   // canonicalize constant to RHS
    6043      115596 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    6044         582 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    6045        1158 :     return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
    6046             :   // fold (xor x, 0) -> x
    6047      114435 :   if (isNullConstant(N1))
    6048           0 :     return N0;
    6049             : 
    6050      114435 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6051           6 :     return NewSel;
    6052             : 
    6053             :   // reassociate xor
    6054      164065 :   if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1, N->getFlags()))
    6055         186 :     return RXOR;
    6056             : 
    6057             :   // fold !(x cc y) -> (x !cc y)
    6058      114243 :   SDValue LHS, RHS, CC;
    6059      114243 :   if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
    6060       22627 :     bool isInt = LHS.getValueType().isInteger();
    6061       45254 :     ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
    6062             :                                                isInt);
    6063             : 
    6064       22627 :     if (!LegalOperations ||
    6065         121 :         TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
    6066       22513 :       switch (N0.getOpcode()) {
    6067           0 :       default:
    6068           0 :         llvm_unreachable("Unhandled SetCC Equivalent!");
    6069       22513 :       case ISD::SETCC:
    6070       58634 :         return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
    6071           0 :       case ISD::SELECT_CC:
    6072           0 :         return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
    6073           0 :                                N0.getOperand(3), NotCC);
    6074             :       }
    6075             :     }
    6076             :   }
    6077             : 
    6078             :   // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
    6079      133812 :   if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
    6080       91737 :       N0.getNode()->hasOneUse() &&
    6081           7 :       isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
    6082           3 :     SDValue V = N0.getOperand(0);
    6083             :     SDLoc DL(N0);
    6084           3 :     V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
    6085           3 :                     DAG.getConstant(1, DL, V.getValueType()));
    6086           3 :     AddToWorklist(V.getNode());
    6087           6 :     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
    6088             :   }
    6089             : 
    6090             :   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
    6091      111335 :   if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
    6092       19291 :       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
    6093          82 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    6094          82 :     if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
    6095          53 :       unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
    6096          55 :       LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
    6097          55 :       RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
    6098          53 :       AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
    6099         108 :       return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
    6100             :     }
    6101             :   }
    6102             :   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
    6103       91674 :   if (isAllOnesConstant(N1) && N0.hasOneUse() &&
    6104       46554 :       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
    6105         611 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    6106             :     if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
    6107          60 :       unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
    6108          60 :       LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
    6109          60 :       RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
    6110          60 :       AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
    6111         120 :       return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
    6112             :     }
    6113             :   }
    6114             :   // fold (xor (and x, y), y) -> (and (not x), y)
    6115      183228 :   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
    6116        1828 :       N0->getOperand(1) == N1) {
    6117          73 :     SDValue X = N0->getOperand(0);
    6118          88 :     SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
    6119          73 :     AddToWorklist(NotX.getNode());
    6120         161 :     return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
    6121             :   }
    6122             : 
    6123             :   // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
    6124       91541 :   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
    6125       17008 :     SDValue A = N0.getOpcode() == ISD::ADD ? N0 : N1;
    6126       33916 :     SDValue S = N0.getOpcode() == ISD::SRA ? N0 : N1;
    6127       17008 :     if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
    6128         119 :       SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
    6129         119 :       SDValue S0 = S.getOperand(0);
    6130             :       if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0)) {
    6131             :         unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6132         113 :         if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
    6133         222 :           if (C->getAPIntValue() == (OpSizeInBits - 1))
    6134         225 :             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
    6135             :       }
    6136             :     }
    6137             :   }
    6138             : 
    6139             :   // fold (xor x, x) -> 0
    6140       91430 :   if (N0 == N1)
    6141          38 :     return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
    6142             : 
    6143             :   // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
    6144             :   // Here is a concrete example of this equivalence:
    6145             :   // i16   x ==  14
    6146             :   // i16 shl ==   1 << 14  == 16384 == 0b0100000000000000
    6147             :   // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
    6148             :   //
    6149             :   // =>
    6150             :   //
    6151             :   // i16     ~1      == 0b1111111111111110
    6152             :   // i16 rol(~1, 14) == 0b1011111111111111
    6153             :   //
    6154             :   // Some additional tips to help conceptualize this transform:
    6155             :   // - Try to see the operation as placing a single zero in a value of all ones.
    6156             :   // - There exists no value for x which would allow the result to contain zero.
    6157             :   // - Values of x larger than the bitwidth are undefined and do not require a
    6158             :   //   consistent result.
    6159             :   // - Pushing the zero left requires shifting one bits in from the right.
    6160             :   // A rotate left of ~1 is a nice way of achieving the desired result.
    6161      137659 :   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
    6162         626 :       && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
    6163             :     SDLoc DL(N);
    6164          79 :     return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
    6165          79 :                        N0.getOperand(1));
    6166             :   }
    6167             : 
    6168             :   // Simplify: xor (op x...), (op y...)  -> (op (xor x, y))
    6169       91332 :   if (N0.getOpcode() == N1.getOpcode())
    6170       14834 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    6171         435 :       return Tmp;
    6172             : 
    6173             :   // Unfold  ((x ^ y) & m) ^ y  into  (x & m) | (y & ~m)  if profitable
    6174       90897 :   if (SDValue MM = unfoldMaskedMerge(N))
    6175         114 :     return MM;
    6176             : 
    6177             :   // Simplify the expression using non-local knowledge.
    6178       90783 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    6179         306 :     return SDValue(N, 0);
    6180             : 
    6181       90477 :   return SDValue();
    6182             : }
    6183             : 
    6184             : /// Handle transforms common to the three shifts, when the shift amount is a
    6185             : /// constant.
    6186           0 : SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
    6187           0 :   SDNode *LHS = N->getOperand(0).getNode();
    6188           0 :   if (!LHS->hasOneUse()) return SDValue();
    6189             : 
    6190             :   // We want to pull some binops through shifts, so that we have (and (shift))
    6191             :   // instead of (shift (and)), likewise for add, or, xor, etc.  This sort of
    6192             :   // thing happens with address calculations, so it's important to canonicalize
    6193             :   // it.
    6194             :   bool HighBitSet = false;  // Can we transform this if the high bit is set?
    6195             : 
    6196           0 :   switch (LHS->getOpcode()) {
    6197           0 :   default: return SDValue();
    6198             :   case ISD::OR:
    6199             :   case ISD::XOR:
    6200             :     HighBitSet = false; // We can only transform sra if the high bit is clear.
    6201             :     break;
    6202           0 :   case ISD::AND:
    6203             :     HighBitSet = true;  // We can only transform sra if the high bit is set.
    6204           0 :     break;
    6205           0 :   case ISD::ADD:
    6206           0 :     if (N->getOpcode() != ISD::SHL)
    6207           0 :       return SDValue(); // only shl(add) not sr[al](add).
    6208             :     HighBitSet = false; // We can only transform sra if the high bit is clear.
    6209             :     break;
    6210             :   }
    6211             : 
    6212             :   // We require the RHS of the binop to be a constant and not opaque as well.
    6213           0 :   ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
    6214           0 :   if (!BinOpCst) return SDValue();
    6215             : 
    6216             :   // FIXME: disable this unless the input to the binop is a shift by a constant
    6217             :   // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
    6218           0 :   SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
    6219           0 :   bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
    6220           0 :                  BinOpLHSVal->getOpcode() == ISD::SRA ||
    6221             :                  BinOpLHSVal->getOpcode() == ISD::SRL;
    6222           0 :   bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
    6223             :                         BinOpLHSVal->getOpcode() == ISD::SELECT;
    6224             : 
    6225           0 :   if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
    6226             :       !isCopyOrSelect)
    6227           0 :     return SDValue();
    6228             : 
    6229           0 :   if (isCopyOrSelect && N->hasOneUse())
    6230           0 :     return SDValue();
    6231             : 
    6232           0 :   EVT VT = N->getValueType(0);
    6233             : 
    6234             :   // If this is a signed shift right, and the high bit is modified by the
    6235             :   // logical operation, do not perform the transformation. The highBitSet
    6236             :   // boolean indicates the value of the high bit of the constant which would
    6237             :   // cause it to be modified for this operation.
    6238           0 :   if (N->getOpcode() == ISD::SRA) {
    6239           0 :     bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
    6240           0 :     if (BinOpRHSSignSet != HighBitSet)
    6241           0 :       return SDValue();
    6242             :   }
    6243             : 
    6244           0 :   if (!TLI.isDesirableToCommuteWithShift(N, Level))
    6245           0 :     return SDValue();
    6246             : 
    6247             :   // Fold the constants, shifting the binop RHS by the shift amount.
    6248           0 :   SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
    6249             :                                N->getValueType(0),
    6250           0 :                                LHS->getOperand(1), N->getOperand(1));
    6251             :   assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!");
    6252             : 
    6253             :   // Create the new shift.
    6254           0 :   SDValue NewShift = DAG.getNode(N->getOpcode(),
    6255           0 :                                  SDLoc(LHS->getOperand(0)),
    6256           0 :                                  VT, LHS->getOperand(0), N->getOperand(1));
    6257             : 
    6258             :   // Create the new binop.
    6259           0 :   return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
    6260             : }
    6261             : 
    6262         844 : SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
    6263             :   assert(N->getOpcode() == ISD::TRUNCATE);
    6264             :   assert(N->getOperand(0).getOpcode() == ISD::AND);
    6265             : 
    6266             :   // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
    6267         827 :   if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
    6268         587 :     SDValue N01 = N->getOperand(0).getOperand(1);
    6269         587 :     if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
    6270             :       SDLoc DL(N);
    6271         584 :       EVT TruncVT = N->getValueType(0);
    6272         584 :       SDValue N00 = N->getOperand(0).getOperand(0);
    6273        1168 :       SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
    6274        1168 :       SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
    6275         584 :       AddToWorklist(Trunc00.getNode());
    6276         584 :       AddToWorklist(Trunc01.getNode());
    6277        1168 :       return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
    6278             :     }
    6279             :   }
    6280             : 
    6281         260 :   return SDValue();
    6282             : }
    6283             : 
    6284        2760 : SDValue DAGCombiner::visitRotate(SDNode *N) {
    6285             :   SDLoc dl(N);
    6286        2760 :   SDValue N0 = N->getOperand(0);
    6287        2760 :   SDValue N1 = N->getOperand(1);
    6288        5520 :   EVT VT = N->getValueType(0);
    6289             :   unsigned Bitsize = VT.getScalarSizeInBits();
    6290             : 
    6291             :   // fold (rot x, 0) -> x
    6292        2760 :   if (isNullConstantOrNullSplatConstant(N1))
    6293           4 :     return N0;
    6294             : 
    6295             :   // fold (rot x, c) -> (rot x, c % BitSize)
    6296        2756 :   if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
    6297        3303 :     if (Cst->getAPIntValue().uge(Bitsize)) {
    6298          10 :       uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
    6299          10 :       return DAG.getNode(N->getOpcode(), dl, VT, N0,
    6300          10 :                          DAG.getConstant(RotAmt, dl, N1.getValueType()));
    6301             :     }
    6302             :   }
    6303             : 
    6304             :   // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
    6305        2746 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6306         257 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6307          53 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6308         159 :       return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
    6309             :   }
    6310             : 
    6311             :   unsigned NextOp = N0.getOpcode();
    6312             :   // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
    6313        2693 :   if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
    6314          17 :     SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
    6315          34 :     SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
    6316          17 :     if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
    6317          17 :       EVT ShiftVT = C1->getValueType(0);
    6318          17 :       bool SameSide = (N->getOpcode() == NextOp);
    6319          17 :       unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
    6320          17 :       if (SDValue CombinedShift =
    6321          17 :               DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
    6322          17 :         SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
    6323          17 :         SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
    6324             :             ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
    6325          17 :             BitsizeC.getNode());
    6326          34 :         return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
    6327          34 :                            CombinedShiftNorm);
    6328             :       }
    6329             :     }
    6330             :   }
    6331        2676 :   return SDValue();
    6332             : }
    6333             : 
    6334      126522 : SDValue DAGCombiner::visitSHL(SDNode *N) {
    6335      126522 :   SDValue N0 = N->getOperand(0);
    6336      126522 :   SDValue N1 = N->getOperand(1);
    6337      126522 :   EVT VT = N0.getValueType();
    6338             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6339             : 
    6340             :   // fold vector ops
    6341      126522 :   if (VT.isVector()) {
    6342        3776 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6343           6 :       return FoldedVOp;
    6344             : 
    6345             :     BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
    6346             :     // If setcc produces all-one true value then:
    6347             :     // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
    6348        2327 :     if (N1CV && N1CV->isConstant()) {
    6349        2225 :       if (N0.getOpcode() == ISD::AND) {
    6350          19 :         SDValue N00 = N0->getOperand(0);
    6351          19 :         SDValue N01 = N0->getOperand(1);
    6352             :         BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
    6353             : 
    6354          18 :         if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
    6355          12 :             TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
    6356             :                 TargetLowering::ZeroOrNegativeOneBooleanContent) {
    6357           4 :           if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
    6358           8 :                                                      N01CV, N1CV))
    6359           8 :             return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
    6360             :         }
    6361             :       }
    6362             :     }
    6363             :   }
    6364             : 
    6365      126512 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6366             : 
    6367             :   // fold (shl c1, c2) -> c1<<c2
    6368             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6369      126512 :   if (N0C && N1C && !N1C->isOpaque())
    6370        7530 :     return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
    6371             :   // fold (shl 0, x) -> 0
    6372      122747 :   if (isNullConstantOrNullSplatConstant(N0))
    6373          82 :     return N0;
    6374             :   // fold (shl x, c >= size(x)) -> undef
    6375             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6376             :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6377      212246 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6378             :   };
    6379      245330 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6380          49 :     return DAG.getUNDEF(VT);
    6381             :   // fold (shl x, 0) -> x
    6382      227992 :   if (N1C && N1C->isNullValue())
    6383         443 :     return N0;
    6384             :   // fold (shl undef, x) -> 0
    6385      122173 :   if (N0.isUndef())
    6386          22 :     return DAG.getConstant(0, SDLoc(N), VT);
    6387             : 
    6388      122162 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6389          12 :     return NewSel;
    6390             : 
    6391             :   // if (shl x, c) is known to be zero, return 0
    6392      122150 :   if (DAG.MaskedValueIsZero(SDValue(N, 0),
    6393      244300 :                             APInt::getAllOnesValue(OpSizeInBits)))
    6394        4238 :     return DAG.getConstant(0, SDLoc(N), VT);
    6395             :   // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
    6396      120031 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6397        5704 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6398         725 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6399        1022 :       return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
    6400             :   }
    6401             : 
    6402      119547 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6403        6733 :     return SDValue(N, 0);
    6404             : 
    6405             :   // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
    6406      112814 :   if (N0.getOpcode() == ISD::SHL) {
    6407             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    6408             :                                           ConstantSDNode *RHS) {
    6409             :       APInt c1 = LHS->getAPIntValue();
    6410             :       APInt c2 = RHS->getAPIntValue();
    6411             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6412             :       return (c1 + c2).uge(OpSizeInBits);
    6413             :     };
    6414         738 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    6415          12 :       return DAG.getConstant(0, SDLoc(N), VT);
    6416             : 
    6417             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    6418             :                                        ConstantSDNode *RHS) {
    6419             :       APInt c1 = LHS->getAPIntValue();
    6420             :       APInt c2 = RHS->getAPIntValue();
    6421             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6422             :       return (c1 + c2).ult(OpSizeInBits);
    6423             :     };
    6424         726 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    6425             :       SDLoc DL(N);
    6426          90 :       EVT ShiftVT = N1.getValueType();
    6427         180 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    6428         180 :       return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
    6429             :     }
    6430             :   }
    6431             : 
    6432             :   // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
    6433             :   // For this to be valid, the second form must not preserve any of the bits
    6434             :   // that are shifted out by the inner shift in the first form.  This means
    6435             :   // the outer shift size must be >= the number of bits added by the ext.
    6436             :   // As a corollary, we don't care what kind of ext it is.
    6437       95971 :   if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
    6438       82393 :               N0.getOpcode() == ISD::ANY_EXTEND ||
    6439      130507 :               N0.getOpcode() == ISD::SIGN_EXTEND) &&
    6440       17789 :       N0.getOperand(0).getOpcode() == ISD::SHL) {
    6441          47 :     SDValue N0Op0 = N0.getOperand(0);
    6442          47 :     if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6443          39 :       APInt c1 = N0Op0C1->getAPIntValue();
    6444          39 :       APInt c2 = N1C->getAPIntValue();
    6445          39 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6446             : 
    6447          39 :       EVT InnerShiftVT = N0Op0.getValueType();
    6448          39 :       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
    6449          78 :       if (c2.uge(OpSizeInBits - InnerShiftSize)) {
    6450             :         SDLoc DL(N0);
    6451          15 :         APInt Sum = c1 + c2;
    6452          15 :         if (Sum.uge(OpSizeInBits))
    6453           4 :           return DAG.getConstant(0, DL, VT);
    6454             : 
    6455          11 :         return DAG.getNode(
    6456             :             ISD::SHL, DL, VT,
    6457          22 :             DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
    6458          11 :             DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
    6459             :       }
    6460             :     }
    6461             :   }
    6462             : 
    6463             :   // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
    6464             :   // Only fold this if the inner zext has no other uses to avoid increasing
    6465             :   // the total number of instructions.
    6466      122082 :   if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
    6467        8431 :       N0.getOperand(0).getOpcode() == ISD::SRL) {
    6468             :     SDValue N0Op0 = N0.getOperand(0);
    6469         103 :     if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6470         103 :       if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
    6471         103 :         uint64_t c1 = N0Op0C1->getZExtValue();
    6472         103 :         uint64_t c2 = N1C->getZExtValue();
    6473         103 :         if (c1 == c2) {
    6474          14 :           SDValue NewOp0 = N0.getOperand(0);
    6475          28 :           EVT CountVT = NewOp0.getOperand(1).getValueType();
    6476             :           SDLoc DL(N);
    6477          14 :           SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
    6478             :                                        NewOp0,
    6479          14 :                                        DAG.getConstant(c2, DL, CountVT));
    6480          14 :           AddToWorklist(NewSHL.getNode());
    6481          28 :           return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
    6482             :         }
    6483             :       }
    6484             :     }
    6485             :   }
    6486             : 
    6487             :   // fold (shl (sr[la] exact X,  C1), C2) -> (shl    X, (C2-C1)) if C1 <= C2
    6488             :   // fold (shl (sr[la] exact X,  C1), C2) -> (sr[la] X, (C2-C1)) if C1  > C2
    6489      112689 :   if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
    6490        5149 :       N0->getFlags().hasExact()) {
    6491        1863 :     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
    6492        1863 :       uint64_t C1 = N0C1->getZExtValue();
    6493        1863 :       uint64_t C2 = N1C->getZExtValue();
    6494             :       SDLoc DL(N);
    6495        1863 :       if (C1 <= C2)
    6496        1833 :         return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
    6497        1833 :                            DAG.getConstant(C2 - C1, DL, N1.getValueType()));
    6498          30 :       return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
    6499          30 :                          DAG.getConstant(C1 - C2, DL, N1.getValueType()));
    6500             :     }
    6501             :   }
    6502             : 
    6503             :   // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
    6504             :   //                               (and (srl x, (sub c1, c2), MASK)
    6505             :   // Only fold this if the inner shift has no other uses -- if it does, folding
    6506             :   // this will increase the total number of instructions.
    6507      113301 :   if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    6508        1808 :     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
    6509        1656 :       uint64_t c1 = N0C1->getZExtValue();
    6510        1656 :       if (c1 < OpSizeInBits) {
    6511        1656 :         uint64_t c2 = N1C->getZExtValue();
    6512        1656 :         APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
    6513        1656 :         SDValue Shift;
    6514        1656 :         if (c2 > c1) {
    6515         128 :           Mask <<= c2 - c1;
    6516             :           SDLoc DL(N);
    6517         128 :           Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
    6518         128 :                               DAG.getConstant(c2 - c1, DL, N1.getValueType()));
    6519             :         } else {
    6520        1528 :           Mask.lshrInPlace(c1 - c2);
    6521             :           SDLoc DL(N);
    6522        1528 :           Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
    6523        1528 :                               DAG.getConstant(c1 - c2, DL, N1.getValueType()));
    6524             :         }
    6525             :         SDLoc DL(N0);
    6526        1656 :         return DAG.getNode(ISD::AND, DL, VT, Shift,
    6527        1656 :                            DAG.getConstant(Mask, DL, VT));
    6528             :       }
    6529             :     }
    6530             :   }
    6531             : 
    6532             :   // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
    6533      109241 :   if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
    6534          71 :       isConstantOrConstantVector(N1, /* No Opaques */ true)) {
    6535             :     SDLoc DL(N);
    6536          71 :     SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
    6537         142 :     SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
    6538         142 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
    6539             :   }
    6540             : 
    6541             :   // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
    6542             :   // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
    6543             :   // Variant of version done on multiply, except mul by a power of 2 is turned
    6544             :   // into a shift.
    6545      105983 :   if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
    6546        3072 :       N0.getNode()->hasOneUse() &&
    6547        6122 :       isConstantOrConstantVector(N1, /* No Opaques */ true) &&
    6548      113284 :       isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true) &&
    6549        1135 :       TLI.isDesirableToCommuteWithShift(N, Level)) {
    6550        1229 :     SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
    6551        1061 :     SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
    6552        1061 :     AddToWorklist(Shl0.getNode());
    6553        1061 :     AddToWorklist(Shl1.getNode());
    6554        2290 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
    6555             :   }
    6556             : 
    6557             :   // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
    6558        1387 :   if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
    6559      110808 :       isConstantOrConstantVector(N1, /* No Opaques */ true) &&
    6560        1383 :       isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
    6561        1287 :     SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
    6562        1287 :     if (isConstantOrConstantVector(Shl))
    6563        2576 :       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
    6564             :   }
    6565             : 
    6566      106751 :   if (N1C && !N1C->isOpaque())
    6567       90020 :     if (SDValue NewSHL = visitShiftByConstant(N, N1C))
    6568         594 :       return NewSHL;
    6569             : 
    6570      106157 :   return SDValue();
    6571             : }
    6572             : 
    6573       29698 : SDValue DAGCombiner::visitSRA(SDNode *N) {
    6574       29698 :   SDValue N0 = N->getOperand(0);
    6575       29698 :   SDValue N1 = N->getOperand(1);
    6576       29698 :   EVT VT = N0.getValueType();
    6577       29698 :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6578             : 
    6579             :   // Arithmetic shifting an all-sign-bit value is a no-op.
    6580             :   // fold (sra 0, x) -> 0
    6581             :   // fold (sra -1, x) -> -1
    6582       29698 :   if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
    6583         455 :     return N0;
    6584             : 
    6585             :   // fold vector ops
    6586       29243 :   if (VT.isVector())
    6587        3111 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6588           3 :       return FoldedVOp;
    6589             : 
    6590       29240 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6591             : 
    6592             :   // fold (sra c1, c2) -> (sra c1, c2)
    6593             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6594       29240 :   if (N0C && N1C && !N1C->isOpaque())
    6595           0 :     return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
    6596             :   // fold (sra x, c >= size(x)) -> undef
    6597             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6598             :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6599       53534 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6600       29240 :   };
    6601       58480 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6602          23 :     return DAG.getUNDEF(VT);
    6603             :   // fold (sra x, 0) -> x
    6604       55479 :   if (N1C && N1C->isNullValue())
    6605          15 :     return N0;
    6606             : 
    6607       29202 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6608           3 :     return NewSel;
    6609             : 
    6610             :   // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
    6611             :   // sext_inreg.
    6612       29199 :   if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
    6613        2459 :     unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
    6614        2459 :     EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
    6615        2459 :     if (VT.isVector())
    6616         139 :       ExtVT = EVT::getVectorVT(*DAG.getContext(),
    6617         139 :                                ExtVT, VT.getVectorNumElements());
    6618        2459 :     if ((!LegalOperations ||
    6619        1761 :          TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
    6620         731 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    6621        2193 :                          N0.getOperand(0), DAG.getValueType(ExtVT));
    6622             :   }
    6623             : 
    6624             :   // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
    6625             :   // clamp (add c1, c2) to max shift.
    6626       28468 :   if (N0.getOpcode() == ISD::SRA) {
    6627             :     SDLoc DL(N);
    6628         332 :     EVT ShiftVT = N1.getValueType();
    6629         332 :     EVT ShiftSVT = ShiftVT.getScalarType();
    6630             :     SmallVector<SDValue, 16> ShiftValues;
    6631             : 
    6632             :     auto SumOfShifts = [&](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    6633             :       APInt c1 = LHS->getAPIntValue();
    6634             :       APInt c2 = RHS->getAPIntValue();
    6635             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6636             :       APInt Sum = c1 + c2;
    6637             :       unsigned ShiftSum =
    6638             :           Sum.uge(OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue();
    6639             :       ShiftValues.push_back(DAG.getConstant(ShiftSum, DL, ShiftSVT));
    6640             :       return true;
    6641             :     };
    6642         664 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) {
    6643         256 :       SDValue ShiftValue;
    6644         256 :       if (VT.isVector())
    6645          30 :         ShiftValue = DAG.getBuildVector(ShiftVT, DL, ShiftValues);
    6646             :       else
    6647         241 :         ShiftValue = ShiftValues[0];
    6648         512 :       return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue);
    6649             :     }
    6650             :   }
    6651             : 
    6652             :   // fold (sra (shl X, m), (sub result_size, n))
    6653             :   // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
    6654             :   // result_size - n != m.
    6655             :   // If truncate is free for the target sext(shl) is likely to result in better
    6656             :   // code.
    6657       28212 :   if (N0.getOpcode() == ISD::SHL && N1C) {
    6658             :     // Get the two constanst of the shifts, CN0 = m, CN = n.
    6659        3368 :     const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
    6660        3368 :     if (N01C) {
    6661        3368 :       LLVMContext &Ctx = *DAG.getContext();
    6662             :       // Determine what the truncate's result bitsize and type would be.
    6663        6736 :       EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue());
    6664             : 
    6665        3368 :       if (VT.isVector())
    6666          45 :         TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorNumElements());
    6667             : 
    6668             :       // Determine the residual right-shift amount.
    6669        6736 :       int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
    6670             : 
    6671             :       // If the shift is not a no-op (in which case this should be just a sign
    6672             :       // extend already), the truncated to type is legal, sign_extend is legal
    6673             :       // on that type, and the truncate to that type is both legal and free,
    6674             :       // perform the transform.
    6675             :       if ((ShiftAmt > 0) &&
    6676        1594 :           TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
    6677        3469 :           TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
    6678         101 :           TLI.isTruncateFree(VT, TruncVT)) {
    6679             :         SDLoc DL(N);
    6680          13 :         SDValue Amt = DAG.getConstant(ShiftAmt, DL,
    6681          26 :             getShiftAmountTy(N0.getOperand(0).getValueType()));
    6682          13 :         SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
    6683          13 :                                     N0.getOperand(0), Amt);
    6684          13 :         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
    6685          13 :                                     Shift);
    6686          13 :         return DAG.getNode(ISD::SIGN_EXTEND, DL,
    6687          26 :                            N->getValueType(0), Trunc);
    6688             :       }
    6689             :     }
    6690             :   }
    6691             : 
    6692             :   // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
    6693       28199 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6694         320 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6695          18 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6696          30 :       return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
    6697             :   }
    6698             : 
    6699             :   // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
    6700             :   //      if c1 is equal to the number of bits the trunc removes
    6701        4735 :   if (N0.getOpcode() == ISD::TRUNCATE &&
    6702        4735 :       (N0.getOperand(0).getOpcode() == ISD::SRL ||
    6703        2280 :        N0.getOperand(0).getOpcode() == ISD::SRA) &&
    6704        4026 :       N0.getOperand(0).hasOneUse() &&
    6705       31788 :       N0.getOperand(0).getOperand(1).hasOneUse() &&
    6706             :       N1C) {
    6707         106 :     SDValue N0Op0 = N0.getOperand(0);
    6708         106 :     if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6709         212 :       unsigned LargeShiftVal = LargeShift->getZExtValue();
    6710         106 :       EVT LargeVT = N0Op0.getValueType();
    6711             : 
    6712         106 :       if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) {
    6713             :         SDLoc DL(N);
    6714             :         SDValue Amt =
    6715         100 :           DAG.getConstant(LargeShiftVal + N1C->getZExtValue(), DL,
    6716         300 :                           getShiftAmountTy(N0Op0.getOperand(0).getValueType()));
    6717         100 :         SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT,
    6718         100 :                                   N0Op0.getOperand(0), Amt);
    6719         200 :         return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
    6720             :       }
    6721             :     }
    6722             :   }
    6723             : 
    6724             :   // Simplify, based on bits shifted out of the LHS.
    6725       28084 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6726        1014 :     return SDValue(N, 0);
    6727             : 
    6728             :   // If the sign bit is known to be zero, switch this to a SRL.
    6729       27070 :   if (DAG.SignBitIsZero(N0))
    6730         238 :     return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
    6731             : 
    6732       26951 :   if (N1C && !N1C->isOpaque())
    6733       24140 :     if (SDValue NewSRA = visitShiftByConstant(N, N1C))
    6734           0 :       return NewSRA;
    6735             : 
    6736       26951 :   return SDValue();
    6737             : }
    6738             : 
    6739      161976 : SDValue DAGCombiner::visitSRL(SDNode *N) {
    6740      161976 :   SDValue N0 = N->getOperand(0);
    6741      161976 :   SDValue N1 = N->getOperand(1);
    6742      161976 :   EVT VT = N0.getValueType();
    6743             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6744             : 
    6745             :   // fold vector ops
    6746      161976 :   if (VT.isVector())
    6747        4779 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6748           4 :       return FoldedVOp;
    6749             : 
    6750      161972 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6751             : 
    6752             :   // fold (srl c1, c2) -> c1 >>u c2
    6753             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6754      161972 :   if (N0C && N1C && !N1C->isOpaque())
    6755        5264 :     return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
    6756             :   // fold (srl 0, x) -> 0
    6757      159340 :   if (isNullConstantOrNullSplatConstant(N0))
    6758          59 :     return N0;
    6759             :   // fold (srl x, c >= size(x)) -> undef
    6760             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6761             :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6762      294422 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6763             :   };
    6764      318562 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6765          27 :     return DAG.getUNDEF(VT);
    6766             :   // fold (srl x, 0) -> x
    6767      305669 :   if (N1C && N1C->isNullValue())
    6768        2218 :     return N0;
    6769             : 
    6770      157036 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6771           4 :     return NewSel;
    6772             : 
    6773             :   // if (srl x, c) is known to be zero, return 0
    6774      301226 :   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
    6775      445374 :                                    APInt::getAllOnesValue(OpSizeInBits)))
    6776          92 :     return DAG.getConstant(0, SDLoc(N), VT);
    6777             : 
    6778             :   // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
    6779      156986 :   if (N0.getOpcode() == ISD::SRL) {
    6780             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    6781             :                                           ConstantSDNode *RHS) {
    6782             :       APInt c1 = LHS->getAPIntValue();
    6783             :       APInt c2 = RHS->getAPIntValue();
    6784             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6785             :       return (c1 + c2).uge(OpSizeInBits);
    6786             :     };
    6787       10568 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    6788          10 :       return DAG.getConstant(0, SDLoc(N), VT);
    6789             : 
    6790             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    6791             :                                        ConstantSDNode *RHS) {
    6792             :       APInt c1 = LHS->getAPIntValue();
    6793             :       APInt c2 = RHS->getAPIntValue();
    6794             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6795             :       return (c1 + c2).ult(OpSizeInBits);
    6796             :     };
    6797       10558 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    6798             :       SDLoc DL(N);
    6799        4416 :       EVT ShiftVT = N1.getValueType();
    6800        8832 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    6801        8832 :       return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
    6802             :     }
    6803             :   }
    6804             : 
    6805             :   // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
    6806      152565 :   if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
    6807       14543 :       N0.getOperand(0).getOpcode() == ISD::SRL) {
    6808        4643 :     if (auto N001C = isConstOrConstSplat(N0.getOperand(0).getOperand(1))) {
    6809        4643 :       uint64_t c1 = N001C->getZExtValue();
    6810        4643 :       uint64_t c2 = N1C->getZExtValue();
    6811        9286 :       EVT InnerShiftVT = N0.getOperand(0).getValueType();
    6812        9286 :       EVT ShiftCountVT = N0.getOperand(0).getOperand(1).getValueType();
    6813        4643 :       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
    6814             :       // This is only valid if the OpSizeInBits + c1 = size of inner shift.
    6815        4643 :       if (c1 + OpSizeInBits == InnerShiftSize) {
    6816             :         SDLoc DL(N0);
    6817        3187 :         if (c1 + c2 >= InnerShiftSize)
    6818           0 :           return DAG.getConstant(0, DL, VT);
    6819        3187 :         return DAG.getNode(ISD::TRUNCATE, DL, VT,
    6820             :                            DAG.getNode(ISD::SRL, DL, InnerShiftVT,
    6821        3187 :                                        N0.getOperand(0).getOperand(0),
    6822             :                                        DAG.getConstant(c1 + c2, DL,
    6823        3187 :                                                        ShiftCountVT)));
    6824             :       }
    6825             :     }
    6826             :   }
    6827             : 
    6828             :   // fold (srl (shl x, c), c) -> (and x, cst2)
    6829      150710 :   if (N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
    6830        1332 :       isConstantOrConstantVector(N1, /* NoOpaques */ true)) {
    6831             :     SDLoc DL(N);
    6832             :     SDValue Mask =
    6833         410 :         DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
    6834         205 :     AddToWorklist(Mask.getNode());
    6835         410 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask);
    6836             :   }
    6837             : 
    6838             :   // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
    6839      149173 :   if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    6840             :     // Shifting in all undef bits?
    6841         764 :     EVT SmallVT = N0.getOperand(0).getValueType();
    6842             :     unsigned BitSize = SmallVT.getScalarSizeInBits();
    6843         764 :     if (N1C->getZExtValue() >= BitSize)
    6844         168 :       return DAG.getUNDEF(VT);
    6845             : 
    6846         382 :     if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
    6847         168 :       uint64_t ShiftAmt = N1C->getZExtValue();
    6848             :       SDLoc DL0(N0);
    6849         168 :       SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
    6850             :                                        N0.getOperand(0),
    6851             :                           DAG.getConstant(ShiftAmt, DL0,
    6852         168 :                                           getShiftAmountTy(SmallVT)));
    6853         168 :       AddToWorklist(SmallShift.getNode());
    6854         168 :       APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt);
    6855             :       SDLoc DL(N);
    6856         168 :       return DAG.getNode(ISD::AND, DL, VT,
    6857         168 :                          DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
    6858         168 :                          DAG.getConstant(Mask, DL, VT));
    6859             :     }
    6860             :   }
    6861             : 
    6862             :   // fold (srl (sra X, Y), 31) -> (srl X, 31).  This srl only looks at the sign
    6863             :   // bit, which is unmodified by sra.
    6864      285182 :   if (N1C && N1C->getZExtValue() + 1 == OpSizeInBits) {
    6865        4360 :     if (N0.getOpcode() == ISD::SRA)
    6866         826 :       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
    6867             :   }
    6868             : 
    6869             :   // fold (srl (ctlz x), "5") -> x  iff x has one bit set (the low bit).
    6870      148613 :   if (N1C && N0.getOpcode() == ISD::CTLZ &&
    6871         348 :       N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
    6872         171 :     KnownBits Known;
    6873         348 :     DAG.computeKnownBits(N0.getOperand(0), Known);
    6874             : 
    6875             :     // If any of the input bits are KnownOne, then the input couldn't be all
    6876             :     // zeros, thus the result of the srl will always be zero.
    6877         174 :     if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
    6878             : 
    6879             :     // If all of the bits input the to ctlz node are known to be zero, then
    6880             :     // the result of the ctlz is "32" and the result of the shift is one.
    6881         174 :     APInt UnknownBits = ~Known.Zero;
    6882         174 :     if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
    6883             : 
    6884             :     // Otherwise, check to see if there is exactly one bit input to the ctlz.
    6885         174 :     if (UnknownBits.isPowerOf2()) {
    6886             :       // Okay, we know that only that the single bit specified by UnknownBits
    6887             :       // could be set on input to the CTLZ node. If this bit is set, the SRL
    6888             :       // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
    6889             :       // to an SRL/XOR pair, which is likely to simplify more.
    6890           3 :       unsigned ShAmt = UnknownBits.countTrailingZeros();
    6891           3 :       SDValue Op = N0.getOperand(0);
    6892             : 
    6893           3 :       if (ShAmt) {
    6894             :         SDLoc DL(N0);
    6895           3 :         Op = DAG.getNode(ISD::SRL, DL, VT, Op,
    6896             :                   DAG.getConstant(ShAmt, DL,
    6897           3 :                                   getShiftAmountTy(Op.getValueType())));
    6898           3 :         AddToWorklist(Op.getNode());
    6899             :       }
    6900             : 
    6901             :       SDLoc DL(N);
    6902           3 :       return DAG.getNode(ISD::XOR, DL, VT,
    6903           3 :                          Op, DAG.getConstant(1, DL, VT));
    6904             :     }
    6905             :   }
    6906             : 
    6907             :   // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
    6908      148610 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6909        3157 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6910          48 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6911          65 :       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
    6912             :   }
    6913             : 
    6914             :   // fold operands of srl based on knowledge that the low bits are not
    6915             :   // demanded.
    6916      148578 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6917        7846 :     return SDValue(N, 0);
    6918             : 
    6919      140732 :   if (N1C && !N1C->isOpaque())
    6920      127921 :     if (SDValue NewSRL = visitShiftByConstant(N, N1C))
    6921         600 :       return NewSRL;
    6922             : 
    6923             :   // Attempt to convert a srl of a load into a narrower zero-extending load.
    6924      140132 :   if (SDValue NarrowLoad = ReduceLoadWidth(N))
    6925         119 :     return NarrowLoad;
    6926             : 
    6927             :   // Here is a common situation. We want to optimize:
    6928             :   //
    6929             :   //   %a = ...
    6930             :   //   %b = and i32 %a, 2
    6931             :   //   %c = srl i32 %b, 1
    6932             :   //   brcond i32 %c ...
    6933             :   //
    6934             :   // into
    6935             :   //
    6936             :   //   %a = ...
    6937             :   //   %b = and %a, 2
    6938             :   //   %c = setcc eq %b, 0
    6939             :   //   brcond %c ...
    6940             :   //
    6941             :   // However when after the source operand of SRL is optimized into AND, the SRL
    6942             :   // itself may not be optimized further. Look for it and add the BRCOND into
    6943             :   // the worklist.
    6944             :   if (N->hasOneUse()) {
    6945             :     SDNode *Use = *N->use_begin();
    6946      266674 :     if (Use->getOpcode() == ISD::BRCOND)
    6947           4 :       AddToWorklist(Use);
    6948      133333 :     else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
    6949             :       // Also look pass the truncate.
    6950             :       Use = *Use->use_begin();
    6951       29606 :       if (Use->getOpcode() == ISD::BRCOND)
    6952           2 :         AddToWorklist(Use);
    6953             :     }
    6954             :   }
    6955             : 
    6956      140013 :   return SDValue();
    6957             : }
    6958             : 
    6959           0 : SDValue DAGCombiner::visitABS(SDNode *N) {
    6960           0 :   SDValue N0 = N->getOperand(0);
    6961           0 :   EVT VT = N->getValueType(0);
    6962             : 
    6963             :   // fold (abs c1) -> c2
    6964           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6965           0 :     return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0);
    6966             :   // fold (abs (abs x)) -> (abs x)
    6967           0 :   if (N0.getOpcode() == ISD::ABS)
    6968           0 :     return N0;
    6969             :   // fold (abs x) -> x iff not-negative
    6970           0 :   if (DAG.SignBitIsZero(N0))
    6971           0 :     return N0;
    6972           0 :   return SDValue();
    6973             : }
    6974             : 
    6975           0 : SDValue DAGCombiner::visitBSWAP(SDNode *N) {
    6976           0 :   SDValue N0 = N->getOperand(0);
    6977           0 :   EVT VT = N->getValueType(0);
    6978             : 
    6979             :   // fold (bswap c1) -> c2
    6980           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6981           0 :     return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
    6982             :   // fold (bswap (bswap x)) -> x
    6983           0 :   if (N0.getOpcode() == ISD::BSWAP)
    6984           0 :     return N0->getOperand(0);
    6985           0 :   return SDValue();
    6986             : }
    6987             : 
    6988           0 : SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
    6989           0 :   SDValue N0 = N->getOperand(0);
    6990           0 :   EVT VT = N->getValueType(0);
    6991             : 
    6992             :   // fold (bitreverse c1) -> c2
    6993           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6994           0 :     return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0);
    6995             :   // fold (bitreverse (bitreverse x)) -> x
    6996           0 :   if (N0.getOpcode() == ISD::BITREVERSE)
    6997           0 :     return N0.getOperand(0);
    6998           0 :   return SDValue();
    6999             : }
    7000             : 
    7001        1212 : SDValue DAGCombiner::visitCTLZ(SDNode *N) {
    7002        1212 :   SDValue N0 = N->getOperand(0);
    7003        1212 :   EVT VT = N->getValueType(0);
    7004             : 
    7005             :   // fold (ctlz c1) -> c2
    7006        1212 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7007           4 :     return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
    7008             : 
    7009             :   // If the value is known never to be zero, switch to the undef version.
    7010        1210 :   if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) {
    7011         400 :     if (DAG.isKnownNeverZero(N0))
    7012          14 :       return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7013             :   }
    7014             : 
    7015        1203 :   return SDValue();
    7016             : }
    7017             : 
    7018           0 : SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
    7019           0 :   SDValue N0 = N->getOperand(0);
    7020           0 :   EVT VT = N->getValueType(0);
    7021             : 
    7022             :   // fold (ctlz_zero_undef c1) -> c2
    7023           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7024           0 :     return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7025           0 :   return SDValue();
    7026             : }
    7027             : 
    7028         474 : SDValue DAGCombiner::visitCTTZ(SDNode *N) {
    7029         474 :   SDValue N0 = N->getOperand(0);
    7030         474 :   EVT VT = N->getValueType(0);
    7031             : 
    7032             :   // fold (cttz c1) -> c2
    7033         474 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7034           0 :     return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
    7035             : 
    7036             :   // If the value is known never to be zero, switch to the undef version.
    7037         474 :   if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) {
    7038         312 :     if (DAG.isKnownNeverZero(N0))
    7039          34 :       return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7040             :   }
    7041             : 
    7042         457 :   return SDValue();
    7043             : }
    7044             : 
    7045           0 : SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
    7046           0 :   SDValue N0 = N->getOperand(0);
    7047           0 :   EVT VT = N->getValueType(0);
    7048             : 
    7049             :   // fold (cttz_zero_undef c1) -> c2
    7050           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7051           0 :     return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7052           0 :   return SDValue();
    7053             : }
    7054             : 
    7055           0 : SDValue DAGCombiner::visitCTPOP(SDNode *N) {
    7056           0 :   SDValue N0 = N->getOperand(0);
    7057           0 :   EVT VT = N->getValueType(0);
    7058             : 
    7059             :   // fold (ctpop c1) -> c2
    7060           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7061           0 :     return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
    7062           0 :   return SDValue();
    7063             : }
    7064             : 
    7065             : // FIXME: This should be checking for no signed zeros on individual operands, as
    7066             : // well as no nans.
    7067       28756 : static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS) {
    7068       28756 :   const TargetOptions &Options = DAG.getTarget().Options;
    7069       28756 :   EVT VT = LHS.getValueType();
    7070             : 
    7071         808 :   return Options.NoSignedZerosFPMath && VT.isFloatingPoint() &&
    7072       29355 :          DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS);
    7073             : }
    7074             : 
    7075             : /// Generate Min/Max node
    7076           0 : static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
    7077             :                                    SDValue RHS, SDValue True, SDValue False,
    7078             :                                    ISD::CondCode CC, const TargetLowering &TLI,
    7079             :                                    SelectionDAG &DAG) {
    7080             :   if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
    7081           0 :     return SDValue();
    7082             : 
    7083           0 :   EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    7084           0 :   switch (CC) {
    7085           0 :   case ISD::SETOLT:
    7086             :   case ISD::SETOLE:
    7087             :   case ISD::SETLT:
    7088             :   case ISD::SETLE:
    7089             :   case ISD::SETULT:
    7090             :   case ISD::SETULE: {
    7091             :     unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
    7092           0 :     if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
    7093           0 :       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    7094           0 :     return SDValue();
    7095             :   }
    7096           0 :   case ISD::SETOGT:
    7097             :   case ISD::SETOGE:
    7098             :   case ISD::SETGT:
    7099             :   case ISD::SETGE:
    7100             :   case ISD::SETUGT:
    7101             :   case ISD::SETUGE: {
    7102             :     unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
    7103           0 :     if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
    7104           0 :       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    7105           0 :     return SDValue();
    7106             :   }
    7107           0 :   default:
    7108           0 :     return SDValue();
    7109             :   }
    7110             : }
    7111             : 
    7112       31948 : SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
    7113       31948 :   SDValue Cond = N->getOperand(0);
    7114       31948 :   SDValue N1 = N->getOperand(1);
    7115       31948 :   SDValue N2 = N->getOperand(2);
    7116       63896 :   EVT VT = N->getValueType(0);
    7117       31948 :   EVT CondVT = Cond.getValueType();
    7118             :   SDLoc DL(N);
    7119             : 
    7120       31948 :   if (!VT.isInteger())
    7121        7164 :     return SDValue();
    7122             : 
    7123             :   auto *C1 = dyn_cast<ConstantSDNode>(N1);
    7124             :   auto *C2 = dyn_cast<ConstantSDNode>(N2);
    7125       24784 :   if (!C1 || !C2)
    7126       20472 :     return SDValue();
    7127             : 
    7128             :   // Only do this before legalization to avoid conflicting with target-specific
    7129             :   // transforms in the other direction (create a select from a zext/sext). There
    7130             :   // is also a target-independent combine here in DAGCombiner in the other
    7131             :   // direction for (select Cond, -1, 0) when the condition is not i1.
    7132        2930 :   if (CondVT == MVT::i1 && !LegalOperations) {
    7133        3055 :     if (C1->isNullValue() && C2->isOne()) {
    7134             :       // select Cond, 0, 1 --> zext (!Cond)
    7135         218 :       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    7136             :       if (VT != MVT::i1)
    7137          66 :         NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond);
    7138         109 :       return NotCond;
    7139             :     }
    7140        1427 :     if (C1->isNullValue() && C2->isAllOnesValue()) {
    7141             :       // select Cond, 0, -1 --> sext (!Cond)
    7142          52 :       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    7143             :       if (VT != MVT::i1)
    7144          52 :         NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond);
    7145          26 :       return NotCond;
    7146             :     }
    7147        1435 :     if (C1->isOne() && C2->isNullValue()) {
    7148             :       // select Cond, 1, 0 --> zext (Cond)
    7149             :       if (VT != MVT::i1)
    7150         156 :         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
    7151          78 :       return Cond;
    7152             :     }
    7153        1318 :     if (C1->isAllOnesValue() && C2->isNullValue()) {
    7154             :       // select Cond, -1, 0 --> sext (Cond)
    7155             :       if (VT != MVT::i1)
    7156         114 :         Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
    7157          57 :       return Cond;
    7158             :     }
    7159             : 
    7160             :     // For any constants that differ by 1, we can transform the select into an
    7161             :     // extend and add. Use a target hook because some targets may prefer to
    7162             :     // transform in the other direction.
    7163        1140 :     if (TLI.convertSelectOfConstantsToMath(VT)) {
    7164        1800 :       if (C1->getAPIntValue() - 1 == C2->getAPIntValue()) {
    7165             :         // select Cond, C1, C1-1 --> add (zext Cond), C1-1
    7166             :         if (VT != MVT::i1)
    7167          66 :           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
    7168          66 :         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
    7169             :       }
    7170        1701 :       if (C1->getAPIntValue() + 1 == C2->getAPIntValue()) {
    7171             :         // select Cond, C1, C1+1 --> add (sext Cond), C1+1
    7172             :         if (VT != MVT::i1)
    7173         170 :           Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
    7174         170 :         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
    7175             :       }
    7176             :     }
    7177             : 
    7178        1022 :     return SDValue();
    7179             :   }
    7180             : 
    7181             :   // fold (select Cond, 0, 1) -> (xor Cond, 1)
    7182             :   // We can't do this reliably if integer based booleans have different contents
    7183             :   // to floating point based booleans. This is because we can't tell whether we
    7184             :   // have an integer-based boolean or a floating-point-based boolean unless we
    7185             :   // can find the SETCC that produced it and inspect its operands. This is
    7186             :   // fairly easy if C is the SETCC node, but it can potentially be
    7187             :   // undiscoverable (or not reasonably discoverable). For example, it could be
    7188             :   // in another basic block or it could require searching a complicated
    7189             :   // expression.
    7190        2902 :   if (CondVT.isInteger() &&
    7191        2902 :       TLI.getBooleanContents(/*isVec*/false, /*isFloat*/true) ==
    7192        1248 :           TargetLowering::ZeroOrOneBooleanContent &&
    7193             :       TLI.getBooleanContents(/*isVec*/false, /*isFloat*/false) ==
    7194        1248 :           TargetLowering::ZeroOrOneBooleanContent &&
    7195        4846 :       C1->isNullValue() && C2->isOne()) {
    7196             :     SDValue NotCond =
    7197           0 :         DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
    7198           0 :     if (VT.bitsEq(CondVT))
    7199           0 :       return NotCond;
    7200           0 :     return DAG.getZExtOrTrunc(NotCond, DL, VT);
    7201             :   }
    7202             : 
    7203        2902 :   return SDValue();
    7204             : }
    7205             : 
    7206       32128 : SDValue DAGCombiner::visitSELECT(SDNode *N) {
    7207       32128 :   SDValue N0 = N->getOperand(0);
    7208       32128 :   SDValue N1 = N->getOperand(1);
    7209       32128 :   SDValue N2 = N->getOperand(2);
    7210       64256 :   EVT VT = N->getValueType(0);
    7211             :   EVT VT0 = N0.getValueType();
    7212             :   SDLoc DL(N);
    7213             : 
    7214             :   // fold (select C, X, X) -> X
    7215       32128 :   if (N1 == N2)
    7216          69 :     return N1;
    7217             : 
    7218             :   if (const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
    7219             :     // fold (select true, X, Y) -> X
    7220             :     // fold (select false, X, Y) -> Y
    7221         233 :     return !N0C->isNullValue() ? N1 : N2;
    7222             :   }
    7223             : 
    7224             :   // fold (select X, X, Y) -> (or X, Y)
    7225             :   // fold (select X, 1, Y) -> (or C, Y)
    7226         178 :   if (VT == VT0 && VT == MVT::i1 && (N0 == N1 || isOneConstant(N1)))
    7227          54 :     return DAG.getNode(ISD::OR, DL, VT, N0, N2);
    7228             : 
    7229       31948 :   if (SDValue V = foldSelectOfConstants(N))
    7230         388 :     return V;
    7231             : 
    7232             :   // fold (select C, 0, X) -> (and (not C), X)
    7233          81 :   if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) {
    7234           7 :     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
    7235           7 :     AddToWorklist(NOTNode.getNode());
    7236          14 :     return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2);
    7237             :   }
    7238             :   // fold (select C, X, 1) -> (or (not C), X)
    7239          74 :   if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) {
    7240           2 :     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
    7241           2 :     AddToWorklist(NOTNode.getNode());
    7242           4 :     return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1);
    7243             :   }
    7244             :   // fold (select X, Y, X) -> (and X, Y)
    7245             :   // fold (select X, Y, 0) -> (and X, Y)
    7246          70 :   if (VT == VT0 && VT == MVT::i1 && (N0 == N2 || isNullConstant(N2)))
    7247          30 :     return DAG.getNode(ISD::AND, DL, VT, N0, N1);
    7248             : 
    7249             :   // If we can fold this based on the true/false value, do so.
    7250       31536 :   if (SimplifySelectOps(N, N1, N2))
    7251         108 :     return SDValue(N, 0); // Don't revisit N.
    7252             : 
    7253             :   if (VT0 == MVT::i1) {
    7254             :     // The code in this block deals with the following 2 equivalences:
    7255             :     //    select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
    7256             :     //    select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
    7257             :     // The target can specify its preferred form with the
    7258             :     // shouldNormalizeToSelectSequence() callback. However we always transform
    7259             :     // to the right anyway if we find the inner select exists in the DAG anyway
    7260             :     // and we always transform to the left side if we know that we can further
    7261             :     // optimize the combination of the conditions.
    7262             :     bool normalizeToSequence =
    7263       21887 :         TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
    7264             :     // select (and Cond0, Cond1), X, Y
    7265             :     //   -> select Cond0, (select Cond1, X, Y), Y
    7266       21887 :     if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
    7267         688 :       SDValue Cond0 = N0->getOperand(0);
    7268         688 :       SDValue Cond1 = N0->getOperand(1);
    7269             :       SDValue InnerSelect =
    7270        2064 :           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
    7271         688 :       if (normalizeToSequence || !InnerSelect.use_empty())
    7272          38 :         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
    7273          38 :                            InnerSelect, N2);
    7274             :     }
    7275             :     // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
    7276       21849 :     if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
    7277         177 :       SDValue Cond0 = N0->getOperand(0);
    7278         177 :       SDValue Cond1 = N0->getOperand(1);
    7279             :       SDValue InnerSelect =
    7280         531 :           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
    7281         177 :       if (normalizeToSequence || !InnerSelect.use_empty())
    7282          54 :         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
    7283          54 :                            InnerSelect);
    7284             :     }
    7285             : 
    7286             :     // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
    7287       21795 :     if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
    7288        1425 :       SDValue N1_0 = N1->getOperand(0);
    7289        1425 :       SDValue N1_1 = N1->getOperand(1);
    7290        1425 :       SDValue N1_2 = N1->getOperand(2);
    7291        1425 :       if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
    7292             :         // Create the actual and node if we can generate good code for it.
    7293         214 :         if (!normalizeToSequence) {
    7294         292 :           SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
    7295         438 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, N2);
    7296             :         }
    7297             :         // Otherwise see if we can optimize the "and" to a better pattern.
    7298          68 :         if (SDValue Combined = visitANDLike(N0, N1_0, N))
    7299           6 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
    7300          12 :                              N2);
    7301             :       }
    7302             :     }
    7303             :     // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
    7304       21643 :     if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
    7305         833 :       SDValue N2_0 = N2->getOperand(0);
    7306         833 :       SDValue N2_1 = N2->getOperand(1);
    7307         833 :       SDValue N2_2 = N2->getOperand(2);
    7308         833 :       if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
    7309             :         // Create the actual or node if we can generate good code for it.
    7310         117 :         if (!normalizeToSequence) {
    7311          34 :           SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
    7312          34 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, N2_2);
    7313             :         }
    7314             :         // Otherwise see if we can optimize to a better pattern.
    7315         100 :         if (SDValue Combined = visitORLike(N0, N2_0, N))
    7316           6 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
    7317           6 :                              N2_2);
    7318             :       }
    7319             :     }
    7320             :   }
    7321             : 
    7322             :   if (VT0 == MVT::i1) {
    7323             :     // select (not Cond), N1, N2 -> select Cond, N2, N1
    7324       21620 :     if (isBitwiseNot(N0))
    7325         564 :       return DAG.getNode(ISD::SELECT, DL, VT, N0->getOperand(0), N2, N1);
    7326             :   }
    7327             : 
    7328             :   // Fold selects based on a setcc into other things, such as min/max/abs.
    7329       30973 :   if (N0.getOpcode() == ISD::SETCC) {
    7330       25179 :     SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1);
    7331       25179 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7332             : 
    7333             :     // select (fcmp lt x, y), x, y -> fminnum x, y
    7334             :     // select (fcmp gt x, y), x, y -> fmaxnum x, y
    7335             :     //
    7336             :     // This is OK if we don't care what happens if either operand is a NaN.
    7337       25179 :     if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2))
    7338         136 :       if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2,
    7339         136 :                                                 CC, TLI, DAG))
    7340         103 :         return FMinMax;
    7341             : 
    7342       25076 :     if (TLI.isOperationLegal(ISD::SELECT_CC, VT) ||
    7343       24715 :         (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)))
    7344        3589 :       return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2,
    7345        3589 :                          N0.getOperand(2));
    7346             : 
    7347       21487 :     return SimplifySelect(DL, N0, N1, N2);
    7348             :   }
    7349             : 
    7350        5794 :   return SDValue();
    7351             : }
    7352             : 
    7353             : static
    7354          19 : std::pair<SDValue, SDValue> SplitVSETCC(const SDNode *N, SelectionDAG &DAG) {
    7355             :   SDLoc DL(N);
    7356             :   EVT LoVT, HiVT;
    7357          38 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    7358             : 
    7359             :   // Split the inputs.
    7360             :   SDValue Lo, Hi, LL, LH, RL, RH;
    7361          19 :   std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
    7362          19 :   std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
    7363             : 
    7364          57 :   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
    7365          57 :   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
    7366             : 
    7367          19 :   return std::make_pair(Lo, Hi);
    7368             : }
    7369             : 
    7370             : // This function assumes all the vselect's arguments are CONCAT_VECTOR
    7371             : // nodes and that the condition is a BV of ConstantSDNodes (or undefs).
    7372          43 : static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
    7373             :   SDLoc DL(N);
    7374          43 :   SDValue Cond = N->getOperand(0);
    7375          43 :   SDValue LHS = N->getOperand(1);
    7376          43 :   SDValue RHS = N->getOperand(2);
    7377          86 :   EVT VT = N->getValueType(0);
    7378          43 :   int NumElems = VT.getVectorNumElements();
    7379             :   assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
    7380             :          RHS.getOpcode() == ISD::CONCAT_VECTORS &&
    7381             :          Cond.getOpcode() == ISD::BUILD_VECTOR);
    7382             : 
    7383             :   // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
    7384             :   // binary ones here.
    7385          43 :   if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
    7386          11 :     return SDValue();
    7387             : 
    7388             :   // We're sure we have an even number of elements due to the
    7389             :   // concat_vectors we have as arguments to vselect.
    7390             :   // Skip BV elements until we find one that's not an UNDEF
    7391             :   // After we find an UNDEF element, keep looping until we get to half the
    7392             :   // length of the BV and see if all the non-undef nodes are the same.
    7393             :   ConstantSDNode *BottomHalf = nullptr;
    7394          98 :   for (int i = 0; i < NumElems / 2; ++i) {
    7395         170 :     if (Cond->getOperand(i)->isUndef())
    7396             :       continue;
    7397             : 
    7398          85 :     if (BottomHalf == nullptr)
    7399             :       BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    7400          53 :     else if (Cond->getOperand(i).getNode() != BottomHalf)
    7401          19 :       return SDValue();
    7402             :   }
    7403             : 
    7404             :   // Do the same for the second half of the BuildVector
    7405             :   ConstantSDNode *TopHalf = nullptr;
    7406          48 :   for (int i = NumElems / 2; i < NumElems; ++i) {
    7407          76 :     if (Cond->getOperand(i)->isUndef())
    7408             :       continue;
    7409             : 
    7410          38 :     if (TopHalf == nullptr)
    7411             :       TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    7412          25 :     else if (Cond->getOperand(i).getNode() != TopHalf)
    7413           3 :       return SDValue();
    7414             :   }
    7415             : 
    7416             :   assert(TopHalf && BottomHalf &&
    7417             :          "One half of the selector was all UNDEFs and the other was all the "
    7418             :          "same value. This should have been addressed before this function.");
    7419             :   return DAG.getNode(
    7420             :       ISD::CONCAT_VECTORS, DL, VT,
    7421          20 :       BottomHalf->isNullValue() ? RHS->getOperand(0) : LHS->getOperand(0),
    7422          30 :       TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
    7423             : }
    7424             : 
    7425         219 : SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
    7426         219 :   if (Level >= AfterLegalizeTypes)
    7427          98 :     return SDValue();
    7428             : 
    7429             :   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
    7430         121 :   SDValue Mask = MSC->getMask();
    7431         121 :   SDValue Data  = MSC->getValue();
    7432             :   SDLoc DL(N);
    7433             : 
    7434             :   // If the MSCATTER data type requires splitting and the mask is provided by a
    7435             :   // SETCC, then split both nodes and its operands before legalization. This
    7436             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7437             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7438         121 :   if (Mask.getOpcode() != ISD::SETCC)
    7439          96 :     return SDValue();
    7440             : 
    7441             :   // Check if any splitting is required.
    7442          50 :   if (TLI.getTypeAction(*DAG.getContext(), Data.getValueType()) !=
    7443             :       TargetLowering::TypeSplitVector)
    7444          20 :     return SDValue();
    7445             :   SDValue MaskLo, MaskHi;
    7446           5 :   std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7447             : 
    7448             :   EVT LoVT, HiVT;
    7449          10 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0));
    7450             : 
    7451           5 :   SDValue Chain = MSC->getChain();
    7452             : 
    7453           5 :   EVT MemoryVT = MSC->getMemoryVT();
    7454           5 :   unsigned Alignment = MSC->getOriginalAlignment();
    7455             : 
    7456           5 :   EVT LoMemVT, HiMemVT;
    7457           5 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7458             : 
    7459             :   SDValue DataLo, DataHi;
    7460           5 :   std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    7461             : 
    7462           5 :   SDValue Scale = MSC->getScale();
    7463           5 :   SDValue BasePtr = MSC->getBasePtr();
    7464             :   SDValue IndexLo, IndexHi;
    7465          10 :   std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->getIndex(), DL);
    7466             : 
    7467           5 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    7468          15 :     getMachineMemOperand(MSC->getPointerInfo(),
    7469             :                           MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
    7470           5 :                           Alignment, MSC->getAAInfo(), MSC->getRanges());
    7471             : 
    7472           5 :   SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo, Scale };
    7473           5 :   SDValue Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
    7474          10 :                                     DataLo.getValueType(), DL, OpsLo, MMO);
    7475             : 
    7476             :   // The order of the Scatter operation after split is well defined. The "Hi"
    7477             :   // part comes after the "Lo". So these two operations should be chained one
    7478             :   // after another.
    7479           5 :   SDValue OpsHi[] = { Lo, DataHi, MaskHi, BasePtr, IndexHi, Scale };
    7480           5 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
    7481          10 :                               DL, OpsHi, MMO);
    7482             : }
    7483             : 
    7484         758 : SDValue DAGCombiner::visitMSTORE(SDNode *N) {
    7485         758 :   if (Level >= AfterLegalizeTypes)
    7486         449 :     return SDValue();
    7487             : 
    7488             :   MaskedStoreSDNode *MST = dyn_cast<MaskedStoreSDNode>(N);
    7489         309 :   SDValue Mask = MST->getMask();
    7490         309 :   SDValue Data  = MST->getValue();
    7491         618 :   EVT VT = Data.getValueType();
    7492             :   SDLoc DL(N);
    7493             : 
    7494             :   // If the MSTORE data type requires splitting and the mask is provided by a
    7495             :   // SETCC, then split both nodes and its operands before legalization. This
    7496             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7497             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7498         309 :   if (Mask.getOpcode() == ISD::SETCC) {
    7499             :     // Check if any splitting is required.
    7500          36 :     if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7501             :         TargetLowering::TypeSplitVector)
    7502          34 :       return SDValue();
    7503             : 
    7504             :     SDValue MaskLo, MaskHi, Lo, Hi;
    7505           2 :     std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7506             : 
    7507           2 :     SDValue Chain = MST->getChain();
    7508           2 :     SDValue Ptr   = MST->getBasePtr();
    7509             : 
    7510           2 :     EVT MemoryVT = MST->getMemoryVT();
    7511           2 :     unsigned Alignment = MST->getOriginalAlignment();
    7512             : 
    7513             :     // if Alignment is equal to the vector size,
    7514             :     // take the half of it for the second part
    7515             :     unsigned SecondHalfAlignment =
    7516           2 :       (Alignment == VT.getSizeInBits() / 8) ? Alignment / 2 : Alignment;
    7517             : 
    7518           2 :     EVT LoMemVT, HiMemVT;
    7519           2 :     std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7520             : 
    7521             :     SDValue DataLo, DataHi;
    7522           2 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    7523             : 
    7524           2 :     MachineMemOperand *MMO = DAG.getMachineFunction().
    7525           6 :       getMachineMemOperand(MST->getPointerInfo(),
    7526             :                            MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
    7527           2 :                            Alignment, MST->getAAInfo(), MST->getRanges());
    7528             : 
    7529           2 :     Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
    7530           2 :                             MST->isTruncatingStore(),
    7531           4 :                             MST->isCompressingStore());
    7532             : 
    7533           2 :     Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    7534           2 :                                      MST->isCompressingStore());
    7535             :     unsigned HiOffset = LoMemVT.getStoreSize();
    7536             : 
    7537           6 :     MMO = DAG.getMachineFunction().getMachineMemOperand(
    7538             :         MST->getPointerInfo().getWithOffset(HiOffset),
    7539             :         MachineMemOperand::MOStore, HiMemVT.getStoreSize(), SecondHalfAlignment,
    7540           2 :         MST->getAAInfo(), MST->getRanges());
    7541             : 
    7542           2 :     Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
    7543           2 :                             MST->isTruncatingStore(),
    7544           4 :                             MST->isCompressingStore());
    7545             : 
    7546           2 :     AddToWorklist(Lo.getNode());
    7547           2 :     AddToWorklist(Hi.getNode());
    7548             : 
    7549           4 :     return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    7550             :   }
    7551         273 :   return SDValue();
    7552             : }
    7553             : 
    7554        1004 : SDValue DAGCombiner::visitMGATHER(SDNode *N) {
    7555        1004 :   if (Level >= AfterLegalizeTypes)
    7556         518 :     return SDValue();
    7557             : 
    7558             :   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
    7559         486 :   SDValue Mask = MGT->getMask();
    7560             :   SDLoc DL(N);
    7561             : 
    7562             :   // If the MGATHER result requires splitting and the mask is provided by a
    7563             :   // SETCC, then split both nodes and its operands before legalization. This
    7564             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7565             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7566             : 
    7567         486 :   if (Mask.getOpcode() != ISD::SETCC)
    7568         456 :     return SDValue();
    7569             : 
    7570          30 :   EVT VT = N->getValueType(0);
    7571             : 
    7572             :   // Check if any splitting is required.
    7573          30 :   if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7574             :       TargetLowering::TypeSplitVector)
    7575          24 :     return SDValue();
    7576             : 
    7577             :   SDValue MaskLo, MaskHi, Lo, Hi;
    7578           6 :   std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7579             : 
    7580           6 :   SDValue PassThru = MGT->getPassThru();
    7581             :   SDValue PassThruLo, PassThruHi;
    7582           6 :   std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, DL);
    7583             : 
    7584             :   EVT LoVT, HiVT;
    7585           6 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
    7586             : 
    7587           6 :   SDValue Chain = MGT->getChain();
    7588           6 :   EVT MemoryVT = MGT->getMemoryVT();
    7589           6 :   unsigned Alignment = MGT->getOriginalAlignment();
    7590             : 
    7591           6 :   EVT LoMemVT, HiMemVT;
    7592           6 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7593             : 
    7594           6 :   SDValue Scale = MGT->getScale();
    7595           6 :   SDValue BasePtr = MGT->getBasePtr();
    7596           6 :   SDValue Index = MGT->getIndex();
    7597             :   SDValue IndexLo, IndexHi;
    7598           6 :   std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
    7599             : 
    7600           6 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    7601          18 :     getMachineMemOperand(MGT->getPointerInfo(),
    7602             :                           MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    7603           6 :                           Alignment, MGT->getAAInfo(), MGT->getRanges());
    7604             : 
    7605           6 :   SDValue OpsLo[] = { Chain, PassThruLo, MaskLo, BasePtr, IndexLo, Scale };
    7606           6 :   Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, DL, OpsLo,
    7607           6 :                            MMO);
    7608             : 
    7609           6 :   SDValue OpsHi[] = { Chain, PassThruHi, MaskHi, BasePtr, IndexHi, Scale };
    7610           6 :   Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, DL, OpsHi,
    7611           6 :                            MMO);
    7612             : 
    7613           6 :   AddToWorklist(Lo.getNode());
    7614           6 :   AddToWorklist(Hi.getNode());
    7615             : 
    7616             :   // Build a factor node to remember that this load is independent of the
    7617             :   // other one.
    7618           6 :   Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
    7619           6 :                       Hi.getValue(1));
    7620             : 
    7621             :   // Legalized the chain result - switch anything that used the old chain to
    7622             :   // use the new one.
    7623          12 :   DAG.ReplaceAllUsesOfValueWith(SDValue(MGT, 1), Chain);
    7624             : 
    7625          12 :   SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    7626             : 
    7627           6 :   SDValue RetOps[] = { GatherRes, Chain };
    7628          12 :   return DAG.getMergeValues(RetOps, DL);
    7629             : }
    7630             : 
    7631        1291 : SDValue DAGCombiner::visitMLOAD(SDNode *N) {
    7632        1291 :   if (Level >= AfterLegalizeTypes)
    7633         722 :     return SDValue();
    7634             : 
    7635             :   MaskedLoadSDNode *MLD = dyn_cast<MaskedLoadSDNode>(N);
    7636         569 :   SDValue Mask = MLD->getMask();
    7637             :   SDLoc DL(N);
    7638             : 
    7639             :   // If the MLOAD result requires splitting and the mask is provided by a
    7640             :   // SETCC, then split both nodes and its operands before legalization. This
    7641             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7642             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7643         569 :   if (Mask.getOpcode() == ISD::SETCC) {
    7644          76 :     EVT VT = N->getValueType(0);
    7645             : 
    7646             :     // Check if any splitting is required.
    7647          76 :     if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7648             :         TargetLowering::TypeSplitVector)
    7649          70 :       return SDValue();
    7650             : 
    7651             :     SDValue MaskLo, MaskHi, Lo, Hi;
    7652           6 :     std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7653             : 
    7654           6 :     SDValue PassThru = MLD->getPassThru();
    7655             :     SDValue PassThruLo, PassThruHi;
    7656           6 :     std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, DL);
    7657             : 
    7658             :     EVT LoVT, HiVT;
    7659          12 :     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
    7660             : 
    7661           6 :     SDValue Chain = MLD->getChain();
    7662           6 :     SDValue Ptr   = MLD->getBasePtr();
    7663           6 :     EVT MemoryVT = MLD->getMemoryVT();
    7664           6 :     unsigned Alignment = MLD->getOriginalAlignment();
    7665             : 
    7666             :     // if Alignment is equal to the vector size,
    7667             :     // take the half of it for the second part
    7668             :     unsigned SecondHalfAlignment =
    7669          12 :       (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
    7670             :          Alignment/2 : Alignment;
    7671             : 
    7672           6 :     EVT LoMemVT, HiMemVT;
    7673           6 :     std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7674             : 
    7675           6 :     MachineMemOperand *MMO = DAG.getMachineFunction().
    7676          18 :     getMachineMemOperand(MLD->getPointerInfo(),
    7677             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    7678           6 :                          Alignment, MLD->getAAInfo(), MLD->getRanges());
    7679             : 
    7680           6 :     Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, PassThruLo, LoMemVT,
    7681           6 :                            MMO, ISD::NON_EXTLOAD, MLD->isExpandingLoad());
    7682             : 
    7683           6 :     Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    7684           6 :                                      MLD->isExpandingLoad());
    7685             :     unsigned HiOffset = LoMemVT.getStoreSize();
    7686             : 
    7687          18 :     MMO = DAG.getMachineFunction().getMachineMemOperand(
    7688             :         MLD->getPointerInfo().getWithOffset(HiOffset),
    7689             :         MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), SecondHalfAlignment,
    7690           6 :         MLD->getAAInfo(), MLD->getRanges());
    7691             : 
    7692           6 :     Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, PassThruHi, HiMemVT,
    7693           6 :                            MMO, ISD::NON_EXTLOAD, MLD->isExpandingLoad());
    7694             : 
    7695           6 :     AddToWorklist(Lo.getNode());
    7696           6 :     AddToWorklist(Hi.getNode());
    7697             : 
    7698             :     // Build a factor node to remember that this load is independent of the
    7699             :     // other one.
    7700           6 :     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
    7701           6 :                         Hi.getValue(1));
    7702             : 
    7703             :     // Legalized the chain result - switch anything that used the old chain to
    7704             :     // use the new one.
    7705          12 :     DAG.ReplaceAllUsesOfValueWith(SDValue(MLD, 1), Chain);
    7706             : 
    7707          12 :     SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    7708             : 
    7709           6 :     SDValue RetOps[] = { LoadRes, Chain };
    7710          12 :     return DAG.getMergeValues(RetOps, DL);
    7711             :   }
    7712         493 :   return SDValue();
    7713             : }
    7714             : 
    7715             : /// A vector select of 2 constant vectors can be simplified to math/logic to
    7716             : /// avoid a variable select instruction and possibly avoid constant loads.
    7717           0 : SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
    7718           0 :   SDValue Cond = N->getOperand(0);
    7719           0 :   SDValue N1 = N->getOperand(1);
    7720           0 :   SDValue N2 = N->getOperand(2);
    7721           0 :   EVT VT = N->getValueType(0);
    7722           0 :   if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
    7723           0 :       !TLI.convertSelectOfConstantsToMath(VT) ||
    7724           0 :       !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) ||
    7725           0 :       !ISD::isBuildVectorOfConstantSDNodes(N2.getNode()))
    7726           0 :     return SDValue();
    7727             : 
    7728             :   // Check if we can use the condition value to increment/decrement a single
    7729             :   // constant value. This simplifies a select to an add and removes a constant
    7730             :   // load/materialization from the general case.
    7731             :   bool AllAddOne = true;
    7732             :   bool AllSubOne = true;
    7733             :   unsigned Elts = VT.getVectorNumElements();
    7734           0 :   for (unsigned i = 0; i != Elts; ++i) {
    7735           0 :     SDValue N1Elt = N1.getOperand(i);
    7736           0 :     SDValue N2Elt = N2.getOperand(i);
    7737           0 :     if (N1Elt.isUndef() || N2Elt.isUndef())
    7738           0 :       continue;
    7739             : 
    7740           0 :     const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
    7741           0 :     const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
    7742           0 :     if (C1 != C2 + 1)
    7743             :       AllAddOne = false;
    7744           0 :     if (C1 != C2 - 1)
    7745             :       AllSubOne = false;
    7746             :   }
    7747             : 
    7748             :   // Further simplifications for the extra-special cases where the constants are
    7749             :   // all 0 or all -1 should be implemented as folds of these patterns.
    7750             :   SDLoc DL(N);
    7751           0 :   if (AllAddOne || AllSubOne) {
    7752             :     // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
    7753             :     // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
    7754           0 :     auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
    7755           0 :     SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
    7756           0 :     return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
    7757             :   }
    7758             : 
    7759             :   // The general case for select-of-constants:
    7760             :   // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
    7761             :   // ...but that only makes sense if a vselect is slower than 2 logic ops, so
    7762             :   // leave that to a machine-specific pass.
    7763           0 :   return SDValue();
    7764             : }
    7765             : 
    7766       39285 : SDValue DAGCombiner::visitVSELECT(SDNode *N) {
    7767       39285 :   SDValue N0 = N->getOperand(0);
    7768       39285 :   SDValue N1 = N->getOperand(1);
    7769       39285 :   SDValue N2 = N->getOperand(2);
    7770             :   SDLoc DL(N);
    7771             : 
    7772             :   // fold (vselect C, X, X) -> X
    7773             :   if (N1 == N2)
    7774           4 :     return N1;
    7775             : 
    7776             :   // Canonicalize integer abs.
    7777             :   // vselect (setg[te] X,  0),  X, -X ->
    7778             :   // vselect (setgt    X, -1),  X, -X ->
    7779             :   // vselect (setl[te] X,  0), -X,  X ->
    7780             :   // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
    7781       39281 :   if (N0.getOpcode() == ISD::SETCC) {
    7782       12620 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    7783       12620 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7784             :     bool isAbs = false;
    7785       12620 :     bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
    7786             : 
    7787       20319 :     if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
    7788       12490 :          (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
    7789       12988 :         N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
    7790         356 :       isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode());
    7791        7729 :     else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
    7792       12351 :              N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
    7793          87 :       isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
    7794             : 
    7795         443 :     if (isAbs) {
    7796         443 :       EVT VT = LHS.getValueType();
    7797         443 :       if (TLI.isOperationLegalOrCustom(ISD::ABS, VT))
    7798         712 :         return DAG.getNode(ISD::ABS, DL, VT, LHS);
    7799             : 
    7800          87 :       SDValue Shift = DAG.getNode(
    7801             :           ISD::SRA, DL, VT, LHS,
    7802          87 :           DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
    7803         174 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
    7804          87 :       AddToWorklist(Shift.getNode());
    7805          87 :       AddToWorklist(Add.getNode());
    7806         174 :       return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
    7807             :     }
    7808             : 
    7809             :     // vselect x, y (fcmp lt x, y) -> fminnum x, y
    7810             :     // vselect x, y (fcmp gt x, y) -> fmaxnum x, y
    7811             :     //
    7812             :     // This is OK if we don't care about what happens if either operand is a
    7813             :     // NaN.
    7814             :     //
    7815       24354 :     EVT VT = N->getValueType(0);
    7816       12177 :     if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N0.getOperand(0), N0.getOperand(1))) {
    7817          35 :       ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7818          35 :       if (SDValue FMinMax = combineMinNumMaxNum(
    7819          70 :             DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG))
    7820          10 :         return FMinMax;
    7821             :     }
    7822             : 
    7823             :     // If this select has a condition (setcc) with narrower operands than the
    7824             :     // select, try to widen the compare to match the select width.
    7825             :     // TODO: This should be extended to handle any constant.
    7826             :     // TODO: This could be extended to handle non-loading patterns, but that
    7827             :     //       requires thorough testing to avoid regressions.
    7828       12167 :     if (isNullConstantOrNullSplatConstant(RHS)) {
    7829             :       EVT NarrowVT = LHS.getValueType();
    7830        4779 :       EVT WideVT = N1.getValueType().changeVectorElementTypeToInteger();
    7831        4779 :       EVT SetCCVT = getSetCCResultType(LHS.getValueType());
    7832             :       unsigned SetCCWidth = SetCCVT.getScalarSizeInBits();
    7833             :       unsigned WideWidth = WideVT.getScalarSizeInBits();
    7834             :       bool IsSigned = isSignedIntSetCC(CC);
    7835        4779 :       auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
    7836          53 :       if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() &&
    7837          51 :           SetCCWidth != 1 && SetCCWidth < WideWidth &&
    7838        4797 :           TLI.isLoadExtLegalOrCustom(LoadExtOpcode, WideVT, NarrowVT) &&
    7839             :           TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) {
    7840             :         // Both compare operands can be widened for free. The LHS can use an
    7841             :         // extended load, and the RHS is a constant:
    7842             :         //   vselect (ext (setcc load(X), C)), N1, N2 -->
    7843             :         //   vselect (setcc extload(X), C'), N1, N2
    7844          10 :         auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    7845          20 :         SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
    7846          20 :         SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
    7847          10 :         EVT WideSetCCVT = getSetCCResultType(WideVT);
    7848          10 :         SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
    7849          20 :         return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
    7850             :       }
    7851             :     }
    7852             :   }
    7853             : 
    7854       38818 :   if (SimplifySelectOps(N, N1, N2))
    7855           6 :     return SDValue(N, 0);  // Don't revisit N.
    7856             : 
    7857             :   // Fold (vselect (build_vector all_ones), N1, N2) -> N1
    7858       38812 :   if (ISD::isBuildVectorAllOnes(N0.getNode()))
    7859          78 :     return N1;
    7860             :   // Fold (vselect (build_vector all_zeros), N1, N2) -> N2
    7861       38734 :   if (ISD::isBuildVectorAllZeros(N0.getNode()))
    7862         368 :     return N2;
    7863             : 
    7864             :   // The ConvertSelectToConcatVector function is assuming both the above
    7865             :   // checks for (vselect (build_vector all{ones,zeros) ...) have been made
    7866             :   // and addressed.
    7867         641 :   if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
    7868       38558 :       N2.getOpcode() == ISD::CONCAT_VECTORS &&
    7869         192 :       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
    7870          43 :     if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
    7871          10 :       return CV;
    7872             :   }
    7873             : 
    7874       38356 :   if (SDValue V = foldVSelectOfConstants(N))
    7875          40 :     return V;
    7876             : 
    7877       38316 :   return SDValue();
    7878             : }
    7879             : 
    7880       16218 : SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
    7881       16218 :   SDValue N0 = N->getOperand(0);
    7882       16218 :   SDValue N1 = N->getOperand(1);
    7883       16218 :   SDValue N2 = N->getOperand(2);
    7884       16218 :   SDValue N3 = N->getOperand(3);
    7885       16218 :   SDValue N4 = N->getOperand(4);
    7886       16218 :   ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
    7887             : 
    7888             :   // fold select_cc lhs, rhs, x, x, cc -> x
    7889             :   if (N2 == N3)
    7890          27 :     return N2;
    7891             : 
    7892             :   // Determine if the condition we're dealing with is constant
    7893       16191 :   if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
    7894       32390 :                                   CC, SDLoc(N), false)) {
    7895         321 :     AddToWorklist(SCC.getNode());
    7896             : 
    7897             :     if (ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
    7898          54 :       if (!SCCC->isNullValue())
    7899          17 :         return N2;    // cond always true -> true val
    7900             :       else
    7901          10 :         return N3;    // cond always false -> false val
    7902         294 :     } else if (SCC->isUndef()) {
    7903             :       // When the condition is UNDEF, just return the first operand. This is
    7904             :       // coherent the DAG creation, no setcc node is created in this case
    7905           0 :       return N2;
    7906         294 :     } else if (SCC.getOpcode() == ISD::SETCC) {
    7907             :       // Fold to a simpler select_cc
    7908         548 :       return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(),
    7909             :                          SCC.getOperand(0), SCC.getOperand(1), N2, N3,
    7910         274 :                          SCC.getOperand(2));
    7911             :     }
    7912             :   }
    7913             : 
    7914             :   // If we can fold this based on the true/false value, do so.
    7915       15890 :   if (SimplifySelectOps(N, N2, N3))
    7916           0 :     return SDValue(N, 0);  // Don't revisit N.
    7917             : 
    7918             :   // fold select_cc into other things, such as min/max/abs
    7919       31780 :   return SimplifySelectCC(SDLoc(N), N0, N1, N2, N3, CC);
    7920             : }
    7921             : 
    7922      297657 : SDValue DAGCombiner::visitSETCC(SDNode *N) {
    7923             :   // setcc is very commonly used as an argument to brcond. This pattern
    7924             :   // also lend itself to numerous combines and, as a result, it is desired
    7925             :   // we keep the argument to a brcond as a setcc as much as possible.
    7926             :   bool PreferSetCC =
    7927      292250 :       N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND;
    7928             : 
    7929             :   SDValue Combined = SimplifySetCC(
    7930             :       N->getValueType(0), N->getOperand(0), N->getOperand(1),
    7931      582441 :       cast<CondCodeSDNode>(N->getOperand(2))->get(), SDLoc(N), !PreferSetCC);
    7932             : 
    7933      297657 :   if (!Combined)
    7934      280201 :     return SDValue();
    7935             : 
    7936             :   // If we prefer to have a setcc, and we don't, we'll try our best to
    7937             :   // recreate one using rebuildSetCC.
    7938       17456 :   if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
    7939         685 :     SDValue NewSetCC = rebuildSetCC(Combined);
    7940             : 
    7941             :     // We don't have anything interesting to combine to.
    7942         685 :     if (NewSetCC.getNode() == N)
    7943         115 :       return SDValue();
    7944             : 
    7945         570 :     if (NewSetCC)
    7946           0 :       return NewSetCC;
    7947             :   }
    7948             : 
    7949       17341 :   return Combined;
    7950             : }
    7951             : 
    7952           0 : SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
    7953           0 :   SDValue LHS = N->getOperand(0);
    7954           0 :   SDValue RHS = N->getOperand(1);
    7955           0 :   SDValue Carry = N->getOperand(2);
    7956           0 :   SDValue Cond = N->getOperand(3);
    7957             : 
    7958             :   // If Carry is false, fold to a regular SETCC.
    7959           0 :   if (isNullConstant(Carry))
    7960           0 :     return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
    7961             : 
    7962           0 :   return SDValue();
    7963             : }
    7964             : 
    7965             : /// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
    7966             : /// a build_vector of constants.
    7967             : /// This function is called by the DAGCombiner when visiting sext/zext/aext
    7968             : /// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
    7969             : /// Vector extends are not folded if operations are legal; this is to
    7970             : /// avoid introducing illegal build_vector dag nodes.
    7971      288253 : static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
    7972             :                                          SelectionDAG &DAG, bool LegalTypes,
    7973             :                                          bool LegalOperations) {
    7974      288253 :   unsigned Opcode = N->getOpcode();
    7975      288253 :   SDValue N0 = N->getOperand(0);
    7976      576506 :   EVT VT = N->getValueType(0);
    7977             : 
    7978             :   assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
    7979             :          Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
    7980             :          Opcode == ISD::ZERO_EXTEND_VECTOR_INREG)
    7981             :          && "Expected EXTEND dag node in input!");
    7982             : 
    7983             :   // fold (sext c1) -> c1
    7984             :   // fold (zext c1) -> c1
    7985             :   // fold (aext c1) -> c1
    7986             :   if (isa<ConstantSDNode>(N0))
    7987         567 :     return DAG.getNode(Opcode, SDLoc(N), VT, N0).getNode();
    7988             : 
    7989             :   // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
    7990             :   // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
    7991             :   // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
    7992      287686 :   EVT SVT = VT.getScalarType();
    7993      308737 :   if (!(VT.isVector() &&
    7994        9347 :       (!LegalTypes || (!LegalOperations && TLI.isTypeLegal(SVT))) &&
    7995       21051 :       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())))
    7996      287591 :     return nullptr;
    7997             : 
    7998             :   // We can fold this node into a build_vector.
    7999          95 :   unsigned VTBits = SVT.getSizeInBits();
    8000         190 :   unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits();
    8001             :   SmallVector<SDValue, 8> Elts;
    8002             :   unsigned NumElts = VT.getVectorNumElements();
    8003             :   SDLoc DL(N);
    8004             : 
    8005         507 :   for (unsigned i=0; i != NumElts; ++i) {
    8006         412 :     SDValue Op = N0->getOperand(i);
    8007         412 :     if (Op->isUndef()) {
    8008          86 :       Elts.push_back(DAG.getUNDEF(SVT));
    8009          86 :       continue;
    8010             :     }
    8011             : 
    8012             :     SDLoc DL(Op);
    8013             :     // Get the constant value and if needed trunc it to the size of the type.
    8014             :     // Nodes like build_vector might have constants wider than the scalar type.
    8015         652 :     APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
    8016         326 :     if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
    8017         112 :       Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
    8018             :     else
    8019         214 :       Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
    8020             :   }
    8021             : 
    8022          95 :   return DAG.getBuildVector(VT, DL, Elts).getNode();
    8023             : }
    8024             : 
    8025             : // ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
    8026             : // "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
    8027             : // transformation. Returns true if extension are possible and the above
    8028             : // mentioned transformation is profitable.
    8029           0 : static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
    8030             :                                     unsigned ExtOpc,
    8031             :                                     SmallVectorImpl<SDNode *> &ExtendNodes,
    8032             :                                     const TargetLowering &TLI) {
    8033             :   bool HasCopyToRegUses = false;
    8034           0 :   bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType());
    8035           0 :   for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
    8036             :                             UE = N0.getNode()->use_end();
    8037           0 :        UI != UE; ++UI) {
    8038           0 :     SDNode *User = *UI;
    8039           0 :     if (User == N)
    8040           0 :       continue;
    8041           0 :     if (UI.getUse().getResNo() != N0.getResNo())
    8042           0 :       continue;
    8043             :     // FIXME: Only extend SETCC N, N and SETCC N, c for now.
    8044           0 :     if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
    8045           0 :       ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
    8046           0 :       if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
    8047             :         // Sign bits will be lost after a zext.
    8048           0 :         return false;
    8049             :       bool Add = false;
    8050           0 :       for (unsigned i = 0; i != 2; ++i) {
    8051           0 :         SDValue UseOp = User->getOperand(i);
    8052           0 :         if (UseOp == N0)
    8053           0 :           continue;
    8054             :         if (!isa<ConstantSDNode>(UseOp))
    8055           0 :           return false;
    8056             :         Add = true;
    8057             :       }
    8058           0 :       if (Add)
    8059           0 :         ExtendNodes.push_back(User);
    8060           0 :       continue;
    8061             :     }
    8062             :     // If truncates aren't free and there are users we can't
    8063             :     // extend, it isn't worthwhile.
    8064           0 :     if (!isTruncFree)
    8065           0 :       return false;
    8066             :     // Remember if this value is live-out.
    8067           0 :     if (User->getOpcode() == ISD::CopyToReg)
    8068             :       HasCopyToRegUses = true;
    8069             :   }
    8070             : 
    8071           0 :   if (HasCopyToRegUses) {
    8072             :     bool BothLiveOut = false;
    8073           0 :     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
    8074           0 :          UI != UE; ++UI) {
    8075             :       SDUse &Use = UI.getUse();
    8076           0 :       if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
    8077             :         BothLiveOut = true;
    8078             :         break;
    8079             :       }
    8080             :     }
    8081           0 :     if (BothLiveOut)
    8082             :       // Both unextended and extended values are live out. There had better be
    8083             :       // a good reason for the transformation.
    8084           0 :       return ExtendNodes.size();
    8085             :   }
    8086             :   return true;
    8087             : }
    8088             : 
    8089           0 : void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
    8090             :                                   SDValue OrigLoad, SDValue ExtLoad,
    8091             :                                   ISD::NodeType ExtType) {
    8092             :   // Extend SetCC uses if necessary.
    8093           0 :   SDLoc DL(ExtLoad);
    8094           0 :   for (SDNode *SetCC : SetCCs) {
    8095             :     SmallVector<SDValue, 4> Ops;
    8096             : 
    8097           0 :     for (unsigned j = 0; j != 2; ++j) {
    8098           0 :       SDValue SOp = SetCC->getOperand(j);
    8099           0 :       if (SOp == OrigLoad)
    8100           0 :         Ops.push_back(ExtLoad);
    8101             :       else
    8102           0 :         Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
    8103             :     }
    8104             : 
    8105           0 :     Ops.push_back(SetCC->getOperand(2));
    8106           0 :     CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
    8107             :   }
    8108           0 : }
    8109             : 
    8110             : // FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
    8111      128330 : SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
    8112      128330 :   SDValue N0 = N->getOperand(0);
    8113      256660 :   EVT DstVT = N->getValueType(0);
    8114             :   EVT SrcVT = N0.getValueType();
    8115             : 
    8116             :   assert((N->getOpcode() == ISD::SIGN_EXTEND ||
    8117             :           N->getOpcode() == ISD::ZERO_EXTEND) &&
    8118             :          "Unexpected node type (not an extend)!");
    8119             : 
    8120             :   // fold (sext (load x)) to multiple smaller sextloads; same for zext.
    8121             :   // For example, on a target with legal v4i32, but illegal v8i32, turn:
    8122             :   //   (v8i32 (sext (v8i16 (load x))))
    8123             :   // into:
    8124             :   //   (v8i32 (concat_vectors (v4i32 (sextload x)),
    8125             :   //                          (v4i32 (sextload (x + 16)))))
    8126             :   // Where uses of the original load, i.e.:
    8127             :   //   (v8i16 (load x))
    8128             :   // are replaced with:
    8129             :   //   (v8i16 (truncate
    8130             :   //     (v8i32 (concat_vectors (v4i32 (sextload x)),
    8131             :   //                            (v4i32 (sextload (x + 16)))))))
    8132             :   //
    8133             :   // This combine is only applicable to illegal, but splittable, vectors.
    8134             :   // All legal types, and illegal non-vector types, are handled elsewhere.
    8135             :   // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
    8136             :   //
    8137      128330 :   if (N0->getOpcode() != ISD::LOAD)
    8138      124292 :     return SDValue();
    8139             : 
    8140             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8141             : 
    8142        3793 :   if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
    8143        6249 :       !N0.hasOneUse() || LN0->isVolatile() || !DstVT.isVector() ||
    8144        2867 :       !DstVT.isPow2VectorType() || !TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
    8145        3636 :     return SDValue();
    8146             : 
    8147             :   SmallVector<SDNode *, 4> SetCCs;
    8148         804 :   if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI))
    8149           0 :     return SDValue();
    8150             : 
    8151             :   ISD::LoadExtType ExtType =
    8152         402 :       N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
    8153             : 
    8154             :   // Try to split the vector types to get down to legal types.
    8155         402 :   EVT SplitSrcVT = SrcVT;
    8156         402 :   EVT SplitDstVT = DstVT;
    8157        1666 :   while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
    8158             :          SplitSrcVT.getVectorNumElements() > 1) {
    8159         531 :     SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
    8160         531 :     SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
    8161             :   }
    8162             : 
    8163             :   if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
    8164         202 :     return SDValue();
    8165             : 
    8166             :   SDLoc DL(N);
    8167             :   const unsigned NumSplits =
    8168         200 :       DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
    8169             :   const unsigned Stride = SplitSrcVT.getStoreSize();
    8170             :   SmallVector<SDValue, 4> Loads;
    8171             :   SmallVector<SDValue, 4> Chains;
    8172             : 
    8173         200 :   SDValue BasePtr = LN0->getBasePtr();
    8174         613 :   for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
    8175         413 :     const unsigned Offset = Idx * Stride;
    8176         413 :     const unsigned Align = MinAlign(LN0->getAlignment(), Offset);
    8177             : 
    8178         413 :     SDValue SplitLoad = DAG.getExtLoad(
    8179         413 :         ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), BasePtr,
    8180             :         LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT, Align,
    8181        1652 :         LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    8182             : 
    8183         413 :     BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
    8184         413 :                           DAG.getConstant(Stride, DL, BasePtr.getValueType()));
    8185             : 
    8186         413 :     Loads.push_back(SplitLoad.getValue(0));
    8187         413 :     Chains.push_back(SplitLoad.getValue(1));
    8188             :   }
    8189             : 
    8190         400 :   SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
    8191         400 :   SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
    8192             : 
    8193             :   // Simplify TF.
    8194         200 :   AddToWorklist(NewChain.getNode());
    8195             : 
    8196         200 :   CombineTo(N, NewValue);
    8197             : 
    8198             :   // Replace uses of the original load (before extension)
    8199             :   // with a truncate of the concatenated sextloaded vectors.
    8200             :   SDValue Trunc =
    8201         206 :       DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
    8202         400 :   ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode());
    8203             :   CombineTo(N0.getNode(), Trunc, NewChain);
    8204         200 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8205             : }
    8206             : 
    8207             : // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
    8208             : //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
    8209       98193 : SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
    8210             :   assert(N->getOpcode() == ISD::ZERO_EXTEND);
    8211       98193 :   EVT VT = N->getValueType(0);
    8212             : 
    8213             :   // and/or/xor
    8214       98193 :   SDValue N0 = N->getOperand(0);
    8215       95924 :   if (!(N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8216       15578 :         N0.getOpcode() == ISD::XOR) ||
    8217      111502 :       N0.getOperand(1).getOpcode() != ISD::Constant ||
    8218       13353 :       (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT)))
    8219       84840 :     return SDValue();
    8220             : 
    8221             :   // shl/shr
    8222       13353 :   SDValue N1 = N0->getOperand(0);
    8223       13353 :   if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) ||
    8224       14369 :       N1.getOperand(1).getOpcode() != ISD::Constant ||
    8225         324 :       (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT)))
    8226       12322 :     return SDValue();
    8227             : 
    8228             :   // load
    8229        1031 :   if (!isa<LoadSDNode>(N1.getOperand(0)))
    8230         979 :     return SDValue();
    8231             :   LoadSDNode *Load = cast<LoadSDNode>(N1.getOperand(0));
    8232             :   EVT MemVT = Load->getMemoryVT();
    8233         100 :   if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) ||
    8234          99 :       Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
    8235           5 :     return SDValue();
    8236             : 
    8237             : 
    8238             :   // If the shift op is SHL, the logic op must be AND, otherwise the result
    8239             :   // will be wrong.
    8240          47 :   if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND)
    8241           2 :     return SDValue();
    8242             : 
    8243          78 :   if (!N0.hasOneUse() || !N1.hasOneUse())
    8244          12 :     return SDValue();
    8245             : 
    8246             :   SmallVector<SDNode*, 4> SetCCs;
    8247          66 :   if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0),
    8248             :                                ISD::ZERO_EXTEND, SetCCs, TLI))
    8249           9 :     return SDValue();
    8250             : 
    8251             :   // Actually do the transformation.
    8252          24 :   SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
    8253             :                                    Load->getChain(), Load->getBasePtr(),
    8254          32 :                                    Load->getMemoryVT(), Load->getMemOperand());
    8255             : 
    8256             :   SDLoc DL1(N1);
    8257          24 :   SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
    8258          24 :                               N1.getOperand(1));
    8259             : 
    8260          24 :   APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8261          48 :   Mask = Mask.zext(VT.getSizeInBits());
    8262             :   SDLoc DL0(N0);
    8263          24 :   SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
    8264          24 :                             DAG.getConstant(Mask, DL0, VT));
    8265             : 
    8266          24 :   ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
    8267          24 :   CombineTo(N, And);
    8268          24 :   if (SDValue(Load, 0).hasOneUse()) {
    8269          46 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
    8270             :   } else {
    8271           1 :     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
    8272           2 :                                 Load->getValueType(0), ExtLoad);
    8273             :     CombineTo(Load, Trunc, ExtLoad.getValue(1));
    8274             :   }
    8275          24 :   return SDValue(N,0); // Return N so it doesn't get rechecked!
    8276             : }
    8277             : 
    8278             : /// If we're narrowing or widening the result of a vector select and the final
    8279             : /// size is the same size as a setcc (compare) feeding the select, then try to
    8280             : /// apply the cast operation to the select's operands because matching vector
    8281             : /// sizes for a select condition and other operands should be more efficient.
    8282      953308 : SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
    8283      953308 :   unsigned CastOpcode = Cast->getOpcode();
    8284             :   assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
    8285             :           CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
    8286             :           CastOpcode == ISD::FP_ROUND) &&
    8287             :          "Unexpected opcode for vector select narrowing/widening");
    8288             : 
    8289             :   // We only do this transform before legal ops because the pattern may be
    8290             :   // obfuscated by target-specific operations after legalization. Do not create
    8291             :   // an illegal select op, however, because that may be difficult to lower.
    8292      953308 :   EVT VT = Cast->getValueType(0);
    8293      953308 :   if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
    8294      479725 :     return SDValue();
    8295             : 
    8296      473583 :   SDValue VSel = Cast->getOperand(0);
    8297      473617 :   if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
    8298          34 :       VSel.getOperand(0).getOpcode() != ISD::SETCC)
    8299      473549 :     return SDValue();
    8300             : 
    8301             :   // Does the setcc have the same vector size as the casted select?
    8302          34 :   SDValue SetCC = VSel.getOperand(0);
    8303          68 :   EVT SetCCVT = getSetCCResultType(SetCC.getOperand(0).getValueType());
    8304          34 :   if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
    8305          18 :     return SDValue();
    8306             : 
    8307             :   // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
    8308          16 :   SDValue A = VSel.getOperand(1);
    8309          16 :   SDValue B = VSel.getOperand(2);
    8310          16 :   SDValue CastA, CastB;
    8311             :   SDLoc DL(Cast);
    8312          16 :   if (CastOpcode == ISD::FP_ROUND) {
    8313             :     // FP_ROUND (fptrunc) has an extra flag operand to pass along.
    8314          12 :     CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
    8315          12 :     CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
    8316             :   } else {
    8317          24 :     CastA = DAG.getNode(CastOpcode, DL, VT, A);
    8318          24 :     CastB = DAG.getNode(CastOpcode, DL, VT, B);
    8319             :   }
    8320          32 :   return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
    8321             : }
    8322             : 
    8323             : // fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    8324             : // fold ([s|z]ext (     extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    8325      128076 : static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
    8326             :                                      const TargetLowering &TLI, EVT VT,
    8327             :                                      bool LegalOperations, SDNode *N,
    8328             :                                      SDValue N0, ISD::LoadExtType ExtLoadType) {
    8329             :   SDNode *N0Node = N0.getNode();
    8330      128076 :   bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD) ? ISD::isSEXTLoad(N0Node)
    8331             :                                                    : ISD::isZEXTLoad(N0Node);
    8332             :   if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) ||
    8333      128271 :       !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse())
    8334      127999 :     return {};
    8335             : 
    8336             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8337          77 :   EVT MemVT = LN0->getMemoryVT();
    8338         146 :   if ((LegalOperations || LN0->isVolatile()) &&
    8339          69 :       !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT))
    8340          56 :     return {};
    8341             : 
    8342             :   SDValue ExtLoad =
    8343          21 :       DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
    8344          23 :                      LN0->getBasePtr(), MemVT, LN0->getMemOperand());
    8345          21 :   Combiner.CombineTo(N, ExtLoad);
    8346          42 :   DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8347          21 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8348             : }
    8349             : 
    8350             : // fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    8351             : // Only generate vector extloads when 1) they're legal, and 2) they are
    8352             : // deemed desirable by the target.
    8353      150701 : static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
    8354             :                                   const TargetLowering &TLI, EVT VT,
    8355             :                                   bool LegalOperations, SDNode *N, SDValue N0,
    8356             :                                   ISD::LoadExtType ExtLoadType,
    8357             :                                   ISD::NodeType ExtOpc) {
    8358             :   if (!ISD::isNON_EXTLoad(N0.getNode()) ||
    8359       26164 :       !ISD::isUNINDEXEDLoad(N0.getNode()) ||
    8360       24576 :       ((LegalOperations || VT.isVector() ||
    8361       26155 :         cast<LoadSDNode>(N0)->isVolatile()) &&
    8362        5539 :        !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType())))
    8363      126400 :     return {};
    8364             : 
    8365             :   bool DoXform = true;
    8366             :   SmallVector<SDNode *, 4> SetCCs;
    8367       24301 :   if (!N0.hasOneUse())
    8368        2351 :     DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI);
    8369       24301 :   if (VT.isVector())
    8370        4698 :     DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
    8371       24301 :   if (!DoXform)
    8372        1930 :     return {};
    8373             : 
    8374             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8375       22371 :   SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
    8376             :                                    LN0->getBasePtr(), N0.getValueType(),
    8377       26397 :                                    LN0->getMemOperand());
    8378       22371 :   Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
    8379             :   // If the load value is used only by N, replace it via CombineTo N.
    8380             :   bool NoReplaceTrunc = SDValue(LN0, 0).hasOneUse();
    8381       22371 :   Combiner.CombineTo(N, ExtLoad);
    8382       22371 :   if (NoReplaceTrunc) {
    8383       21624 :     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8384             :   } else {
    8385             :     SDValue Trunc =
    8386        1494 :         DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
    8387             :     Combiner.CombineTo(LN0, Trunc, ExtLoad.getValue(1));
    8388             :   }
    8389       22371 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8390             : }
    8391             : 
    8392      128054 : static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
    8393             :                                        bool LegalOperations) {
    8394             :   assert((N->getOpcode() == ISD::SIGN_EXTEND ||
    8395             :           N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext");
    8396             : 
    8397      128054 :   SDValue SetCC = N->getOperand(0);
    8398      115826 :   if (LegalOperations || SetCC.getOpcode() != ISD::SETCC ||
    8399      128054 :       !SetCC.hasOneUse() || SetCC.getValueType() != MVT::i1)
    8400      122834 :     return SDValue();
    8401             : 
    8402        5220 :   SDValue X = SetCC.getOperand(0);
    8403        5220 :   SDValue Ones = SetCC.getOperand(1);
    8404        5220 :   ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get();
    8405       10440 :   EVT VT = N->getValueType(0);
    8406             :   EVT XVT = X.getValueType();
    8407             :   // setge X, C is canonicalized to setgt, so we do not need to match that
    8408             :   // pattern. The setlt sibling is folded in SimplifySelectCC() because it does
    8409             :   // not require the 'not' op.
    8410        5220 :   if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) {
    8411             :     // Invert and smear/shift the sign bit:
    8412             :     // sext i1 (setgt iN X, -1) --> sra (not X), (N - 1)
    8413             :     // zext i1 (setgt iN X, -1) --> srl (not X), (N - 1)
    8414             :     SDLoc DL(N);
    8415          32 :     SDValue NotX = DAG.getNOT(DL, X, VT);
    8416          32 :     SDValue ShiftAmount = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
    8417          32 :     auto ShiftOpcode = N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL;
    8418          32 :     return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount);
    8419             :   }
    8420        5188 :   return SDValue();
    8421             : }
    8422             : 
    8423       52105 : SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
    8424       52105 :   SDValue N0 = N->getOperand(0);
    8425      104210 :   EVT VT = N->getValueType(0);
    8426             :   SDLoc DL(N);
    8427             : 
    8428       52105 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8429       52105 :                                               LegalOperations))
    8430          73 :     return SDValue(Res, 0);
    8431             : 
    8432             :   // fold (sext (sext x)) -> (sext x)
    8433             :   // fold (sext (aext x)) -> (sext x)
    8434      104064 :   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    8435          12 :     return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
    8436             : 
    8437       52026 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8438             :     // fold (sext (truncate (load x))) -> (sext (smaller load x))
    8439             :     // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
    8440        6743 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8441          39 :       SDNode *oye = N0.getOperand(0).getNode();
    8442          39 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8443          39 :         CombineTo(N0.getNode(), NarrowLoad);
    8444             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8445          39 :         AddToWorklist(oye);
    8446             :       }
    8447          39 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8448             :     }
    8449             : 
    8450             :     // See if the value being truncated is already sign extended.  If so, just
    8451             :     // eliminate the trunc/sext pair.
    8452        6704 :     SDValue Op = N0.getOperand(0);
    8453        6704 :     unsigned OpBits   = Op.getScalarValueSizeInBits();
    8454        6704 :     unsigned MidBits  = N0.getScalarValueSizeInBits();
    8455             :     unsigned DestBits = VT.getScalarSizeInBits();
    8456        6704 :     unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
    8457             : 
    8458        6704 :     if (OpBits == DestBits) {
    8459             :       // Op is i32, Mid is i8, and Dest is i32.  If Op has more than 24 sign
    8460             :       // bits, it is already ready.
    8461        3866 :       if (NumSignBits > DestBits-MidBits)
    8462         903 :         return Op;
    8463        2838 :     } else if (OpBits < DestBits) {
    8464             :       // Op is i32, Mid is i8, and Dest is i64.  If Op has more than 24 sign
    8465             :       // bits, just sext from i32.
    8466        2525 :       if (NumSignBits > OpBits-MidBits)
    8467          50 :         return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
    8468             :     } else {
    8469             :       // Op is i64, Mid is i8, and Dest is i32.  If Op has more than 56 sign
    8470             :       // bits, just truncate to i32.
    8471         313 :       if (NumSignBits > OpBits-MidBits)
    8472          44 :         return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
    8473             :     }
    8474             : 
    8475             :     // fold (sext (truncate x)) -> (sextinreg x).
    8476        5754 :     if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
    8477             :                                                  N0.getValueType())) {
    8478        5754 :       if (OpBits < DestBits)
    8479        2522 :         Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
    8480        3254 :       else if (OpBits > DestBits)
    8481         291 :         Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
    8482        5754 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
    8483       11508 :                          DAG.getValueType(N0.getValueType()));
    8484             :     }
    8485             :   }
    8486             : 
    8487             :   // Try to simplify (sext (load x)).
    8488       45283 :   if (SDValue foldedExt =
    8489       45283 :           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
    8490       45283 :                              ISD::SEXTLOAD, ISD::SIGN_EXTEND))
    8491       15241 :     return foldedExt;
    8492             : 
    8493             :   // fold (sext (load x)) to multiple smaller sextloads.
    8494             :   // Only on illegal but splittable vectors.
    8495       30042 :   if (SDValue ExtLoad = CombineExtLoad(N))
    8496         135 :     return ExtLoad;
    8497             : 
    8498             :   // Try to simplify (sext (sextload x)).
    8499       29907 :   if (SDValue foldedExt = tryToFoldExtOfExtload(
    8500       29907 :           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
    8501           4 :     return foldedExt;
    8502             : 
    8503             :   // fold (sext (and/or/xor (load x), cst)) ->
    8504             :   //      (and/or/xor (sextload x), (sext cst))
    8505       29903 :   if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8506        1245 :        N0.getOpcode() == ISD::XOR) &&
    8507          31 :       isa<LoadSDNode>(N0.getOperand(0)) &&
    8508       29934 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8509           1 :       (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    8510             :     LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    8511             :     EVT MemVT = LN00->getMemoryVT();
    8512           1 :     if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) &&
    8513           2 :       LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
    8514             :       SmallVector<SDNode*, 4> SetCCs;
    8515           1 :       bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
    8516             :                                              ISD::SIGN_EXTEND, SetCCs, TLI);
    8517           1 :       if (DoXform) {
    8518           2 :         SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
    8519             :                                          LN00->getChain(), LN00->getBasePtr(),
    8520             :                                          LN00->getMemoryVT(),
    8521           2 :                                          LN00->getMemOperand());
    8522           2 :         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8523           1 :         Mask = Mask.sext(VT.getSizeInBits());
    8524           1 :         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
    8525           1 :                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
    8526           2 :         ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND);
    8527           1 :         bool NoReplaceTruncAnd = !N0.hasOneUse();
    8528             :         bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
    8529           1 :         CombineTo(N, And);
    8530             :         // If N0 has multiple uses, change other uses as well.
    8531           1 :         if (NoReplaceTruncAnd) {
    8532             :           SDValue TruncAnd =
    8533           0 :               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
    8534           0 :           CombineTo(N0.getNode(), TruncAnd);
    8535             :         }
    8536           1 :         if (NoReplaceTrunc) {
    8537           2 :           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
    8538             :         } else {
    8539           0 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
    8540           0 :                                       LN00->getValueType(0), ExtLoad);
    8541             :           CombineTo(LN00, Trunc, ExtLoad.getValue(1));
    8542             :         }
    8543           1 :         return SDValue(N,0); // Return N so it doesn't get rechecked!
    8544             :       }
    8545             :     }
    8546             :   }
    8547             : 
    8548       29902 :   if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
    8549          10 :     return V;
    8550             : 
    8551       59784 :   if (N0.getOpcode() == ISD::SETCC) {
    8552        5403 :     SDValue N00 = N0.getOperand(0);
    8553        5403 :     SDValue N01 = N0.getOperand(1);
    8554        5403 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    8555       10806 :     EVT N00VT = N0.getOperand(0).getValueType();
    8556             : 
    8557             :     // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
    8558             :     // Only do this before legalize for now.
    8559        5403 :     if (VT.isVector() && !LegalOperations &&
    8560        2899 :         TLI.getBooleanContents(N00VT) ==
    8561             :             TargetLowering::ZeroOrNegativeOneBooleanContent) {
    8562             :       // On some architectures (such as SSE/NEON/etc) the SETCC result type is
    8563             :       // of the same size as the compared operands. Only optimize sext(setcc())
    8564             :       // if this is the case.
    8565        2870 :       EVT SVT = getSetCCResultType(N00VT);
    8566             : 
    8567             :       // We know that the # elements of the results is the same as the
    8568             :       // # elements of the compare (and the # elements of the compare result
    8569             :       // for that matter).  Check to see that they are the same size.  If so,
    8570             :       // we know that the element size of the sext'd result matches the
    8571             :       // element size of the compare operands.
    8572        2870 :       if (VT.getSizeInBits() == SVT.getSizeInBits())
    8573        2635 :         return DAG.getSetCC(DL, VT, N00, N01, CC);
    8574             : 
    8575             :       // If the desired elements are smaller or larger than the source
    8576             :       // elements, we can use a matching integer vector type and then
    8577             :       // truncate/sign extend.
    8578         514 :       EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
    8579         517 :       if (SVT == MatchingVecType) {
    8580         279 :         SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
    8581         279 :         return DAG.getSExtOrTrunc(VsetCC, DL, VT);
    8582             :       }
    8583             :     }
    8584             : 
    8585             :     // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
    8586             :     // Here, T can be 1 or -1, depending on the type of the setcc and
    8587             :     // getBooleanContents().
    8588        2768 :     unsigned SetCCWidth = N0.getScalarValueSizeInBits();
    8589             : 
    8590             :     // To determine the "true" side of the select, we need to know the high bit
    8591             :     // of the value returned by the setcc if it evaluates to true.
    8592             :     // If the type of the setcc is i1, then the true case of the select is just
    8593             :     // sext(i1 1), that is, -1.
    8594             :     // If the type of the setcc is larger (say, i8) then the value of the high
    8595             :     // bit depends on getBooleanContents(), so ask TLI for a real "true" value
    8596             :     // of the appropriate width.
    8597             :     SDValue ExtTrueVal = (SetCCWidth == 1)
    8598        2767 :                              ? DAG.getAllOnesConstant(DL, VT)
    8599        2768 :                              : DAG.getBoolConstant(true, DL, VT, N00VT);
    8600        2768 :     SDValue Zero = DAG.getConstant(0, DL, VT);
    8601        2768 :     if (SDValue SCC =
    8602        2768 :             SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC, true))
    8603          22 :       return SCC;
    8604             : 
    8605        2746 :     if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) {
    8606         871 :       EVT SetCCVT = getSetCCResultType(N00VT);
    8607             :       // Don't do this transform for i1 because there's a select transform
    8608             :       // that would reverse it.
    8609             :       // TODO: We should not do this transform at all without a target hook
    8610             :       // because a sext is likely cheaper than a select?
    8611         871 :       if (SetCCVT.getScalarSizeInBits() != 1 &&
    8612         107 :           (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) {
    8613         107 :         SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
    8614         107 :         return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
    8615             :       }
    8616             :     }
    8617             :   }
    8618             : 
    8619             :   // fold (sext x) -> (zext x) if the sign bit is known zero.
    8620       53591 :   if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
    8621       26463 :       DAG.SignBitIsZero(N0))
    8622         884 :     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0);
    8623             : 
    8624       26686 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    8625           4 :     return NewVSel;
    8626             : 
    8627       26682 :   return SDValue();
    8628             : }
    8629             : 
    8630             : // isTruncateOf - If N is a truncate of some other value, return true, record
    8631             : // the value being truncated in Op and which of Op's bits are zero/one in Known.
    8632             : // This function computes KnownBits to avoid a duplicated call to
    8633             : // computeKnownBits in the caller.
    8634           0 : static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
    8635             :                          KnownBits &Known) {
    8636           0 :   if (N->getOpcode() == ISD::TRUNCATE) {
    8637           0 :     Op = N->getOperand(0);
    8638           0 :     DAG.computeKnownBits(Op, Known);
    8639           0 :     return true;
    8640             :   }
    8641             : 
    8642           0 :   if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
    8643           0 :       cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
    8644           0 :     return false;
    8645             : 
    8646           0 :   SDValue Op0 = N->getOperand(0);
    8647           0 :   SDValue Op1 = N->getOperand(1);
    8648             :   assert(Op0.getValueType() == Op1.getValueType());
    8649             : 
    8650           0 :   if (isNullConstant(Op0))
    8651           0 :     Op = Op1;
    8652           0 :   else if (isNullConstant(Op1))
    8653           0 :     Op = Op0;
    8654             :   else
    8655           0 :     return false;
    8656             : 
    8657           0 :   DAG.computeKnownBits(Op, Known);
    8658             : 
    8659           0 :   if (!(Known.Zero | 1).isAllOnesValue())
    8660           0 :     return false;
    8661             : 
    8662             :   return true;
    8663             : }
    8664             : 
    8665      140434 : SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
    8666      140434 :   SDValue N0 = N->getOperand(0);
    8667      140434 :   EVT VT = N->getValueType(0);
    8668             : 
    8669      140434 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8670      140434 :                                               LegalOperations))
    8671         452 :     return SDValue(Res, 0);
    8672             : 
    8673             :   // fold (zext (zext x)) -> (zext x)
    8674             :   // fold (zext (aext x)) -> (zext x)
    8675      279964 :   if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    8676         279 :     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
    8677         558 :                        N0.getOperand(0));
    8678             : 
    8679             :   // fold (zext (truncate x)) -> (zext x) or
    8680             :   //      (zext (truncate x)) -> (truncate x)
    8681             :   // This is valid when the truncated bits of x are already zero.
    8682             :   // FIXME: We should extend this to work for vectors too.
    8683      139703 :   SDValue Op;
    8684      139703 :   KnownBits Known;
    8685      139703 :   if (!VT.isVector() && isTruncateOf(DAG, N0, Op, Known)) {
    8686             :     APInt TruncatedBits =
    8687       33749 :       (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ?
    8688             :       APInt(Op.getValueSizeInBits(), 0) :
    8689             :       APInt::getBitsSet(Op.getValueSizeInBits(),
    8690             :                         N0.getValueSizeInBits(),
    8691       33747 :                         std::min(Op.getValueSizeInBits(),
    8692       67498 :                                  VT.getSizeInBits()));
    8693       33749 :     if (TruncatedBits.isSubsetOf(Known.Zero))
    8694       31997 :       return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
    8695             :   }
    8696             : 
    8697             :   // fold (zext (truncate x)) -> (and x, mask)
    8698      248654 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8699             :     // fold (zext (truncate (load x))) -> (zext (smaller load x))
    8700             :     // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
    8701       18558 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8702          13 :       SDNode *oye = N0.getOperand(0).getNode();
    8703          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8704          13 :         CombineTo(N0.getNode(), NarrowLoad);
    8705             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8706          13 :         AddToWorklist(oye);
    8707             :       }
    8708          13 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8709             :     }
    8710             : 
    8711       37090 :     EVT SrcVT = N0.getOperand(0).getValueType();
    8712       18545 :     EVT MinVT = N0.getValueType();
    8713             : 
    8714             :     // Try to mask before the extension to avoid having to generate a larger mask,
    8715             :     // possibly over several sub-vectors.
    8716       30565 :     if (SrcVT.bitsLT(VT) && VT.isVector()) {
    8717          51 :       if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
    8718             :                                TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) {
    8719          51 :         SDValue Op = N0.getOperand(0);
    8720         102 :         Op = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
    8721          51 :         AddToWorklist(Op.getNode());
    8722          51 :         SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
    8723             :         // Transfer the debug info; the new node is equivalent to N0.
    8724          51 :         DAG.transferDbgValues(N0, ZExtOrTrunc);
    8725          51 :         return ZExtOrTrunc;
    8726             :       }
    8727             :     }
    8728             : 
    8729       18494 :     if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
    8730       18644 :       SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
    8731       18494 :       AddToWorklist(Op.getNode());
    8732       36988 :       SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
    8733             :       // We may safely transfer the debug info describing the truncate node over
    8734             :       // to the equivalent and operation.
    8735       18494 :       DAG.transferDbgValues(N0, And);
    8736       18494 :       return And;
    8737             :     }
    8738             :   }
    8739             : 
    8740             :   // Fold (zext (and (trunc x), cst)) -> (and x, cst),
    8741             :   // if either of the casts is not free.
    8742      105769 :   if (N0.getOpcode() == ISD::AND &&
    8743        2639 :       N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
    8744      106620 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8745         708 :       (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
    8746         409 :                            N0.getValueType()) ||
    8747         110 :        !TLI.isZExtFree(N0.getValueType(), VT))) {
    8748         702 :     SDValue X = N0.getOperand(0).getOperand(0);
    8749         359 :     X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
    8750         702 :     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8751         702 :     Mask = Mask.zext(VT.getSizeInBits());
    8752             :     SDLoc DL(N);
    8753         351 :     return DAG.getNode(ISD::AND, DL, VT,
    8754         351 :                        X, DAG.getConstant(Mask, DL, VT));
    8755             :   }
    8756             : 
    8757             :   // Try to simplify (zext (load x)).
    8758      105418 :   if (SDValue foldedExt =
    8759      105418 :           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
    8760      105418 :                              ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
    8761        7130 :     return foldedExt;
    8762             : 
    8763             :   // fold (zext (load x)) to multiple smaller zextloads.
    8764             :   // Only on illegal but splittable vectors.
    8765       98288 :   if (SDValue ExtLoad = CombineExtLoad(N))
    8766          65 :     return ExtLoad;
    8767             : 
    8768             :   // fold (zext (and/or/xor (load x), cst)) ->
    8769             :   //      (and/or/xor (zextload x), (zext cst))
    8770             :   // Unless (and (load x) cst) will match as a zextload already and has
    8771             :   // additional users.
    8772       98223 :   if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8773       15608 :        N0.getOpcode() == ISD::XOR) &&
    8774         204 :       isa<LoadSDNode>(N0.getOperand(0)) &&
    8775       98427 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8776          83 :       (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    8777             :     LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    8778             :     EVT MemVT = LN00->getMemoryVT();
    8779          30 :     if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) &&
    8780          70 :         LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
    8781             :       bool DoXform = true;
    8782             :       SmallVector<SDNode*, 4> SetCCs;
    8783          30 :       if (!N0.hasOneUse()) {
    8784          18 :         if (N0.getOpcode() == ISD::AND) {
    8785             :           auto *AndC = cast<ConstantSDNode>(N0.getOperand(1));
    8786           0 :           EVT LoadResultTy = AndC->getValueType(0);
    8787           0 :           EVT ExtVT;
    8788           0 :           if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
    8789             :             DoXform = false;
    8790             :         }
    8791             :       }
    8792             :       if (DoXform)
    8793          60 :         DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
    8794             :                                           ISD::ZERO_EXTEND, SetCCs, TLI);
    8795          30 :       if (DoXform) {
    8796          60 :         SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
    8797             :                                          LN00->getChain(), LN00->getBasePtr(),
    8798             :                                          LN00->getMemoryVT(),
    8799          30 :                                          LN00->getMemOperand());
    8800          60 :         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8801          60 :         Mask = Mask.zext(VT.getSizeInBits());
    8802             :         SDLoc DL(N);
    8803          30 :         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
    8804          30 :                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
    8805          60 :         ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
    8806          30 :         bool NoReplaceTruncAnd = !N0.hasOneUse();
    8807             :         bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
    8808          30 :         CombineTo(N, And);
    8809             :         // If N0 has multiple uses, change other uses as well.
    8810          30 :         if (NoReplaceTruncAnd) {
    8811             :           SDValue TruncAnd =
    8812          27 :               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
    8813           9 :           CombineTo(N0.getNode(), TruncAnd);
    8814             :         }
    8815          30 :         if (NoReplaceTrunc) {
    8816          58 :           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
    8817             :         } else {
    8818           1 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
    8819           2 :                                       LN00->getValueType(0), ExtLoad);
    8820             :           CombineTo(LN00, Trunc, ExtLoad.getValue(1));
    8821             :         }
    8822          30 :         return SDValue(N,0); // Return N so it doesn't get rechecked!
    8823             :       }
    8824             :     }
    8825             :   }
    8826             : 
    8827             :   // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
    8828             :   //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
    8829       98193 :   if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
    8830          24 :     return ZExtLoad;
    8831             : 
    8832             :   // Try to simplify (zext (zextload x)).
    8833       98169 :   if (SDValue foldedExt = tryToFoldExtOfExtload(
    8834       98169 :           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
    8835          17 :     return foldedExt;
    8836             : 
    8837       98152 :   if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
    8838          22 :     return V;
    8839             : 
    8840      196260 :   if (N0.getOpcode() == ISD::SETCC) {
    8841             :     // Only do this before legalize for now.
    8842       54709 :     if (!LegalOperations && VT.isVector() &&
    8843       28309 :         N0.getValueType().getVectorElementType() == MVT::i1) {
    8844         660 :       EVT N00VT = N0.getOperand(0).getValueType();
    8845         990 :       if (getSetCCResultType(N00VT) == N0.getValueType())
    8846         137 :         return SDValue();
    8847             : 
    8848             :       // We know that the # elements of the results is the same as the #
    8849             :       // elements of the compare (and the # elements of the compare result for
    8850             :       // that matter). Check to see that they are the same size. If so, we know
    8851             :       // that the element size of the sext'd result matches the element size of
    8852             :       // the compare operands.
    8853             :       SDLoc DL(N);
    8854         193 :       SDValue VecOnes = DAG.getConstant(1, DL, VT);
    8855         193 :       if (VT.getSizeInBits() == N00VT.getSizeInBits()) {
    8856             :         // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
    8857         175 :         SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
    8858         350 :                                      N0.getOperand(1), N0.getOperand(2));
    8859         350 :         return DAG.getNode(ISD::AND, DL, VT, VSetCC, VecOnes);
    8860             :       }
    8861             : 
    8862             :       // If the desired elements are smaller or larger than the source
    8863             :       // elements we can use a matching integer vector type and then
    8864             :       // truncate/sign extend.
    8865          18 :       EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
    8866             :       SDValue VsetCC =
    8867          18 :           DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
    8868          36 :                       N0.getOperand(1), N0.getOperand(2));
    8869          18 :       return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT),
    8870          18 :                          VecOnes);
    8871             :     }
    8872             : 
    8873             :     // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    8874             :     SDLoc DL(N);
    8875       27647 :     if (SDValue SCC = SimplifySelectCC(
    8876       27647 :             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
    8877       27647 :             DAG.getConstant(0, DL, VT),
    8878       55294 :             cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
    8879         137 :       return SCC;
    8880             :   }
    8881             : 
    8882             :   // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
    8883       97663 :   if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
    8884        7162 :       isa<ConstantSDNode>(N0.getOperand(1)) &&
    8885      104836 :       N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
    8886          11 :       N0.hasOneUse()) {
    8887          10 :     SDValue ShAmt = N0.getOperand(1);
    8888          20 :     unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
    8889          10 :     if (N0.getOpcode() == ISD::SHL) {
    8890           9 :       SDValue InnerZExt = N0.getOperand(0);
    8891             :       // If the original shl may be shifting out bits, do not perform this
    8892             :       // transformation.
    8893           9 :       unsigned KnownZeroBits = InnerZExt.getValueSizeInBits() -
    8894          18 :         InnerZExt.getOperand(0).getValueSizeInBits();
    8895           9 :       if (ShAmtVal > KnownZeroBits)
    8896           0 :         return SDValue();
    8897             :     }
    8898             : 
    8899             :     SDLoc DL(N);
    8900             : 
    8901             :     // Ensure that the shift amount is wide enough for the shifted value.
    8902          10 :     if (VT.getSizeInBits() >= 256)
    8903           0 :       ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
    8904             : 
    8905          10 :     return DAG.getNode(N0.getOpcode(), DL, VT,
    8906          10 :                        DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
    8907          10 :                        ShAmt);
    8908             :   }
    8909             : 
    8910       97653 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    8911           2 :     return NewVSel;
    8912             : 
    8913       97651 :   return SDValue();
    8914             : }
    8915             : 
    8916       87023 : SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
    8917       87023 :   SDValue N0 = N->getOperand(0);
    8918       87023 :   EVT VT = N->getValueType(0);
    8919             : 
    8920       87023 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8921       87023 :                                               LegalOperations))
    8922          78 :     return SDValue(Res, 0);
    8923             : 
    8924             :   // fold (aext (aext x)) -> (aext x)
    8925             :   // fold (aext (zext x)) -> (zext x)
    8926             :   // fold (aext (sext x)) -> (sext x)
    8927       86945 :   if (N0.getOpcode() == ISD::ANY_EXTEND  ||
    8928      173873 :       N0.getOpcode() == ISD::ZERO_EXTEND ||
    8929             :       N0.getOpcode() == ISD::SIGN_EXTEND)
    8930          44 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
    8931             : 
    8932             :   // fold (aext (truncate (load x))) -> (aext (smaller load x))
    8933             :   // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
    8934       86923 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8935        7210 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8936          13 :       SDNode *oye = N0.getOperand(0).getNode();
    8937          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8938          13 :         CombineTo(N0.getNode(), NarrowLoad);
    8939             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8940          13 :         AddToWorklist(oye);
    8941             :       }
    8942          13 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8943             :     }
    8944             :   }
    8945             : 
    8946             :   // fold (aext (truncate x))
    8947       86910 :   if (N0.getOpcode() == ISD::TRUNCATE)
    8948       14483 :     return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
    8949             : 
    8950             :   // Fold (aext (and (trunc x), cst)) -> (and x, cst)
    8951             :   // if the trunc is not free.
    8952        3612 :   if (N0.getOpcode() == ISD::AND &&
    8953        3612 :       N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
    8954       85023 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8955        5202 :       !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
    8956        2601 :                           N0.getValueType())) {
    8957             :     SDLoc DL(N);
    8958           8 :     SDValue X = N0.getOperand(0).getOperand(0);
    8959           8 :     X = DAG.getAnyExtOrTrunc(X, DL, VT);
    8960           8 :     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8961           8 :     Mask = Mask.zext(VT.getSizeInBits());
    8962           8 :     return DAG.getNode(ISD::AND, DL, VT,
    8963           8 :                        X, DAG.getConstant(Mask, DL, VT));
    8964             :   }
    8965             : 
    8966             :   // fold (aext (load x)) -> (aext (truncate (extload x)))
    8967             :   // None of the supported targets knows how to perform load and any_ext
    8968             :   // on vectors in one instruction.  We only perform this transformation on
    8969             :   // scalars.
    8970       15228 :   if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
    8971       15072 :       ISD::isUNINDEXEDLoad(N0.getNode()) &&
    8972       15180 :       TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) {
    8973             :     bool DoXform = true;
    8974             :     SmallVector<SDNode*, 4> SetCCs;
    8975       14701 :     if (!N0.hasOneUse())
    8976       11013 :       DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs,
    8977             :                                         TLI);
    8978       11013 :     if (DoXform) {
    8979             :       LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8980       29018 :       SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
    8981             :                                        LN0->getChain(),
    8982             :                                        LN0->getBasePtr(), N0.getValueType(),
    8983       14594 :                                        LN0->getMemOperand());
    8984       14509 :       ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
    8985             :       // If the load value is used only by N, replace it via CombineTo N.
    8986             :       bool NoReplaceTrunc = N0.hasOneUse();
    8987       14509 :       CombineTo(N, ExtLoad);
    8988       14509 :       if (NoReplaceTrunc) {
    8989        7376 :         DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8990             :       } else {
    8991       10821 :         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
    8992       21642 :                                     N0.getValueType(), ExtLoad);
    8993             :         CombineTo(LN0, Trunc, ExtLoad.getValue(1));
    8994             :       }
    8995       14509 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8996             :     }
    8997             :   }
    8998             : 
    8999             :   // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
    9000             :   // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
    9001             :   // fold (aext ( extload x)) -> (aext (truncate (extload  x)))
    9002             :   if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) &&
    9003       65633 :       ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
    9004             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9005             :     ISD::LoadExtType ExtType = LN0->getExtensionType();
    9006         263 :     EVT MemVT = LN0->getMemoryVT();
    9007         308 :     if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) {
    9008         239 :       SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
    9009             :                                        VT, LN0->getChain(), LN0->getBasePtr(),
    9010         239 :                                        MemVT, LN0->getMemOperand());
    9011         239 :       CombineTo(N, ExtLoad);
    9012         478 :       DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    9013         239 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    9014             :     }
    9015             :   }
    9016             : 
    9017       64957 :   if (N0.getOpcode() == ISD::SETCC) {
    9018             :     // For vectors:
    9019             :     // aext(setcc) -> vsetcc
    9020             :     // aext(setcc) -> truncate(vsetcc)
    9021             :     // aext(setcc) -> aext(vsetcc)
    9022             :     // Only do this before legalize for now.
    9023        3274 :     if (VT.isVector() && !LegalOperations) {
    9024         948 :       EVT N00VT = N0.getOperand(0).getValueType();
    9025         948 :       if (getSetCCResultType(N00VT) == N0.getValueType())
    9026          89 :         return SDValue();
    9027             : 
    9028             :       // We know that the # elements of the results is the same as the
    9029             :       // # elements of the compare (and the # elements of the compare result
    9030             :       // for that matter).  Check to see that they are the same size.  If so,
    9031             :       // we know that the element size of the sext'd result matches the
    9032             :       // element size of the compare operands.
    9033         385 :       if (VT.getSizeInBits() == N00VT.getSizeInBits())
    9034         558 :         return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0),
    9035             :                              N0.getOperand(1),
    9036         279 :                              cast<CondCodeSDNode>(N0.getOperand(2))->get());
    9037             :       // If the desired elements are smaller or larger than the source
    9038             :       // elements we can use a matching integer vector type and then
    9039             :       // truncate/any extend
    9040             :       else {
    9041         106 :         EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
    9042             :         SDValue VsetCC =
    9043         106 :           DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0),
    9044             :                         N0.getOperand(1),
    9045         106 :                         cast<CondCodeSDNode>(N0.getOperand(2))->get());
    9046         212 :         return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT);
    9047             :       }
    9048             :     }
    9049             : 
    9050             :     // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    9051             :     SDLoc DL(N);
    9052        2800 :     if (SDValue SCC = SimplifySelectCC(
    9053        2800 :             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
    9054        2800 :             DAG.getConstant(0, DL, VT),
    9055        2800 :             cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
    9056         138 :       return SCC;
    9057             :   }
    9058             : 
    9059       64345 :   return SDValue();
    9060             : }
    9061             : 
    9062           0 : SDValue DAGCombiner::visitAssertExt(SDNode *N) {
    9063           0 :   unsigned Opcode = N->getOpcode();
    9064           0 :   SDValue N0 = N->getOperand(0);
    9065           0 :   SDValue N1 = N->getOperand(1);
    9066           0 :   EVT AssertVT = cast<VTSDNode>(N1)->getVT();
    9067             : 
    9068             :   // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt)
    9069           0 :   if (N0.getOpcode() == Opcode &&
    9070             :       AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT())
    9071           0 :     return N0;
    9072             : 
    9073           0 :   if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
    9074           0 :       N0.getOperand(0).getOpcode() == Opcode) {
    9075             :     // We have an assert, truncate, assert sandwich. Make one stronger assert
    9076             :     // by asserting on the smallest asserted type to the larger source type.
    9077             :     // This eliminates the later assert:
    9078             :     // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN
    9079             :     // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN
    9080           0 :     SDValue BigA = N0.getOperand(0);
    9081           0 :     EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
    9082             :     assert(BigA_AssertVT.bitsLE(N0.getValueType()) &&
    9083             :            "Asserting zero/sign-extended bits to a type larger than the "
    9084             :            "truncated destination does not provide information");
    9085             : 
    9086             :     SDLoc DL(N);
    9087           0 :     EVT MinAssertVT = AssertVT.bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
    9088           0 :     SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
    9089           0 :     SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
    9090           0 :                                     BigA.getOperand(0), MinAssertVTVal);
    9091           0 :     return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
    9092             :   }
    9093             : 
    9094           0 :   return SDValue();
    9095             : }
    9096             : 
    9097             : /// If the result of a wider load is shifted to right of N  bits and then
    9098             : /// truncated to a narrower type and where N is a multiple of number of bits of
    9099             : /// the narrower type, transform it to a narrower load from address + N / num of
    9100             : /// bits of new type. Also narrow the load if the result is masked with an AND
    9101             : /// to effectively produce a smaller type. If the result is to be extended, also
    9102             : /// fold the extension to form a extending load.
    9103     1115730 : SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
    9104     1115730 :   unsigned Opc = N->getOpcode();
    9105             : 
    9106             :   ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
    9107     1115730 :   SDValue N0 = N->getOperand(0);
    9108     1115730 :   EVT VT = N->getValueType(0);
    9109     1115730 :   EVT ExtVT = VT;
    9110             : 
    9111             :   // This transformation isn't valid for vector loads.
    9112     1115730 :   if (VT.isVector())
    9113       16073 :     return SDValue();
    9114             : 
    9115             :   unsigned ShAmt = 0;
    9116             :   bool HasShiftedOffset = false;
    9117             :   // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
    9118             :   // extended to VT.
    9119     1099657 :   if (Opc == ISD::SIGN_EXTEND_INREG) {
    9120             :     ExtType = ISD::SEXTLOAD;
    9121       40139 :     ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    9122     1059518 :   } else if (Opc == ISD::SRL) {
    9123             :     // Another special-case: SRL is basically zero-extending a narrower value,
    9124             :     // or it maybe shifting a higher subword, half or byte into the lowest
    9125             :     // bits.
    9126             :     ExtType = ISD::ZEXTLOAD;
    9127      136063 :     N0 = SDValue(N, 0);
    9128             : 
    9129             :     auto *LN0 = dyn_cast<LoadSDNode>(N0.getOperand(0));
    9130             :     auto *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    9131      136063 :     if (!N01 || !LN0)
    9132      115341 :       return SDValue();
    9133             : 
    9134       20722 :     uint64_t ShiftAmt = N01->getZExtValue();
    9135       20722 :     uint64_t MemoryWidth = LN0->getMemoryVT().getSizeInBits();
    9136       20722 :     if (LN0->getExtensionType() != ISD::SEXTLOAD && MemoryWidth > ShiftAmt)
    9137       20553 :       ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShiftAmt);
    9138             :     else
    9139         169 :       ExtVT = EVT::getIntegerVT(*DAG.getContext(),
    9140         169 :                                 VT.getSizeInBits() - ShiftAmt);
    9141      923455 :   } else if (Opc == ISD::AND) {
    9142             :     // An AND with a constant mask is the same as a truncate + zero-extend.
    9143             :     auto AndC = dyn_cast<ConstantSDNode>(N->getOperand(1));
    9144             :     if (!AndC)
    9145           0 :       return SDValue();
    9146             : 
    9147       79844 :     const APInt &Mask = AndC->getAPIntValue();
    9148             :     unsigned ActiveBits = 0;
    9149       79844 :     if (Mask.isMask()) {
    9150             :       ActiveBits = Mask.countTrailingOnes();
    9151        8811 :     } else if (Mask.isShiftedMask()) {
    9152        7763 :       ShAmt = Mask.countTrailingZeros();
    9153        7763 :       APInt ShiftedMask = Mask.lshr(ShAmt);
    9154             :       ActiveBits = ShiftedMask.countTrailingOnes();
    9155             :       HasShiftedOffset = true;
    9156             :     } else
    9157        1048 :       return SDValue();
    9158             : 
    9159             :     ExtType = ISD::ZEXTLOAD;
    9160       78796 :     ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    9161             :   }
    9162             : 
    9163     1966536 :   if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    9164       60534 :     SDValue SRL = N0;
    9165             :     if (auto *ConstShift = dyn_cast<ConstantSDNode>(SRL.getOperand(1))) {
    9166       59046 :       ShAmt = ConstShift->getZExtValue();
    9167       59046 :       unsigned EVTBits = ExtVT.getSizeInBits();
    9168             :       // Is the shift amount a multiple of size of VT?
    9169       59046 :       if ((ShAmt & (EVTBits-1)) == 0) {
    9170       50572 :         N0 = N0.getOperand(0);
    9171             :         // Is the load width a multiple of size of VT?
    9172       50572 :         if ((N0.getValueSizeInBits() & (EVTBits-1)) != 0)
    9173          21 :           return SDValue();
    9174             :       }
    9175             : 
    9176             :       // At this point, we must have a load or else we can't do the transform.
    9177       59025 :       if (!isa<LoadSDNode>(N0)) return SDValue();
    9178             : 
    9179             :       auto *LN0 = cast<LoadSDNode>(N0);
    9180             : 
    9181             :       // Because a SRL must be assumed to *need* to zero-extend the high bits
    9182             :       // (as opposed to anyext the high bits), we can't combine the zextload
    9183             :       // lowering of SRL and an sextload.
    9184       21108 :       if (LN0->getExtensionType() == ISD::SEXTLOAD)
    9185         188 :         return SDValue();
    9186             : 
    9187             :       // If the shift amount is larger than the input type then we're not
    9188             :       // accessing any of the loaded bytes.  If the load was a zextload/extload
    9189             :       // then the result of the shift+trunc is zero/undef (handled elsewhere).
    9190       20920 :       if (ShAmt >= LN0->getMemoryVT().getSizeInBits())
    9191           4 :         return SDValue();
    9192             : 
    9193             :       // If the SRL is only used by a masking AND, we may be able to adjust
    9194             :       // the ExtVT to make the AND redundant.
    9195       20916 :       SDNode *Mask = *(SRL->use_begin());
    9196       20916 :       if (Mask->getOpcode() == ISD::AND &&
    9197        6790 :           isa<ConstantSDNode>(Mask->getOperand(1))) {
    9198             :         const APInt &ShiftMask =
    9199        6783 :           cast<ConstantSDNode>(Mask->getOperand(1))->getAPIntValue();
    9200        6783 :         if (ShiftMask.isMask()) {
    9201        5202 :           EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(),
    9202        5202 :                                            ShiftMask.countTrailingOnes());
    9203             :           // If the mask is smaller, recompute the type.
    9204        8726 :           if ((ExtVT.getSizeInBits() > MaskedVT.getSizeInBits()) &&
    9205        8380 :               TLI.isLoadExtLegal(ExtType, N0.getValueType(), MaskedVT))
    9206         700 :             ExtVT = MaskedVT;
    9207             :         }
    9208             :       }
    9209             :     }
    9210             :   }
    9211             : 
    9212             :   // If the load is shifted left (and the result isn't shifted back right),
    9213             :   // we can fold the truncate through the shift.
    9214             :   unsigned ShLeftAmt = 0;
    9215     1832922 :   if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
    9216      949628 :       ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) {
    9217         381 :     if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    9218           9 :       ShLeftAmt = N01->getZExtValue();
    9219           9 :       N0 = N0.getOperand(0);
    9220             :     }
    9221             :   }
    9222             : 
    9223             :   // If we haven't found a load, we can't narrow it.
    9224      945138 :   if (!isa<LoadSDNode>(N0))
    9225      798137 :     return SDValue();
    9226             : 
    9227      147001 :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9228      147001 :   if (!isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
    9229      143897 :     return SDValue();
    9230             : 
    9231             :   auto AdjustBigEndianShift = [&](unsigned ShAmt) {
    9232             :     unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
    9233             :     unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
    9234             :     return LVTStoreBits - EVTStoreBits - ShAmt;
    9235             :   };
    9236             : 
    9237             :   // For big endian targets, we need to adjust the offset to the pointer to
    9238             :   // load the correct bytes.
    9239        3104 :   if (DAG.getDataLayout().isBigEndian())
    9240         167 :     ShAmt = AdjustBigEndianShift(ShAmt);
    9241             : 
    9242        6208 :   EVT PtrType = N0.getOperand(1).getValueType();
    9243        3104 :   uint64_t PtrOff = ShAmt / 8;
    9244        3104 :   unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
    9245        3104 :   SDLoc DL(LN0);
    9246             :   // The original load itself didn't wrap, so an offset within it doesn't.
    9247             :   SDNodeFlags Flags;
    9248             :   Flags.setNoUnsignedWrap(true);
    9249        3104 :   SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
    9250        3104 :                                PtrType, LN0->getBasePtr(),
    9251             :                                DAG.getConstant(PtrOff, DL, PtrType),
    9252        3104 :                                Flags);
    9253        3104 :   AddToWorklist(NewPtr.getNode());
    9254             : 
    9255             :   SDValue Load;
    9256        3104 :   if (ExtType == ISD::NON_EXTLOAD)
    9257        4994 :     Load = DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr,
    9258             :                        LN0->getPointerInfo().getWithOffset(PtrOff), NewAlign,
    9259       12485 :                        LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    9260             :   else
    9261        1214 :     Load = DAG.getExtLoad(ExtType, SDLoc(N0), VT, LN0->getChain(), NewPtr,
    9262             :                           LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
    9263         607 :                           NewAlign, LN0->getMemOperand()->getFlags(),
    9264        3035 :                           LN0->getAAInfo());
    9265             : 
    9266             :   // Replace the old load's chain with the new load's chain.
    9267             :   WorklistRemover DeadNodes(*this);
    9268        3104 :   DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
    9269             : 
    9270             :   // Shift the result left, if we've swallowed a left shift.
    9271        3104 :   SDValue Result = Load;
    9272        3104 :   if (ShLeftAmt != 0) {
    9273           9 :     EVT ShImmTy = getShiftAmountTy(Result.getValueType());
    9274           9 :     if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt))
    9275           0 :       ShImmTy = VT;
    9276             :     // If the shift amount is as large as the result size (but, presumably,
    9277             :     // no larger than the source) then the useful bits of the result are
    9278             :     // zero; we can't simply return the shortened shift, because the result
    9279             :     // of that operation is undefined.
    9280             :     SDLoc DL(N0);
    9281           9 :     if (ShLeftAmt >= VT.getSizeInBits())
    9282           7 :       Result = DAG.getConstant(0, DL, VT);
    9283             :     else
    9284           2 :       Result = DAG.getNode(ISD::SHL, DL, VT,
    9285           2 :                           Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
    9286             :   }
    9287             : 
    9288        3104 :   if (HasShiftedOffset) {
    9289             :     // Recalculate the shift amount after it has been altered to calculate
    9290             :     // the offset.
    9291          79 :     if (DAG.getDataLayout().isBigEndian())
    9292          16 :       ShAmt = AdjustBigEndianShift(ShAmt);
    9293             : 
    9294             :     // We're using a shifted mask, so the load now has an offset. This means we
    9295             :     // now need to shift right the mask to match the new load and then shift
    9296             :     // right the result of the AND.
    9297         158 :     const APInt &Mask = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue();
    9298          79 :     APInt ShiftedMask = Mask.lshr(ShAmt);
    9299          79 :     DAG.UpdateNodeOperands(N, Result, DAG.getConstant(ShiftedMask, DL, VT));
    9300          79 :     SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
    9301          79 :     SDValue Shifted = DAG.getNode(ISD::SHL, DL, VT, SDValue(N, 0),
    9302          79 :                                   ShiftC);
    9303         158 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Shifted);
    9304         158 :     DAG.UpdateNodeOperands(Shifted.getNode(), SDValue(N, 0), ShiftC);
    9305             :   }
    9306             :   // Return the new loaded value.
    9307        3104 :   return Result;
    9308             : }
    9309             : 
    9310       45301 : SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
    9311       45301 :   SDValue N0 = N->getOperand(0);
    9312       45301 :   SDValue N1 = N->getOperand(1);
    9313       45301 :   EVT VT = N->getValueType(0);
    9314       45301 :   EVT EVT = cast<VTSDNode>(N1)->getVT();
    9315             :   unsigned VTBits = VT.getScalarSizeInBits();
    9316             :   unsigned EVTBits = EVT.getScalarSizeInBits();
    9317             : 
    9318       45301 :   if (N0.isUndef())
    9319           1 :     return DAG.getUNDEF(VT);
    9320             : 
    9321             :   // fold (sext_in_reg c1) -> c1
    9322       45300 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    9323          24 :     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
    9324             : 
    9325             :   // If the input is already sign extended, just drop the extension.
    9326       45288 :   if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
    9327        1473 :     return N0;
    9328             : 
    9329             :   // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
    9330       43815 :   if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
    9331           4 :       EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
    9332           4 :     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    9333           8 :                        N0.getOperand(0), N1);
    9334             : 
    9335             :   // fold (sext_in_reg (sext x)) -> (sext x)
    9336             :   // fold (sext_in_reg (aext x)) -> (sext x)
    9337             :   // if x is small enough.
    9338       43811 :   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
    9339        9219 :     SDValue N00 = N0.getOperand(0);
    9340        9219 :     if (N00.getScalarValueSizeInBits() <= EVTBits &&
    9341           4 :         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
    9342           4 :       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
    9343             :   }
    9344             : 
    9345             :   // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
    9346       43795 :   if ((N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
    9347       43795 :        N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
    9348       43823 :        N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
    9349          14 :       N0.getOperand(0).getScalarValueSizeInBits() == EVTBits) {
    9350           0 :     if (!LegalOperations ||
    9351           0 :         TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))
    9352           0 :       return DAG.getSignExtendVectorInReg(N0.getOperand(0), SDLoc(N), VT);
    9353             :   }
    9354             : 
    9355             :   // fold (sext_in_reg (zext x)) -> (sext x)
    9356             :   // iff we are extending the source sign bit.
    9357       43809 :   if (N0.getOpcode() == ISD::ZERO_EXTEND) {
    9358           6 :     SDValue N00 = N0.getOperand(0);
    9359           6 :     if (N00.getScalarValueSizeInBits() == EVTBits &&
    9360           4 :         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
    9361           8 :       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
    9362             :   }
    9363             : 
    9364             :   // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
    9365       87610 :   if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, EVTBits - 1)))
    9366           6 :     return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT.getScalarType());
    9367             : 
    9368             :   // fold operands of sext_in_reg based on knowledge that the top bits are not
    9369             :   // demanded.
    9370       43803 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    9371        2112 :     return SDValue(N, 0);
    9372             : 
    9373             :   // fold (sext_in_reg (load x)) -> (smaller sextload x)
    9374             :   // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
    9375       41691 :   if (SDValue NarrowLoad = ReduceLoadWidth(N))
    9376         206 :     return NarrowLoad;
    9377             : 
    9378             :   // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
    9379             :   // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
    9380             :   // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
    9381       41485 :   if (N0.getOpcode() == ISD::SRL) {
    9382             :     if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
    9383       19790 :       if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
    9384             :         // We can turn this into an SRA iff the input to the SRL is already sign
    9385             :         // extended enough.
    9386       19788 :         unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
    9387       19788 :         if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
    9388        2294 :           return DAG.getNode(ISD::SRA, SDLoc(N), VT,
    9389        4588 :                              N0.getOperand(0), N0.getOperand(1));
    9390             :       }
    9391             :   }
    9392             : 
    9393             :   // fold (sext_inreg (extload x)) -> (sextload x)
    9394             :   // If sextload is not supported by target, we can only do the combine when
    9395             :   // load has one use. Doing otherwise can block folding the extload with other
    9396             :   // extends that the target does support.
    9397             :   if (ISD::isEXTLoad(N0.getNode()) &&
    9398          32 :       ISD::isUNINDEXEDLoad(N0.getNode()) &&
    9399        1690 :       EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
    9400        2392 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile() &&
    9401        1041 :         N0.hasOneUse()) ||
    9402        1059 :        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
    9403             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9404         717 :     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
    9405             :                                      LN0->getChain(),
    9406             :                                      LN0->getBasePtr(), EVT,
    9407         718 :                                      LN0->getMemOperand());
    9408         717 :     CombineTo(N, ExtLoad);
    9409             :     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    9410         717 :     AddToWorklist(ExtLoad.getNode());
    9411         717 :     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    9412             :   }
    9413             :   // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
    9414          73 :   if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
    9415           0 :       N0.hasOneUse() &&
    9416           0 :       EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
    9417           0 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
    9418           0 :        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
    9419             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9420           0 :     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
    9421             :                                      LN0->getChain(),
    9422             :                                      LN0->getBasePtr(), EVT,
    9423           0 :                                      LN0->getMemOperand());
    9424           0 :     CombineTo(N, ExtLoad);
    9425             :     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    9426           0 :     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    9427             :   }
    9428             : 
    9429             :   // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
    9430       38474 :   if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
    9431          58 :     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
    9432          58 :                                            N0.getOperand(1), false))
    9433           8 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    9434          16 :                          BSwap, N1);
    9435             :   }
    9436             : 
    9437       38466 :   return SDValue();
    9438             : }
    9439             : 
    9440        2920 : SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
    9441        2920 :   SDValue N0 = N->getOperand(0);
    9442        5840 :   EVT VT = N->getValueType(0);
    9443             : 
    9444        2920 :   if (N0.isUndef())
    9445           0 :     return DAG.getUNDEF(VT);
    9446             : 
    9447        2920 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    9448        2920 :                                               LegalOperations))
    9449          28 :     return SDValue(Res, 0);
    9450             : 
    9451        2892 :   return SDValue();
    9452             : }
    9453             : 
    9454        5771 : SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
    9455        5771 :   SDValue N0 = N->getOperand(0);
    9456       11542 :   EVT VT = N->getValueType(0);
    9457             : 
    9458        5771 :   if (N0.isUndef())
    9459           0 :     return DAG.getUNDEF(VT);
    9460             : 
    9461        5771 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    9462        5771 :                                               LegalOperations))
    9463          31 :     return SDValue(Res, 0);
    9464             : 
    9465        5740 :   return SDValue();
    9466             : }
    9467             : 
    9468      862466 : SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
    9469      862466 :   SDValue N0 = N->getOperand(0);
    9470      862466 :   EVT VT = N->getValueType(0);
    9471      862466 :   bool isLE = DAG.getDataLayout().isLittleEndian();
    9472             : 
    9473             :   // noop truncate
    9474     1725374 :   if (N0.getValueType() == N->getValueType(0))
    9475           0 :     return N0;
    9476             : 
    9477             :   // fold (truncate (truncate x)) -> (truncate x)
    9478      862466 :   if (N0.getOpcode() == ISD::TRUNCATE)
    9479        4581 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
    9480             : 
    9481             :   // fold (truncate c1) -> c1
    9482      860186 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
    9483        3355 :     SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
    9484        3355 :     if (C.getNode() != N)
    9485        3332 :       return C;
    9486             :   }
    9487             : 
    9488             :   // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
    9489      856854 :   if (N0.getOpcode() == ISD::ZERO_EXTEND ||
    9490     1711149 :       N0.getOpcode() == ISD::SIGN_EXTEND ||
    9491             :       N0.getOpcode() == ISD::ANY_EXTEND) {
    9492             :     // if the source is smaller than the dest, we still need an extend.
    9493       11380 :     if (N0.getOperand(0).getValueType().bitsLT(VT))
    9494        2422 :       return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
    9495             :     // if the source is larger than the dest, than we just need the truncate.
    9496        4479 :     if (N0.getOperand(0).getValueType().bitsGT(VT))
    9497        2259 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
    9498             :     // if the source and dest are the same type, we can drop both the extend
    9499             :     // and the truncate.
    9500        3393 :     return N0.getOperand(0);
    9501             :   }
    9502             : 
    9503             :   // If this is anyext(trunc), don't fold it, allow ourselves to be folded.
    9504      835987 :   if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND))
    9505        1629 :     return SDValue();
    9506             : 
    9507             :   // Fold extract-and-trunc into a narrow extract. For example:
    9508             :   //   i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
    9509             :   //   i32 y = TRUNCATE(i64 x)
    9510             :   //        -- becomes --
    9511             :   //   v16i8 b = BITCAST (v2i64 val)
    9512             :   //   i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
    9513             :   //
    9514             :   // Note: We only run this optimization after type legalization (which often
    9515             :   // creates this pattern) and before operation legalization after which
    9516             :   // we need to be more careful about the vector instructions that we generate.
    9517       17118 :   if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
    9518      856687 :       LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) {
    9519        7152 :     EVT VecTy = N0.getOperand(0).getValueType();
    9520        7152 :     EVT ExTy = N0.getValueType();
    9521       14304 :     EVT TrTy = N->getValueType(0);
    9522             : 
    9523             :     unsigned NumElem = VecTy.getVectorNumElements();
    9524        7152 :     unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits();
    9525             : 
    9526        7152 :     EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
    9527             :     assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
    9528             : 
    9529        7152 :     SDValue EltNo = N0->getOperand(1);
    9530        7152 :     if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
    9531        7082 :       int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
    9532        7082 :       EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
    9533        7082 :       int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
    9534             : 
    9535             :       SDLoc DL(N);
    9536        7082 :       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
    9537        7082 :                          DAG.getBitcast(NVT, N0.getOperand(0)),
    9538        7082 :                          DAG.getConstant(Index, DL, IndexTy));
    9539             :     }
    9540             :   }
    9541             : 
    9542             :   // trunc (select c, a, b) -> select c, (trunc a), (trunc b)
    9543     1684906 :   if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) {
    9544         124 :     EVT SrcVT = N0.getValueType();
    9545         248 :     if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) &&
    9546         124 :         TLI.isTruncateFree(SrcVT, VT)) {
    9547             :       SDLoc SL(N0);
    9548          62 :       SDValue Cond = N0.getOperand(0);
    9549         124 :       SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
    9550         186 :       SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
    9551         126 :       return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1);
    9552             :     }
    9553             :   }
    9554             : 
    9555             :   // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits()
    9556      842391 :   if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
    9557      844879 :       (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) &&
    9558        1244 :       TLI.isTypeDesirableForOp(ISD::SHL, VT)) {
    9559        1850 :     SDValue Amt = N0.getOperand(1);
    9560         422 :     KnownBits Known;
    9561         925 :     DAG.computeKnownBits(Amt, Known);
    9562             :     unsigned Size = VT.getScalarSizeInBits();
    9563        1850 :     if (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size)) {
    9564             :       SDLoc SL(N);
    9565         503 :       EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    9566             : 
    9567        1509 :       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
    9568         503 :       if (AmtVT != Amt.getValueType()) {
    9569           7 :         Amt = DAG.getZExtOrTrunc(Amt, SL, AmtVT);
    9570           7 :         AddToWorklist(Amt.getNode());
    9571             :       }
    9572        1006 :       return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt);
    9573             :     }
    9574             :   }
    9575             : 
    9576             :   // Fold a series of buildvector, bitcast, and truncate if possible.
    9577             :   // For example fold
    9578             :   //   (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to
    9579             :   //   (2xi32 (buildvector x, y)).
    9580       12843 :   if (Level == AfterLegalizeVectorOps && VT.isVector() &&
    9581        1214 :       N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
    9582      841961 :       N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
    9583           1 :       N0.getOperand(0).hasOneUse()) {
    9584           2 :     SDValue BuildVect = N0.getOperand(0);
    9585           1 :     EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType();
    9586           1 :     EVT TruncVecEltTy = VT.getVectorElementType();
    9587             : 
    9588             :     // Check that the element types match.
    9589           1 :     if (BuildVectEltTy == TruncVecEltTy) {
    9590             :       // Now we only need to compute the offset of the truncated elements.
    9591             :       unsigned BuildVecNumElts =  BuildVect.getNumOperands();
    9592             :       unsigned TruncVecNumElts = VT.getVectorNumElements();
    9593           1 :       unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
    9594             : 
    9595             :       assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
    9596             :              "Invalid number of elements");
    9597             : 
    9598             :       SmallVector<SDValue, 8> Opnds;
    9599           3 :       for (unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset)
    9600           2 :         Opnds.push_back(BuildVect.getOperand(i));
    9601             : 
    9602           2 :       return DAG.getBuildVector(VT, SDLoc(N), Opnds);
    9603             :     }
    9604             :   }
    9605             : 
    9606             :   // See if we can simplify the input to this truncate through knowledge that
    9607             :   // only the low bits are being used.
    9608             :   // For example "trunc (or (shl x, 8), y)" // -> trunc y
    9609             :   // Currently we only perform this optimization on scalars because vectors
    9610             :   // may have different active low bits.
    9611      841887 :   if (!VT.isVector()) {
    9612             :     APInt Mask =
    9613      831802 :         APInt::getLowBitsSet(N0.getValueSizeInBits(), VT.getSizeInBits());
    9614      831802 :     if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
    9615        3147 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter);
    9616             :   }
    9617             : 
    9618             :   // fold (truncate (load x)) -> (smaller load x)
    9619             :   // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
    9620      840321 :   if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
    9621      821552 :     if (SDValue Reduced = ReduceLoadWidth(N))
    9622        2432 :       return Reduced;
    9623             : 
    9624             :     // Handle the case where the load remains an extending load even
    9625             :     // after truncation.
    9626      819120 :     if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
    9627             :       LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9628       12527 :       if (!LN0->isVolatile() &&
    9629       24942 :           LN0->getMemoryVT().getStoreSizeInBits() < VT.getSizeInBits()) {
    9630         548 :         SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
    9631             :                                          VT, LN0->getChain(), LN0->getBasePtr(),
    9632             :                                          LN0->getMemoryVT(),
    9633         274 :                                          LN0->getMemOperand());
    9634         274 :         DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
    9635         274 :         return NewLoad;
    9636             :       }
    9637             :     }
    9638             :   }
    9639             : 
    9640             :   // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
    9641             :   // where ... are all 'undef'.
    9642     1675230 :   if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
    9643             :     SmallVector<EVT, 8> VTs;
    9644             :     SDValue V;
    9645             :     unsigned Idx = 0;
    9646             :     unsigned NumDefs = 0;
    9647             : 
    9648         398 :     for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
    9649         776 :       SDValue X = N0.getOperand(i);
    9650         388 :       if (!X.isUndef()) {
    9651         362 :         V = X;
    9652             :         Idx = i;
    9653         362 :         NumDefs++;
    9654             :       }
    9655             :       // Stop if more than one members are non-undef.
    9656         388 :       if (NumDefs > 1)
    9657             :         break;
    9658         212 :       VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
    9659             :                                      VT.getVectorElementType(),
    9660         636 :                                      X.getValueType().getVectorNumElements()));
    9661             :     }
    9662             : 
    9663         186 :     if (NumDefs == 0)
    9664           0 :       return DAG.getUNDEF(VT);
    9665             : 
    9666         186 :     if (NumDefs == 1) {
    9667             :       assert(V.getNode() && "The single defined operand is empty!");
    9668             :       SmallVector<SDValue, 8> Opnds;
    9669          46 :       for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
    9670          36 :         if (i != Idx) {
    9671          52 :           Opnds.push_back(DAG.getUNDEF(VTs[i]));
    9672          26 :           continue;
    9673             :         }
    9674          20 :         SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
    9675          10 :         AddToWorklist(NV.getNode());
    9676          10 :         Opnds.push_back(NV);
    9677             :       }
    9678          20 :       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds);
    9679             :     }
    9680             :   }
    9681             : 
    9682             :   // Fold truncate of a bitcast of a vector to an extract of the low vector
    9683             :   // element.
    9684             :   //
    9685             :   // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx
    9686     1692231 :   if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) {
    9687       16783 :     SDValue VecSrc = N0.getOperand(0);
    9688       16783 :     EVT SrcVT = VecSrc.getValueType();
    9689       28431 :     if (SrcVT.isVector() && SrcVT.getScalarType() == VT &&
    9690       11648 :         (!LegalOperations ||
    9691       10743 :          TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, SrcVT))) {
    9692             :       SDLoc SL(N);
    9693             : 
    9694       10453 :       EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
    9695       10456 :       unsigned Idx = isLE ? 0 : SrcVT.getVectorNumElements() - 1;
    9696       10453 :       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT,
    9697       10453 :                          VecSrc, DAG.getConstant(Idx, SL, IdxVT));
    9698             :     }
    9699             :   }
    9700             : 
    9701             :   // Simplify the operands using demanded-bits information.
    9702     1644230 :   if (!VT.isVector() &&
    9703      817078 :       SimplifyDemandedBits(SDValue(N, 0)))
    9704       10449 :     return SDValue(N, 0);
    9705             : 
    9706             :   // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
    9707             :   // (trunc addcarry(X, Y, Carry)) -> (addcarry trunc(X), trunc(Y), Carry)
    9708             :   // When the adde's carry is not used.
    9709      816726 :   if ((N0.getOpcode() == ISD::ADDE || N0.getOpcode() == ISD::ADDCARRY) &&
    9710      816747 :       N0.hasOneUse() && !N0.getNode()->hasAnyUseOfValue(1) &&
    9711          21 :       (!LegalOperations || TLI.isOperationLegal(N0.getOpcode(), VT))) {
    9712             :     SDLoc SL(N);
    9713          60 :     auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
    9714          60 :     auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
    9715          40 :     auto VTs = DAG.getVTList(VT, N0->getValueType(1));
    9716          40 :     return DAG.getNode(N0.getOpcode(), SL, VTs, X, Y, N0.getOperand(2));
    9717             :   }
    9718             : 
    9719             :   // fold (truncate (extract_subvector(ext x))) ->
    9720             :   //      (extract_subvector x)
    9721             :   // TODO: This can be generalized to cover cases where the truncate and extract
    9722             :   // do not fully cancel each other out.
    9723      816683 :   if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
    9724         153 :     SDValue N00 = N0.getOperand(0);
    9725          25 :     if (N00.getOpcode() == ISD::SIGN_EXTEND ||
    9726         170 :         N00.getOpcode() == ISD::ZERO_EXTEND ||
    9727             :         N00.getOpcode() == ISD::ANY_EXTEND) {
    9728         272 :       if (N00.getOperand(0)->getValueType(0).getVectorElementType() ==
    9729             :           VT.getVectorElementType())
    9730         136 :         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
    9731         272 :                            N00.getOperand(0), N0.getOperand(1));
    9732             :     }
    9733             :   }
    9734             : 
    9735      816547 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    9736           4 :     return NewVSel;
    9737             : 
    9738      816543 :   return SDValue();
    9739             : }
    9740             : 
    9741             : static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
    9742       57163 :   SDValue Elt = N->getOperand(i);
    9743      114326 :   if (Elt.getOpcode() != ISD::MERGE_VALUES)
    9744             :     return Elt.getNode();
    9745         473 :   return Elt.getOperand(Elt.getResNo()).getNode();
    9746             : }
    9747             : 
    9748             : /// build_pair (load, load) -> load
    9749             : /// if load locations are consecutive.
    9750       57163 : SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
    9751             :   assert(N->getOpcode() == ISD::BUILD_PAIR);
    9752             : 
    9753             :   LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
    9754             :   LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
    9755             : 
    9756             :   // A BUILD_PAIR is always having the least significant part in elt 0 and the
    9757             :   // most significant part in elt 1. So when combining into one large load, we
    9758             :   // need to consider the endianness.
    9759       57163 :   if (DAG.getDataLayout().isBigEndian())
    9760             :     std::swap(LD1, LD2);
    9761             : 
    9762       59737 :   if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
    9763             :       LD1->getAddressSpace() != LD2->getAddressSpace())
    9764       54589 :     return SDValue();
    9765        5148 :   EVT LD1VT = LD1->getValueType(0);
    9766             :   unsigned LD1Bytes = LD1VT.getStoreSize();
    9767        2574 :   if (ISD::isNON_EXTLoad(LD2) && LD2->hasOneUse() &&
    9768        2574 :       DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) {
    9769        2436 :     unsigned Align = LD1->getAlignment();
    9770        4872 :     unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
    9771        2436 :         VT.getTypeForEVT(*DAG.getContext()));
    9772             : 
    9773        2436 :     if (NewAlign <= Align &&
    9774        2209 :         (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
    9775        2209 :       return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
    9776        4418 :                          LD1->getPointerInfo(), Align);
    9777             :   }
    9778             : 
    9779         365 :   return SDValue();
    9780             : }
    9781             : 
    9782             : static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
    9783             :   // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi
    9784             :   // and Lo parts; on big-endian machines it doesn't.
    9785          10 :   return DAG.getDataLayout().isBigEndian() ? 1 : 0;
    9786             : }
    9787             : 
    9788      679186 : static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
    9789             :                                     const TargetLowering &TLI) {
    9790             :   // If this is not a bitcast to an FP type or if the target doesn't have
    9791             :   // IEEE754-compliant FP logic, we're done.
    9792      679186 :   EVT VT = N->getValueType(0);
    9793      679186 :   if (!VT.isFloatingPoint() || !TLI.hasBitPreservingFPLogic(VT))
    9794      626245 :     return SDValue();
    9795             : 
    9796             :   // TODO: Handle cases where the integer constant is a different scalar
    9797             :   // bitwidth to the FP.
    9798       52941 :   SDValue N0 = N->getOperand(0);
    9799       52941 :   EVT SourceVT = N0.getValueType();
    9800       52941 :   if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits())
    9801       19038 :     return SDValue();
    9802             : 
    9803             :   unsigned FPOpcode;
    9804             :   APInt SignMask;
    9805       33903 :   switch (N0.getOpcode()) {
    9806        1842 :   case ISD::AND:
    9807             :     FPOpcode = ISD::FABS;
    9808        1842 :     SignMask = ~APInt::getSignMask(SourceVT.getScalarSizeInBits());
    9809        1842 :     break;
    9810         588 :   case ISD::XOR:
    9811             :     FPOpcode = ISD::FNEG;
    9812         588 :     SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
    9813         588 :     break;
    9814             :   // TODO: ISD::OR --> ISD::FNABS?
    9815       31473 :   default:
    9816       31473 :     return SDValue();
    9817             :   }
    9818             : 
    9819             :   // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X
    9820             :   // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X
    9821        2430 :   SDValue LogicOp0 = N0.getOperand(0);
    9822        2430 :   ConstantSDNode *LogicOp1 = isConstOrConstSplat(N0.getOperand(1));
    9823         298 :   if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask &&
    9824        2430 :       LogicOp0.getOpcode() == ISD::BITCAST &&
    9825          52 :       LogicOp0.getOperand(0).getValueType() == VT)
    9826          52 :     return DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0));
    9827             : 
    9828        2404 :   return SDValue();
    9829             : }
    9830             : 
    9831      817480 : SDValue DAGCombiner::visitBITCAST(SDNode *N) {
    9832      817480 :   SDValue N0 = N->getOperand(0);
    9833     1634960 :   EVT VT = N->getValueType(0);
    9834             : 
    9835      817480 :   if (N0.isUndef())
    9836         126 :     return DAG.getUNDEF(VT);
    9837             : 
    9838             :   // If the input is a BUILD_VECTOR with all constant elements, fold this now.
    9839             :   // Only do this before legalize, since afterward the target may be depending
    9840             :   // on the bitconvert.
    9841             :   // First check to see if this is all constant.
    9842      152058 :   if (!LegalTypes &&
    9843      908229 :       N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
    9844             :       VT.isVector()) {
    9845       90023 :     bool isSimple = cast<BuildVectorSDNode>(N0)->isConstant();
    9846             : 
    9847      180046 :     EVT DestEltVT = N->getValueType(0).getVectorElementType();
    9848             :     assert(!DestEltVT.isVector() &&
    9849             :            "Element type of vector ValueType must not be vector!");
    9850       90023 :     if (isSimple)
    9851       89081 :       return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), DestEltVT);
    9852             :   }
    9853             : 
    9854             :   // If the input is a constant, let getNode fold it.
    9855             :   // We always need to check that this is just a fp -> int or int -> conversion
    9856             :   // otherwise we will get back N which will confuse the caller into thinking
    9857             :   // we used CombineTo. This can block target combines from running. If we can't
    9858             :   // allowed legal operations, we need to ensure the resulting operation will be
    9859             :   // legal.
    9860             :   // TODO: Maybe we should check that the return value isn't N explicitly?
    9861         977 :   if ((isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() &&
    9862          67 :        (!LegalOperations || TLI.isOperationLegal(ISD::ConstantFP, VT))) ||
    9863         176 :       (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() &&
    9864          53 :        (!LegalOperations || TLI.isOperationLegal(ISD::Constant, VT))))
    9865         119 :     return DAG.getBitcast(VT, N0);
    9866             : 
    9867             :   // (conv (conv x, t1), t2) -> (conv x, t2)
    9868      728154 :   if (N0.getOpcode() == ISD::BITCAST)
    9869       83732 :     return DAG.getBitcast(VT, N0.getOperand(0));
    9870             : 
    9871             :   // fold (conv (load x)) -> (load (conv*)x)
    9872             :   // If the resultant load doesn't need a higher alignment than the original!
    9873      195509 :   if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
    9874             :       // Do not remove the cast if the types differ in endian layout.
    9875       96534 :       TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) ==
    9876       96534 :           TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) &&
    9877             :       // If the load is volatile, we only want to change the load type if the
    9878             :       // resulting load is legal. Otherwise we might increase the number of
    9879             :       // memory accesses. We don't care if the original type was legal or not
    9880             :       // as we assume software couldn't rely on the number of accesses of an
    9881             :       // illegal type.
    9882       96532 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
    9883       99864 :        TLI.isOperationLegal(ISD::LOAD, VT)) &&
    9884       32174 :       TLI.isLoadBitCastBeneficial(N0.getValueType(), VT)) {
    9885             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9886        7201 :     unsigned OrigAlign = LN0->getAlignment();
    9887             : 
    9888        7201 :     bool Fast = false;
    9889        7201 :     if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
    9890        7201 :                                LN0->getAddressSpace(), OrigAlign, &Fast) &&
    9891             :         Fast) {
    9892             :       SDValue Load =
    9893       14204 :           DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
    9894             :                       LN0->getPointerInfo(), OrigAlign,
    9895       14224 :                       LN0->getMe