LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - DAGCombiner.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 7106 8489 83.7 %
Date: 2018-10-20 13:21:21 Functions: 217 282 77.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
      11             : // both before and after the DAG is legalized.
      12             : //
      13             : // This pass is not a substitute for the LLVM IR instcombine pass. This pass is
      14             : // primarily intended to handle simplification opportunities that are implicit
      15             : // in the LLVM IR and exposed by the various codegen lowering phases.
      16             : //
      17             : //===----------------------------------------------------------------------===//
      18             : 
      19             : #include "llvm/ADT/APFloat.h"
      20             : #include "llvm/ADT/APInt.h"
      21             : #include "llvm/ADT/ArrayRef.h"
      22             : #include "llvm/ADT/DenseMap.h"
      23             : #include "llvm/ADT/None.h"
      24             : #include "llvm/ADT/Optional.h"
      25             : #include "llvm/ADT/STLExtras.h"
      26             : #include "llvm/ADT/SetVector.h"
      27             : #include "llvm/ADT/SmallBitVector.h"
      28             : #include "llvm/ADT/SmallPtrSet.h"
      29             : #include "llvm/ADT/SmallSet.h"
      30             : #include "llvm/ADT/SmallVector.h"
      31             : #include "llvm/ADT/Statistic.h"
      32             : #include "llvm/Analysis/AliasAnalysis.h"
      33             : #include "llvm/Analysis/MemoryLocation.h"
      34             : #include "llvm/CodeGen/DAGCombine.h"
      35             : #include "llvm/CodeGen/ISDOpcodes.h"
      36             : #include "llvm/CodeGen/MachineFrameInfo.h"
      37             : #include "llvm/CodeGen/MachineFunction.h"
      38             : #include "llvm/CodeGen/MachineMemOperand.h"
      39             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      40             : #include "llvm/CodeGen/SelectionDAG.h"
      41             : #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
      42             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      43             : #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
      44             : #include "llvm/CodeGen/TargetLowering.h"
      45             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      46             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      47             : #include "llvm/CodeGen/ValueTypes.h"
      48             : #include "llvm/IR/Attributes.h"
      49             : #include "llvm/IR/Constant.h"
      50             : #include "llvm/IR/DataLayout.h"
      51             : #include "llvm/IR/DerivedTypes.h"
      52             : #include "llvm/IR/Function.h"
      53             : #include "llvm/IR/LLVMContext.h"
      54             : #include "llvm/IR/Metadata.h"
      55             : #include "llvm/Support/Casting.h"
      56             : #include "llvm/Support/CodeGen.h"
      57             : #include "llvm/Support/CommandLine.h"
      58             : #include "llvm/Support/Compiler.h"
      59             : #include "llvm/Support/Debug.h"
      60             : #include "llvm/Support/ErrorHandling.h"
      61             : #include "llvm/Support/KnownBits.h"
      62             : #include "llvm/Support/MachineValueType.h"
      63             : #include "llvm/Support/MathExtras.h"
      64             : #include "llvm/Support/raw_ostream.h"
      65             : #include "llvm/Target/TargetMachine.h"
      66             : #include "llvm/Target/TargetOptions.h"
      67             : #include <algorithm>
      68             : #include <cassert>
      69             : #include <cstdint>
      70             : #include <functional>
      71             : #include <iterator>
      72             : #include <string>
      73             : #include <tuple>
      74             : #include <utility>
      75             : 
      76             : using namespace llvm;
      77             : 
      78             : #define DEBUG_TYPE "dagcombine"
      79             : 
      80             : STATISTIC(NodesCombined   , "Number of dag nodes combined");
      81             : STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
      82             : STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
      83             : STATISTIC(OpsNarrowed     , "Number of load/op/store narrowed");
      84             : STATISTIC(LdStFP2Int      , "Number of fp load/store pairs transformed to int");
      85             : STATISTIC(SlicedLoads, "Number of load sliced");
      86             : STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops");
      87             : 
      88             : static cl::opt<bool>
      89             : CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
      90             :                  cl::desc("Enable DAG combiner's use of IR alias analysis"));
      91             : 
      92             : static cl::opt<bool>
      93             : UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
      94             :         cl::desc("Enable DAG combiner's use of TBAA"));
      95             : 
      96             : #ifndef NDEBUG
      97             : static cl::opt<std::string>
      98             : CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
      99             :                    cl::desc("Only use DAG-combiner alias analysis in this"
     100             :                             " function"));
     101             : #endif
     102             : 
     103             : /// Hidden option to stress test load slicing, i.e., when this option
     104             : /// is enabled, load slicing bypasses most of its profitability guards.
     105             : static cl::opt<bool>
     106             : StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
     107             :                   cl::desc("Bypass the profitability model of load slicing"),
     108             :                   cl::init(false));
     109             : 
     110             : static cl::opt<bool>
     111             :   MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
     112             :                     cl::desc("DAG combiner may split indexing from loads"));
     113             : 
     114             : namespace {
     115             : 
     116             :   class DAGCombiner {
     117             :     SelectionDAG &DAG;
     118             :     const TargetLowering &TLI;
     119             :     CombineLevel Level;
     120             :     CodeGenOpt::Level OptLevel;
     121             :     bool LegalOperations = false;
     122             :     bool LegalTypes = false;
     123             :     bool ForCodeSize;
     124             : 
     125             :     /// Worklist of all of the nodes that need to be simplified.
     126             :     ///
     127             :     /// This must behave as a stack -- new nodes to process are pushed onto the
     128             :     /// back and when processing we pop off of the back.
     129             :     ///
     130             :     /// The worklist will not contain duplicates but may contain null entries
     131             :     /// due to nodes being deleted from the underlying DAG.
     132             :     SmallVector<SDNode *, 64> Worklist;
     133             : 
     134             :     /// Mapping from an SDNode to its position on the worklist.
     135             :     ///
     136             :     /// This is used to find and remove nodes from the worklist (by nulling
     137             :     /// them) when they are deleted from the underlying DAG. It relies on
     138             :     /// stable indices of nodes within the worklist.
     139             :     DenseMap<SDNode *, unsigned> WorklistMap;
     140             : 
     141             :     /// Set of nodes which have been combined (at least once).
     142             :     ///
     143             :     /// This is used to allow us to reliably add any operands of a DAG node
     144             :     /// which have not yet been combined to the worklist.
     145             :     SmallPtrSet<SDNode *, 32> CombinedNodes;
     146             : 
     147             :     // AA - Used for DAG load/store alias analysis.
     148             :     AliasAnalysis *AA;
     149             : 
     150             :     /// When an instruction is simplified, add all users of the instruction to
     151             :     /// the work lists because they might get more simplified now.
     152             :     void AddUsersToWorklist(SDNode *N) {
     153     9474537 :       for (SDNode *Node : N->uses())
     154     6277668 :         AddToWorklist(Node);
     155             :     }
     156             : 
     157             :     /// Call the node-specific routine that folds each particular type of node.
     158             :     SDValue visit(SDNode *N);
     159             : 
     160             :   public:
     161     2767992 :     DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
     162     5535984 :         : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
     163     5535984 :           OptLevel(OL), AA(AA) {
     164     2767992 :       ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
     165             : 
     166     2767992 :       MaximumLegalStoreInBits = 0;
     167   315551088 :       for (MVT VT : MVT::all_valuetypes())
     168   312783096 :         if (EVT(VT).isSimple() && VT != MVT::Other &&
     169   310015104 :             TLI.isTypeLegal(EVT(VT)) &&
     170    41626409 :             VT.getSizeInBits() >= MaximumLegalStoreInBits)
     171    34327350 :           MaximumLegalStoreInBits = VT.getSizeInBits();
     172     2767992 :     }
     173             : 
     174             :     /// Add to the worklist making sure its instance is at the back (next to be
     175             :     /// processed.)
     176   249948491 :     void AddToWorklist(SDNode *N) {
     177             :       assert(N->getOpcode() != ISD::DELETED_NODE &&
     178             :              "Deleted Node added to Worklist");
     179             : 
     180             :       // Skip handle nodes as they can't usefully be combined and confuse the
     181             :       // zero-use deletion strategy.
     182   249948491 :       if (N->getOpcode() == ISD::HANDLENODE)
     183             :         return;
     184             : 
     185   249922932 :       if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
     186    93035691 :         Worklist.push_back(N);
     187             :     }
     188             : 
     189             :     /// Remove all instances of N from the worklist.
     190     7859353 :     void removeFromWorklist(SDNode *N) {
     191             :       CombinedNodes.erase(N);
     192             : 
     193     7859353 :       auto It = WorklistMap.find(N);
     194     7859353 :       if (It == WorklistMap.end())
     195     5403160 :         return; // Not in the worklist.
     196             : 
     197             :       // Null out the entry rather than erasing it to avoid a linear operation.
     198     4912386 :       Worklist[It->second] = nullptr;
     199             :       WorklistMap.erase(It);
     200             :     }
     201             : 
     202             :     void deleteAndRecombine(SDNode *N);
     203             :     bool recursivelyDeleteUnusedNodes(SDNode *N);
     204             : 
     205             :     /// Replaces all uses of the results of one DAG node with new values.
     206             :     SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
     207             :                       bool AddTo = true);
     208             : 
     209             :     /// Replaces all uses of the results of one DAG node with new values.
     210             :     SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
     211      457951 :       return CombineTo(N, &Res, 1, AddTo);
     212             :     }
     213             : 
     214             :     /// Replaces all uses of the results of one DAG node with new values.
     215             :     SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
     216             :                       bool AddTo = true) {
     217      234659 :       SDValue To[] = { Res0, Res1 };
     218        7250 :       return CombineTo(N, To, 2, AddTo);
     219             :     }
     220             : 
     221             :     void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
     222             : 
     223             :   private:
     224             :     unsigned MaximumLegalStoreInBits;
     225             : 
     226             :     /// Check the specified integer node value to see if it can be simplified or
     227             :     /// if things it uses can be simplified by bit propagation.
     228             :     /// If so, return true.
     229     4835936 :     bool SimplifyDemandedBits(SDValue Op) {
     230     4835936 :       unsigned BitWidth = Op.getScalarValueSizeInBits();
     231     4835936 :       APInt Demanded = APInt::getAllOnesValue(BitWidth);
     232     4835936 :       return SimplifyDemandedBits(Op, Demanded);
     233             :     }
     234             : 
     235             :     /// Check the specified vector node value to see if it can be simplified or
     236             :     /// if things it uses can be simplified as it only uses some of the
     237             :     /// elements. If so, return true.
     238      135492 :     bool SimplifyDemandedVectorElts(SDValue Op) {
     239      270984 :       unsigned NumElts = Op.getValueType().getVectorNumElements();
     240      135492 :       APInt Demanded = APInt::getAllOnesValue(NumElts);
     241      135492 :       return SimplifyDemandedVectorElts(Op, Demanded);
     242             :     }
     243             : 
     244             :     bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
     245             :     bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded,
     246             :                                     bool AssumeSingleUse = false);
     247             : 
     248             :     bool CombineToPreIndexedLoadStore(SDNode *N);
     249             :     bool CombineToPostIndexedLoadStore(SDNode *N);
     250             :     SDValue SplitIndexingFromLoad(LoadSDNode *LD);
     251             :     bool SliceUpLoad(SDNode *N);
     252             : 
     253             :     // Scalars have size 0 to distinguish from singleton vectors.
     254             :     SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
     255             :     bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
     256             :     bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
     257             : 
     258             :     /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
     259             :     ///   load.
     260             :     ///
     261             :     /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
     262             :     /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
     263             :     /// \param EltNo index of the vector element to load.
     264             :     /// \param OriginalLoad load that EVE came from to be replaced.
     265             :     /// \returns EVE on success SDValue() on failure.
     266             :     SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
     267             :         SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
     268             :     void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
     269             :     SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
     270             :     SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
     271             :     SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
     272             :     SDValue PromoteIntBinOp(SDValue Op);
     273             :     SDValue PromoteIntShiftOp(SDValue Op);
     274             :     SDValue PromoteExtend(SDValue Op);
     275             :     bool PromoteLoad(SDValue Op);
     276             : 
     277             :     /// Call the node-specific routine that knows how to fold each
     278             :     /// particular type of node. If that doesn't do anything, try the
     279             :     /// target-specific DAG combines.
     280             :     SDValue combine(SDNode *N);
     281             : 
     282             :     // Visitation implementation - Implement dag node combining for different
     283             :     // node types.  The semantics are as follows:
     284             :     // Return Value:
     285             :     //   SDValue.getNode() == 0 - No change was made
     286             :     //   SDValue.getNode() == N - N was replaced, is dead and has been handled.
     287             :     //   otherwise              - N should be replaced by the returned Operand.
     288             :     //
     289             :     SDValue visitTokenFactor(SDNode *N);
     290             :     SDValue visitMERGE_VALUES(SDNode *N);
     291             :     SDValue visitADD(SDNode *N);
     292             :     SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
     293             :     SDValue visitSUB(SDNode *N);
     294             :     SDValue visitADDC(SDNode *N);
     295             :     SDValue visitUADDO(SDNode *N);
     296             :     SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
     297             :     SDValue visitSUBC(SDNode *N);
     298             :     SDValue visitUSUBO(SDNode *N);
     299             :     SDValue visitADDE(SDNode *N);
     300             :     SDValue visitADDCARRY(SDNode *N);
     301             :     SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
     302             :     SDValue visitSUBE(SDNode *N);
     303             :     SDValue visitSUBCARRY(SDNode *N);
     304             :     SDValue visitMUL(SDNode *N);
     305             :     SDValue useDivRem(SDNode *N);
     306             :     SDValue visitSDIV(SDNode *N);
     307             :     SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
     308             :     SDValue visitUDIV(SDNode *N);
     309             :     SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
     310             :     SDValue visitREM(SDNode *N);
     311             :     SDValue visitMULHU(SDNode *N);
     312             :     SDValue visitMULHS(SDNode *N);
     313             :     SDValue visitSMUL_LOHI(SDNode *N);
     314             :     SDValue visitUMUL_LOHI(SDNode *N);
     315             :     SDValue visitSMULO(SDNode *N);
     316             :     SDValue visitUMULO(SDNode *N);
     317             :     SDValue visitIMINMAX(SDNode *N);
     318             :     SDValue visitAND(SDNode *N);
     319             :     SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
     320             :     SDValue visitOR(SDNode *N);
     321             :     SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
     322             :     SDValue visitXOR(SDNode *N);
     323             :     SDValue SimplifyVBinOp(SDNode *N);
     324             :     SDValue visitSHL(SDNode *N);
     325             :     SDValue visitSRA(SDNode *N);
     326             :     SDValue visitSRL(SDNode *N);
     327             :     SDValue visitRotate(SDNode *N);
     328             :     SDValue visitABS(SDNode *N);
     329             :     SDValue visitBSWAP(SDNode *N);
     330             :     SDValue visitBITREVERSE(SDNode *N);
     331             :     SDValue visitCTLZ(SDNode *N);
     332             :     SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
     333             :     SDValue visitCTTZ(SDNode *N);
     334             :     SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
     335             :     SDValue visitCTPOP(SDNode *N);
     336             :     SDValue visitSELECT(SDNode *N);
     337             :     SDValue visitVSELECT(SDNode *N);
     338             :     SDValue visitSELECT_CC(SDNode *N);
     339             :     SDValue visitSETCC(SDNode *N);
     340             :     SDValue visitSETCCCARRY(SDNode *N);
     341             :     SDValue visitSIGN_EXTEND(SDNode *N);
     342             :     SDValue visitZERO_EXTEND(SDNode *N);
     343             :     SDValue visitANY_EXTEND(SDNode *N);
     344             :     SDValue visitAssertExt(SDNode *N);
     345             :     SDValue visitSIGN_EXTEND_INREG(SDNode *N);
     346             :     SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
     347             :     SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
     348             :     SDValue visitTRUNCATE(SDNode *N);
     349             :     SDValue visitBITCAST(SDNode *N);
     350             :     SDValue visitBUILD_PAIR(SDNode *N);
     351             :     SDValue visitFADD(SDNode *N);
     352             :     SDValue visitFSUB(SDNode *N);
     353             :     SDValue visitFMUL(SDNode *N);
     354             :     SDValue visitFMA(SDNode *N);
     355             :     SDValue visitFDIV(SDNode *N);
     356             :     SDValue visitFREM(SDNode *N);
     357             :     SDValue visitFSQRT(SDNode *N);
     358             :     SDValue visitFCOPYSIGN(SDNode *N);
     359             :     SDValue visitFPOW(SDNode *N);
     360             :     SDValue visitSINT_TO_FP(SDNode *N);
     361             :     SDValue visitUINT_TO_FP(SDNode *N);
     362             :     SDValue visitFP_TO_SINT(SDNode *N);
     363             :     SDValue visitFP_TO_UINT(SDNode *N);
     364             :     SDValue visitFP_ROUND(SDNode *N);
     365             :     SDValue visitFP_ROUND_INREG(SDNode *N);
     366             :     SDValue visitFP_EXTEND(SDNode *N);
     367             :     SDValue visitFNEG(SDNode *N);
     368             :     SDValue visitFABS(SDNode *N);
     369             :     SDValue visitFCEIL(SDNode *N);
     370             :     SDValue visitFTRUNC(SDNode *N);
     371             :     SDValue visitFFLOOR(SDNode *N);
     372             :     SDValue visitFMINNUM(SDNode *N);
     373             :     SDValue visitFMAXNUM(SDNode *N);
     374             :     SDValue visitBRCOND(SDNode *N);
     375             :     SDValue visitBR_CC(SDNode *N);
     376             :     SDValue visitLOAD(SDNode *N);
     377             : 
     378             :     SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
     379             :     SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
     380             : 
     381             :     SDValue visitSTORE(SDNode *N);
     382             :     SDValue visitINSERT_VECTOR_ELT(SDNode *N);
     383             :     SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
     384             :     SDValue visitBUILD_VECTOR(SDNode *N);
     385             :     SDValue visitCONCAT_VECTORS(SDNode *N);
     386             :     SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
     387             :     SDValue visitVECTOR_SHUFFLE(SDNode *N);
     388             :     SDValue visitSCALAR_TO_VECTOR(SDNode *N);
     389             :     SDValue visitINSERT_SUBVECTOR(SDNode *N);
     390             :     SDValue visitMLOAD(SDNode *N);
     391             :     SDValue visitMSTORE(SDNode *N);
     392             :     SDValue visitMGATHER(SDNode *N);
     393             :     SDValue visitMSCATTER(SDNode *N);
     394             :     SDValue visitFP_TO_FP16(SDNode *N);
     395             :     SDValue visitFP16_TO_FP(SDNode *N);
     396             : 
     397             :     SDValue visitFADDForFMACombine(SDNode *N);
     398             :     SDValue visitFSUBForFMACombine(SDNode *N);
     399             :     SDValue visitFMULForFMADistributiveCombine(SDNode *N);
     400             : 
     401             :     SDValue XformToShuffleWithZero(SDNode *N);
     402             :     SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
     403             :                            SDValue N1, SDNodeFlags Flags);
     404             : 
     405             :     SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
     406             : 
     407             :     SDValue foldSelectOfConstants(SDNode *N);
     408             :     SDValue foldVSelectOfConstants(SDNode *N);
     409             :     SDValue foldBinOpIntoSelect(SDNode *BO);
     410             :     bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
     411             :     SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
     412             :     SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
     413             :     SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
     414             :                              SDValue N2, SDValue N3, ISD::CondCode CC,
     415             :                              bool NotExtCompare = false);
     416             :     SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
     417             :                                    SDValue N2, SDValue N3, ISD::CondCode CC);
     418             :     SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
     419             :                               const SDLoc &DL);
     420             :     SDValue unfoldMaskedMerge(SDNode *N);
     421             :     SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
     422             :     SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
     423             :                           const SDLoc &DL, bool foldBooleans);
     424             :     SDValue rebuildSetCC(SDValue N);
     425             : 
     426             :     bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
     427             :                            SDValue &CC) const;
     428             :     bool isOneUseSetCC(SDValue N) const;
     429             : 
     430             :     SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
     431             :                                          unsigned HiOp);
     432             :     SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
     433             :     SDValue CombineExtLoad(SDNode *N);
     434             :     SDValue CombineZExtLogicopShiftLoad(SDNode *N);
     435             :     SDValue combineRepeatedFPDivisors(SDNode *N);
     436             :     SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
     437             :     SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
     438             :     SDValue BuildSDIV(SDNode *N);
     439             :     SDValue BuildSDIVPow2(SDNode *N);
     440             :     SDValue BuildUDIV(SDNode *N);
     441             :     SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
     442             :     SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
     443             :     SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
     444             :     SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
     445             :     SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
     446             :     SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
     447             :                                 SDNodeFlags Flags, bool Reciprocal);
     448             :     SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
     449             :                                 SDNodeFlags Flags, bool Reciprocal);
     450             :     SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
     451             :                                bool DemandHighBits = true);
     452             :     SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
     453             :     SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
     454             :                               SDValue InnerPos, SDValue InnerNeg,
     455             :                               unsigned PosOpcode, unsigned NegOpcode,
     456             :                               const SDLoc &DL);
     457             :     SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
     458             :     SDValue MatchLoadCombine(SDNode *N);
     459             :     SDValue ReduceLoadWidth(SDNode *N);
     460             :     SDValue ReduceLoadOpStoreWidth(SDNode *N);
     461             :     SDValue splitMergedValStore(StoreSDNode *ST);
     462             :     SDValue TransformFPLoadStorePair(SDNode *N);
     463             :     SDValue convertBuildVecZextToZext(SDNode *N);
     464             :     SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
     465             :     SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
     466             :     SDValue reduceBuildVecToShuffle(SDNode *N);
     467             :     SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
     468             :                                   ArrayRef<int> VectorMask, SDValue VecIn1,
     469             :                                   SDValue VecIn2, unsigned LeftIdx);
     470             :     SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
     471             : 
     472             :     /// Walk up chain skipping non-aliasing memory nodes,
     473             :     /// looking for aliasing nodes and adding them to the Aliases vector.
     474             :     void GatherAllAliases(SDNode *N, SDValue OriginalChain,
     475             :                           SmallVectorImpl<SDValue> &Aliases);
     476             : 
     477             :     /// Return true if there is any possibility that the two addresses overlap.
     478             :     bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
     479             : 
     480             :     /// Walk up chain skipping non-aliasing memory nodes, looking for a better
     481             :     /// chain (aliasing node.)
     482             :     SDValue FindBetterChain(SDNode *N, SDValue Chain);
     483             : 
     484             :     /// Try to replace a store and any possibly adjacent stores on
     485             :     /// consecutive chains with better chains. Return true only if St is
     486             :     /// replaced.
     487             :     ///
     488             :     /// Notice that other chains may still be replaced even if the function
     489             :     /// returns false.
     490             :     bool findBetterNeighborChains(StoreSDNode *St);
     491             : 
     492             :     /// Holds a pointer to an LSBaseSDNode as well as information on where it
     493             :     /// is located in a sequence of memory operations connected by a chain.
     494             :     struct MemOpLink {
     495             :       // Ptr to the mem node.
     496             :       LSBaseSDNode *MemNode;
     497             : 
     498             :       // Offset from the base ptr.
     499             :       int64_t OffsetFromBase;
     500             : 
     501             :       MemOpLink(LSBaseSDNode *N, int64_t Offset)
     502     1004148 :           : MemNode(N), OffsetFromBase(Offset) {}
     503             :     };
     504             : 
     505             :     /// This is a helper function for visitMUL to check the profitability
     506             :     /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
     507             :     /// MulNode is the original multiply, AddNode is (add x, c1),
     508             :     /// and ConstNode is c2.
     509             :     bool isMulAddWithConstProfitable(SDNode *MulNode,
     510             :                                      SDValue &AddNode,
     511             :                                      SDValue &ConstNode);
     512             : 
     513             :     /// This is a helper function for visitAND and visitZERO_EXTEND.  Returns
     514             :     /// true if the (and (load x) c) pattern matches an extload.  ExtVT returns
     515             :     /// the type of the loaded value to be extended.
     516             :     bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
     517             :                           EVT LoadResultTy, EVT &ExtVT);
     518             : 
     519             :     /// Helper function to calculate whether the given Load/Store can have its
     520             :     /// width reduced to ExtVT.
     521             :     bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
     522             :                            EVT &MemVT, unsigned ShAmt = 0);
     523             : 
     524             :     /// Used by BackwardsPropagateMask to find suitable loads.
     525             :     bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
     526             :                            SmallPtrSetImpl<SDNode*> &NodesWithConsts,
     527             :                            ConstantSDNode *Mask, SDNode *&NodeToMask);
     528             :     /// Attempt to propagate a given AND node back to load leaves so that they
     529             :     /// can be combined into narrow loads.
     530             :     bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
     531             : 
     532             :     /// Helper function for MergeConsecutiveStores which merges the
     533             :     /// component store chains.
     534             :     SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
     535             :                                 unsigned NumStores);
     536             : 
     537             :     /// This is a helper function for MergeConsecutiveStores. When the
     538             :     /// source elements of the consecutive stores are all constants or
     539             :     /// all extracted vector elements, try to merge them into one
     540             :     /// larger store introducing bitcasts if necessary.  \return True
     541             :     /// if a merged store was created.
     542             :     bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
     543             :                                          EVT MemVT, unsigned NumStores,
     544             :                                          bool IsConstantSrc, bool UseVector,
     545             :                                          bool UseTrunc);
     546             : 
     547             :     /// This is a helper function for MergeConsecutiveStores. Stores
     548             :     /// that potentially may be merged with St are placed in
     549             :     /// StoreNodes. RootNode is a chain predecessor to all store
     550             :     /// candidates.
     551             :     void getStoreMergeCandidates(StoreSDNode *St,
     552             :                                  SmallVectorImpl<MemOpLink> &StoreNodes,
     553             :                                  SDNode *&Root);
     554             : 
     555             :     /// Helper function for MergeConsecutiveStores. Checks if
     556             :     /// candidate stores have indirect dependency through their
     557             :     /// operands. RootNode is the predecessor to all stores calculated
     558             :     /// by getStoreMergeCandidates and is used to prune the dependency check.
     559             :     /// \return True if safe to merge.
     560             :     bool checkMergeStoreCandidatesForDependencies(
     561             :         SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
     562             :         SDNode *RootNode);
     563             : 
     564             :     /// Merge consecutive store operations into a wide store.
     565             :     /// This optimization uses wide integers or vectors when possible.
     566             :     /// \return number of stores that were merged into a merged store (the
     567             :     /// affected nodes are stored as a prefix in \p StoreNodes).
     568             :     bool MergeConsecutiveStores(StoreSDNode *St);
     569             : 
     570             :     /// Try to transform a truncation where C is a constant:
     571             :     ///     (trunc (and X, C)) -> (and (trunc X), (trunc C))
     572             :     ///
     573             :     /// \p N needs to be a truncation and its first operand an AND. Other
     574             :     /// requirements are checked by the function (e.g. that trunc is
     575             :     /// single-use) and if missed an empty SDValue is returned.
     576             :     SDValue distributeTruncateThroughAnd(SDNode *N);
     577             : 
     578             :     /// Helper function to determine whether the target supports operation
     579             :     /// given by \p Opcode for type \p VT, that is, whether the operation
     580             :     /// is legal or custom before legalizing operations, and whether is
     581             :     /// legal (but not custom) after legalization.
     582           0 :     bool hasOperation(unsigned Opcode, EVT VT) {
     583           0 :       if (LegalOperations)
     584           0 :         return TLI.isOperationLegal(Opcode, VT);
     585           0 :       return TLI.isOperationLegalOrCustom(Opcode, VT);
     586             :     }
     587             : 
     588             :   public:
     589             :     /// Runs the dag combiner on all nodes in the work list
     590             :     void Run(CombineLevel AtLevel);
     591             : 
     592           0 :     SelectionDAG &getDAG() const { return DAG; }
     593             : 
     594             :     /// Returns a type large enough to hold any valid shift amount - before type
     595             :     /// legalization these can be huge.
     596        8588 :     EVT getShiftAmountTy(EVT LHSTy) {
     597             :       assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
     598        8588 :       return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
     599             :     }
     600             : 
     601             :     /// This method returns true if we are running before type legalization or
     602             :     /// if the specified VT is legal.
     603           0 :     bool isTypeLegal(const EVT &VT) {
     604           0 :       if (!LegalTypes) return true;
     605       15639 :       return TLI.isTypeLegal(VT);
     606             :     }
     607             : 
     608             :     /// Convenience wrapper around TargetLowering::getSetCCResultType
     609           0 :     EVT getSetCCResultType(EVT VT) const {
     610           0 :       return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
     611             :     }
     612             : 
     613             :     void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
     614             :                          SDValue OrigLoad, SDValue ExtLoad,
     615             :                          ISD::NodeType ExtType);
     616             :   };
     617             : 
     618             : /// This class is a DAGUpdateListener that removes any deleted
     619             : /// nodes from the worklist.
     620        4652 : class WorklistRemover : public SelectionDAG::DAGUpdateListener {
     621             :   DAGCombiner &DC;
     622             : 
     623             : public:
     624             :   explicit WorklistRemover(DAGCombiner &dc)
     625    89098941 :     : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
     626             : 
     627       24661 :   void NodeDeleted(SDNode *N, SDNode *E) override {
     628       24661 :     DC.removeFromWorklist(N);
     629       24661 :   }
     630             : };
     631             : 
     632             : } // end anonymous namespace
     633             : 
     634             : //===----------------------------------------------------------------------===//
     635             : //  TargetLowering::DAGCombinerInfo implementation
     636             : //===----------------------------------------------------------------------===//
     637             : 
     638       12454 : void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
     639       12454 :   ((DAGCombiner*)DC)->AddToWorklist(N);
     640       12454 : }
     641             : 
     642         864 : SDValue TargetLowering::DAGCombinerInfo::
     643             : CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
     644         864 :   return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
     645             : }
     646             : 
     647        1079 : SDValue TargetLowering::DAGCombinerInfo::
     648             : CombineTo(SDNode *N, SDValue Res, bool AddTo) {
     649        1079 :   return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
     650             : }
     651             : 
     652        5893 : SDValue TargetLowering::DAGCombinerInfo::
     653             : CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
     654        5893 :   return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
     655             : }
     656             : 
     657        2445 : void TargetLowering::DAGCombinerInfo::
     658             : CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
     659        2445 :   return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
     660             : }
     661             : 
     662             : //===----------------------------------------------------------------------===//
     663             : // Helper Functions
     664             : //===----------------------------------------------------------------------===//
     665             : 
     666     1452273 : void DAGCombiner::deleteAndRecombine(SDNode *N) {
     667     1452273 :   removeFromWorklist(N);
     668             : 
     669             :   // If the operands of this node are only used by the node, they will now be
     670             :   // dead. Make sure to re-visit them and recursively delete dead nodes.
     671     5512790 :   for (const SDValue &Op : N->ops())
     672             :     // For an operand generating multiple values, one of the values may
     673             :     // become dead allowing further simplification (e.g. split index
     674             :     // arithmetic from an indexed load).
     675     7900142 :     if (Op->hasOneUse() || Op->getNumValues() > 1)
     676     1096096 :       AddToWorklist(Op.getNode());
     677             : 
     678     1452273 :   DAG.DeleteNode(N);
     679     1452273 : }
     680             : 
     681             : /// Return 1 if we can compute the negated form of the specified expression for
     682             : /// the same cost as the expression itself, or 2 if we can compute the negated
     683             : /// form more cheaply than the expression itself.
     684      214881 : static char isNegatibleForFree(SDValue Op, bool LegalOperations,
     685             :                                const TargetLowering &TLI,
     686             :                                const TargetOptions *Options,
     687             :                                unsigned Depth = 0) {
     688             :   // fneg is removable even if it has multiple uses.
     689      214881 :   if (Op.getOpcode() == ISD::FNEG) return 2;
     690             : 
     691             :   // Don't allow anything with multiple uses unless we know it is free.
     692      213930 :   EVT VT = Op.getValueType();
     693      213930 :   const SDNodeFlags Flags = Op->getFlags();
     694      213930 :   if (!Op.hasOneUse())
     695       59743 :     if (!(Op.getOpcode() == ISD::FP_EXTEND &&
     696         957 :           TLI.isFPExtFree(VT, Op.getOperand(0).getValueType())))
     697       59418 :       return 0;
     698             : 
     699             :   // Don't recurse exponentially.
     700      154512 :   if (Depth > 6) return 0;
     701             : 
     702      151467 :   switch (Op.getOpcode()) {
     703             :   default: return false;
     704        4578 :   case ISD::ConstantFP: {
     705        4578 :     if (!LegalOperations)
     706             :       return 1;
     707             : 
     708             :     // Don't invert constant FP values after legalization unless the target says
     709             :     // the negated constant is legal.
     710         209 :     return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
     711        1800 :       TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
     712             :   }
     713        8935 :   case ISD::FADD:
     714        8935 :     if (!Options->UnsafeFPMath && !Flags.hasNoSignedZeros())
     715             :       return 0;
     716             : 
     717             :     // After operation legalization, it might not be legal to create new FSUBs.
     718        1545 :     if (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
     719           0 :       return 0;
     720             : 
     721             :     // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     722        3090 :     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
     723             :                                     Options, Depth + 1))
     724        1523 :       return V;
     725             :     // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
     726        1523 :     return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
     727        1523 :                               Depth + 1);
     728        2571 :   case ISD::FSUB:
     729             :     // We can't turn -(A-B) into B-A when we honor signed zeros.
     730        2571 :     if (!Options->NoSignedZerosFPMath &&
     731             :         !Flags.hasNoSignedZeros())
     732        1706 :       return 0;
     733             : 
     734             :     // fold (fneg (fsub A, B)) -> (fsub B, A)
     735             :     return 1;
     736             : 
     737       36030 :   case ISD::FMUL:
     738             :   case ISD::FDIV:
     739             :     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
     740       72060 :     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
     741             :                                     Options, Depth + 1))
     742       34963 :       return V;
     743             : 
     744       34963 :     return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
     745       34963 :                               Depth + 1);
     746             : 
     747        2957 :   case ISD::FP_EXTEND:
     748             :   case ISD::FP_ROUND:
     749             :   case ISD::FSIN:
     750        5914 :     return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
     751        2957 :                               Depth + 1);
     752             :   }
     753             : }
     754             : 
     755             : /// If isNegatibleForFree returns true, return the newly negated expression.
     756        1032 : static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
     757             :                                     bool LegalOperations, unsigned Depth = 0) {
     758        1032 :   const TargetOptions &Options = DAG.getTarget().Options;
     759             :   // fneg is removable even if it has multiple uses.
     760        1032 :   if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
     761             : 
     762             :   assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
     763             : 
     764         537 :   const SDNodeFlags Flags = Op.getNode()->getFlags();
     765             : 
     766         537 :   switch (Op.getOpcode()) {
     767           0 :   default: llvm_unreachable("Unknown code");
     768             :   case ISD::ConstantFP: {
     769         211 :     APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
     770         211 :     V.changeSign();
     771         422 :     return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
     772             :   }
     773          19 :   case ISD::FADD:
     774             :     assert(Options.UnsafeFPMath || Flags.hasNoSignedZeros());
     775             : 
     776             :     // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     777          38 :     if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
     778             :                            DAG.getTargetLoweringInfo(), &Options, Depth+1))
     779          19 :       return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     780             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     781             :                                               LegalOperations, Depth+1),
     782          38 :                          Op.getOperand(1), Flags);
     783             :     // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
     784           0 :     return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     785             :                        GetNegatedExpression(Op.getOperand(1), DAG,
     786             :                                             LegalOperations, Depth+1),
     787           0 :                        Op.getOperand(0), Flags);
     788          21 :   case ISD::FSUB:
     789             :     // fold (fneg (fsub 0, B)) -> B
     790             :     if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
     791          10 :       if (N0CFP->isZero())
     792           5 :         return Op.getOperand(1);
     793             : 
     794             :     // fold (fneg (fsub A, B)) -> (fsub B, A)
     795          16 :     return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
     796          16 :                        Op.getOperand(1), Op.getOperand(0), Flags);
     797             : 
     798         242 :   case ISD::FMUL:
     799             :   case ISD::FDIV:
     800             :     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
     801         484 :     if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
     802             :                            DAG.getTargetLoweringInfo(), &Options, Depth+1))
     803         115 :       return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     804             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     805             :                                               LegalOperations, Depth+1),
     806         230 :                          Op.getOperand(1), Flags);
     807             : 
     808             :     // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
     809         127 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     810             :                        Op.getOperand(0),
     811             :                        GetNegatedExpression(Op.getOperand(1), DAG,
     812         254 :                                             LegalOperations, Depth+1), Flags);
     813             : 
     814             :   case ISD::FP_EXTEND:
     815             :   case ISD::FSIN:
     816          20 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
     817             :                        GetNegatedExpression(Op.getOperand(0), DAG,
     818          80 :                                             LegalOperations, Depth+1));
     819             :   case ISD::FP_ROUND:
     820          24 :       return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
     821             :                          GetNegatedExpression(Op.getOperand(0), DAG,
     822             :                                               LegalOperations, Depth+1),
     823          96 :                          Op.getOperand(1));
     824             :   }
     825             : }
     826             : 
     827             : // APInts must be the same size for most operations, this helper
     828             : // function zero extends the shorter of the pair so that they match.
     829             : // We provide an Offset so that we can create bitwidths that won't overflow.
     830        9664 : static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
     831        9664 :   unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
     832        9664 :   LHS = LHS.zextOrSelf(Bits);
     833        9664 :   RHS = RHS.zextOrSelf(Bits);
     834        9664 : }
     835             : 
     836             : // Return true if this node is a setcc, or is a select_cc
     837             : // that selects between the target values used for true and false, making it
     838             : // equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
     839             : // the appropriate nodes based on the type of node we are checking. This
     840             : // simplifies life a bit for the callers.
     841           0 : bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
     842             :                                     SDValue &CC) const {
     843           0 :   if (N.getOpcode() == ISD::SETCC) {
     844           0 :     LHS = N.getOperand(0);
     845           0 :     RHS = N.getOperand(1);
     846           0 :     CC  = N.getOperand(2);
     847           0 :     return true;
     848             :   }
     849             : 
     850           0 :   if (N.getOpcode() != ISD::SELECT_CC ||
     851           0 :       !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
     852           0 :       !TLI.isConstFalseVal(N.getOperand(3).getNode()))
     853           0 :     return false;
     854             : 
     855           0 :   if (TLI.getBooleanContents(N.getValueType()) ==
     856             :       TargetLowering::UndefinedBooleanContent)
     857           0 :     return false;
     858             : 
     859           0 :   LHS = N.getOperand(0);
     860           0 :   RHS = N.getOperand(1);
     861           0 :   CC  = N.getOperand(4);
     862           0 :   return true;
     863             : }
     864             : 
     865             : /// Return true if this is a SetCC-equivalent operation with only one use.
     866             : /// If this is true, it allows the users to invert the operation for free when
     867             : /// it is profitable to do so.
     868           0 : bool DAGCombiner::isOneUseSetCC(SDValue N) const {
     869           0 :   SDValue N0, N1, N2;
     870           0 :   if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
     871           0 :     return true;
     872             :   return false;
     873             : }
     874             : 
     875             : // Returns the SDNode if it is a constant float BuildVector
     876             : // or constant float.
     877           0 : static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
     878             :   if (isa<ConstantFPSDNode>(N))
     879             :     return N.getNode();
     880      149586 :   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
     881           0 :     return N.getNode();
     882             :   return nullptr;
     883             : }
     884             : 
     885             : // Determines if it is a constant integer or a build vector of constant
     886             : // integers (and undefs).
     887             : // Do not permit build vector implicit truncation.
     888     3407251 : static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
     889             :   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
     890     2892631 :     return !(Const->isOpaque() && NoOpaques);
     891      514620 :   if (N.getOpcode() != ISD::BUILD_VECTOR)
     892             :     return false;
     893      104759 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     894      357702 :   for (const SDValue &Op : N->op_values()) {
     895      507308 :     if (Op.isUndef())
     896             :       continue;
     897             :     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
     898      252069 :     if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
     899      251965 :         (Const->isOpaque() && NoOpaques))
     900             :       return false;
     901             :   }
     902             :   return true;
     903             : }
     904             : 
     905             : // Determines if it is a constant null integer or a splatted vector of a
     906             : // constant null integer (with no undefs).
     907             : // Build vector implicit truncation is not an issue for null values.
     908      831862 : static bool isNullConstantOrNullSplatConstant(SDValue N) {
     909             :   // TODO: may want to use peekThroughBitcast() here.
     910      831862 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     911       27452 :     return Splat->isNullValue();
     912             :   return false;
     913             : }
     914             : 
     915             : // Determines if it is a constant integer of one or a splatted vector of a
     916             : // constant integer of one (with no undefs).
     917             : // Do not permit build vector implicit truncation.
     918        1093 : static bool isOneConstantOrOneSplatConstant(SDValue N) {
     919             :   // TODO: may want to use peekThroughBitcast() here.
     920        1093 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     921        1093 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     922        2150 :     return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
     923             :   return false;
     924             : }
     925             : 
     926             : // Determines if it is a constant integer of all ones or a splatted vector of a
     927             : // constant integer of all ones (with no undefs).
     928             : // Do not permit build vector implicit truncation.
     929      618075 : static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
     930      618075 :   N = peekThroughBitcasts(N);
     931      618075 :   unsigned BitWidth = N.getScalarValueSizeInBits();
     932      618075 :   if (ConstantSDNode *Splat = isConstOrConstSplat(N))
     933      155004 :     return Splat->isAllOnesValue() &&
     934             :            Splat->getAPIntValue().getBitWidth() == BitWidth;
     935             :   return false;
     936             : }
     937             : 
     938             : // Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
     939             : // undef's.
     940       25434 : static bool isAnyConstantBuildVector(const SDNode *N) {
     941       48272 :   return ISD::isBuildVectorOfConstantSDNodes(N) ||
     942       22838 :          ISD::isBuildVectorOfConstantFPSDNodes(N);
     943             : }
     944             : 
     945     4018534 : SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
     946             :                                     SDValue N1, SDNodeFlags Flags) {
     947             :   // Don't reassociate reductions.
     948     4018534 :   if (Flags.hasVectorReduction())
     949         468 :     return SDValue();
     950             : 
     951     4018066 :   EVT VT = N0.getValueType();
     952     4018066 :   if (N0.getOpcode() == Opc && !N0->getFlags().hasVectorReduction()) {
     953     1023756 :     if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
     954      226574 :       if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
     955             :         // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
     956      223392 :         if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
     957      446726 :           return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
     958          29 :         return SDValue();
     959             :       }
     960        3182 :       if (N0.hasOneUse()) {
     961             :         // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
     962             :         // use
     963        3301 :         SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
     964        2047 :         if (!OpNode.getNode())
     965           0 :           return SDValue();
     966        2047 :         AddToWorklist(OpNode.getNode());
     967        4094 :         return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
     968             :       }
     969             :     }
     970             :   }
     971             : 
     972     3792627 :   if (N1.getOpcode() == Opc && !N1->getFlags().hasVectorReduction()) {
     973       14972 :     if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
     974        1645 :       if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
     975             :         // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
     976           0 :         if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
     977           0 :           return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
     978           0 :         return SDValue();
     979             :       }
     980        1645 :       if (N1.hasOneUse()) {
     981             :         // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
     982             :         // use
     983        1197 :         SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
     984        1019 :         if (!OpNode.getNode())
     985           0 :           return SDValue();
     986        1019 :         AddToWorklist(OpNode.getNode());
     987        2038 :         return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
     988             :       }
     989             :     }
     990             :   }
     991             : 
     992     3791608 :   return SDValue();
     993             : }
     994             : 
     995      731867 : SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
     996             :                                bool AddTo) {
     997             :   assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
     998             :   ++NodesCombined;
     999             :   LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
    1000             :              To[0].getNode()->dump(&DAG);
    1001             :              dbgs() << " and " << NumTo - 1 << " other values\n");
    1002             :   for (unsigned i = 0, e = NumTo; i != e; ++i)
    1003             :     assert((!To[i].getNode() ||
    1004             :             N->getValueType(i) == To[i].getValueType()) &&
    1005             :            "Cannot combine value to value of different type!");
    1006             : 
    1007             :   WorklistRemover DeadNodes(*this);
    1008      731867 :   DAG.ReplaceAllUsesWith(N, To);
    1009      731867 :   if (AddTo) {
    1010             :     // Push the new nodes and any users onto the worklist
    1011      859501 :     for (unsigned i = 0, e = NumTo; i != e; ++i) {
    1012      547753 :       if (To[i].getNode()) {
    1013      547753 :         AddToWorklist(To[i].getNode());
    1014      547753 :         AddUsersToWorklist(To[i].getNode());
    1015             :       }
    1016             :     }
    1017             :   }
    1018             : 
    1019             :   // Finally, if the node is now dead, remove it from the graph.  The node
    1020             :   // may not be dead if the replacement process recursively simplified to
    1021             :   // something else needing this node.
    1022      731867 :   if (N->use_empty())
    1023      731825 :     deleteAndRecombine(N);
    1024     1463734 :   return SDValue(N, 0);
    1025             : }
    1026             : 
    1027      160144 : void DAGCombiner::
    1028             : CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
    1029             :   // Replace all uses.  If any nodes become isomorphic to other nodes and
    1030             :   // are deleted, make sure to remove them from our worklist.
    1031             :   WorklistRemover DeadNodes(*this);
    1032      160144 :   DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
    1033             : 
    1034             :   // Push the new node and any (possibly new) users onto the worklist.
    1035      160144 :   AddToWorklist(TLO.New.getNode());
    1036      160144 :   AddUsersToWorklist(TLO.New.getNode());
    1037             : 
    1038             :   // Finally, if the node is now dead, remove it from the graph.  The node
    1039             :   // may not be dead if the replacement process recursively simplified to
    1040             :   // something else needing this node.
    1041      160144 :   if (TLO.Old.getNode()->use_empty())
    1042      159243 :     deleteAndRecombine(TLO.Old.getNode());
    1043      160144 : }
    1044             : 
    1045             : /// Check the specified integer node value to see if it can be simplified or if
    1046             : /// things it uses can be simplified by bit propagation. If so, return true.
    1047     4887719 : bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
    1048     4887719 :   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
    1049     4887719 :   KnownBits Known;
    1050     4887719 :   if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
    1051             :     return false;
    1052             : 
    1053             :   // Revisit the node.
    1054      155126 :   AddToWorklist(Op.getNode());
    1055             : 
    1056             :   // Replace the old value with the new one.
    1057             :   ++NodesCombined;
    1058             :   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
    1059             :              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
    1060             :              dbgs() << '\n');
    1061             : 
    1062      155126 :   CommitTargetLoweringOpt(TLO);
    1063      155126 :   return true;
    1064             : }
    1065             : 
    1066             : /// Check the specified vector node value to see if it can be simplified or
    1067             : /// if things it uses can be simplified as it only uses some of the elements.
    1068             : /// If so, return true.
    1069      407069 : bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded,
    1070             :                                              bool AssumeSingleUse) {
    1071      407069 :   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
    1072             :   APInt KnownUndef, KnownZero;
    1073      407069 :   if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO,
    1074             :                                       0, AssumeSingleUse))
    1075             :     return false;
    1076             : 
    1077             :   // Revisit the node.
    1078        2573 :   AddToWorklist(Op.getNode());
    1079             : 
    1080             :   // Replace the old value with the new one.
    1081             :   ++NodesCombined;
    1082             :   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
    1083             :              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
    1084             :              dbgs() << '\n');
    1085             : 
    1086        2573 :   CommitTargetLoweringOpt(TLO);
    1087        2573 :   return true;
    1088             : }
    1089             : 
    1090         309 : void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
    1091             :   SDLoc DL(Load);
    1092         309 :   EVT VT = Load->getValueType(0);
    1093         618 :   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
    1094             : 
    1095             :   LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
    1096             :              Trunc.getNode()->dump(&DAG); dbgs() << '\n');
    1097             :   WorklistRemover DeadNodes(*this);
    1098         618 :   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
    1099         618 :   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
    1100         309 :   deleteAndRecombine(Load);
    1101         309 :   AddToWorklist(Trunc.getNode());
    1102         309 : }
    1103             : 
    1104        8689 : SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
    1105        8689 :   Replace = false;
    1106             :   SDLoc DL(Op);
    1107             :   if (ISD::isUNINDEXEDLoad(Op.getNode())) {
    1108             :     LoadSDNode *LD = cast<LoadSDNode>(Op);
    1109         399 :     EVT MemVT = LD->getMemoryVT();
    1110             :     ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
    1111             :                                                       : LD->getExtensionType();
    1112         399 :     Replace = true;
    1113         399 :     return DAG.getExtLoad(ExtType, DL, PVT,
    1114             :                           LD->getChain(), LD->getBasePtr(),
    1115         798 :                           MemVT, LD->getMemOperand());
    1116             :   }
    1117             : 
    1118             :   unsigned Opc = Op.getOpcode();
    1119        8290 :   switch (Opc) {
    1120             :   default: break;
    1121           0 :   case ISD::AssertSext:
    1122           0 :     if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
    1123           0 :       return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
    1124           0 :     break;
    1125          36 :   case ISD::AssertZext:
    1126          36 :     if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
    1127          72 :       return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
    1128           0 :     break;
    1129        1703 :   case ISD::Constant: {
    1130             :     unsigned ExtOpc =
    1131        3406 :       Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    1132        3406 :     return DAG.getNode(ExtOpc, DL, PVT, Op);
    1133             :   }
    1134             :   }
    1135             : 
    1136        6551 :   if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
    1137           0 :     return SDValue();
    1138       13102 :   return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
    1139             : }
    1140             : 
    1141           0 : SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
    1142           0 :   if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
    1143           0 :     return SDValue();
    1144           0 :   EVT OldVT = Op.getValueType();
    1145             :   SDLoc DL(Op);
    1146           0 :   bool Replace = false;
    1147           0 :   SDValue NewOp = PromoteOperand(Op, PVT, Replace);
    1148           0 :   if (!NewOp.getNode())
    1149           0 :     return SDValue();
    1150           0 :   AddToWorklist(NewOp.getNode());
    1151             : 
    1152           0 :   if (Replace)
    1153           0 :     ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
    1154           0 :   return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
    1155           0 :                      DAG.getValueType(OldVT));
    1156             : }
    1157             : 
    1158         532 : SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
    1159         532 :   EVT OldVT = Op.getValueType();
    1160             :   SDLoc DL(Op);
    1161         532 :   bool Replace = false;
    1162         532 :   SDValue NewOp = PromoteOperand(Op, PVT, Replace);
    1163         532 :   if (!NewOp.getNode())
    1164           0 :     return SDValue();
    1165         532 :   AddToWorklist(NewOp.getNode());
    1166             : 
    1167         532 :   if (Replace)
    1168          78 :     ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
    1169         532 :   return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
    1170             : }
    1171             : 
    1172             : /// Promote the specified integer binary operation if the target indicates it is
    1173             : /// beneficial. e.g. On x86, it's usually better to promote i16 operations to
    1174             : /// i32 since i16 instructions are longer.
    1175     3975682 : SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
    1176     3975682 :   if (!LegalOperations)
    1177     2255948 :     return SDValue();
    1178             : 
    1179     1719734 :   EVT VT = Op.getValueType();
    1180     1719734 :   if (VT.isVector() || !VT.isInteger())
    1181      182037 :     return SDValue();
    1182             : 
    1183             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1184             :   // promoting it.
    1185             :   unsigned Opc = Op.getOpcode();
    1186     1537697 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1187     1533219 :     return SDValue();
    1188             : 
    1189        4478 :   EVT PVT = VT;
    1190             :   // Consult target whether it is a good idea to promote this operation and
    1191             :   // what's the right type to promote it to.
    1192        4478 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1193             :     assert(PVT != VT && "Don't know what type to promote to!");
    1194             : 
    1195             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1196             : 
    1197        3519 :     bool Replace0 = false;
    1198        3519 :     SDValue N0 = Op.getOperand(0);
    1199        3519 :     SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
    1200             : 
    1201        3519 :     bool Replace1 = false;
    1202        3519 :     SDValue N1 = Op.getOperand(1);
    1203        3519 :     SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
    1204             :     SDLoc DL(Op);
    1205             : 
    1206             :     SDValue RV =
    1207        7038 :         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
    1208             : 
    1209             :     // We are always replacing N0/N1's use in N and only need
    1210             :     // additional replacements if there are additional uses.
    1211        3519 :     Replace0 &= !N0->hasOneUse();
    1212        3519 :     Replace1 &= (N0 != N1) && !N1->hasOneUse();
    1213             : 
    1214             :     // Combine Op here so it is preserved past replacements.
    1215        3519 :     CombineTo(Op.getNode(), RV);
    1216             : 
    1217             :     // If operands have a use ordering, make sure we deal with
    1218             :     // predecessor first.
    1219        3521 :     if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
    1220             :       std::swap(N0, N1);
    1221             :       std::swap(NN0, NN1);
    1222             :     }
    1223             : 
    1224        3519 :     if (Replace0) {
    1225          89 :       AddToWorklist(NN0.getNode());
    1226          89 :       ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
    1227             :     }
    1228        3519 :     if (Replace1) {
    1229          88 :       AddToWorklist(NN1.getNode());
    1230          88 :       ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
    1231             :     }
    1232        3519 :     return Op;
    1233             :   }
    1234         959 :   return SDValue();
    1235             : }
    1236             : 
    1237             : /// Promote the specified integer shift operation if the target indicates it is
    1238             : /// beneficial. e.g. On x86, it's usually better to promote i16 operations to
    1239             : /// i32 since i16 instructions are longer.
    1240      259809 : SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
    1241      259809 :   if (!LegalOperations)
    1242      111529 :     return SDValue();
    1243             : 
    1244      148280 :   EVT VT = Op.getValueType();
    1245      148280 :   if (VT.isVector() || !VT.isInteger())
    1246        2580 :     return SDValue();
    1247             : 
    1248             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1249             :   // promoting it.
    1250             :   unsigned Opc = Op.getOpcode();
    1251      145700 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1252      140181 :     return SDValue();
    1253             : 
    1254        5519 :   EVT PVT = VT;
    1255             :   // Consult target whether it is a good idea to promote this operation and
    1256             :   // what's the right type to promote it to.
    1257        5519 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1258             :     assert(PVT != VT && "Don't know what type to promote to!");
    1259             : 
    1260             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1261             : 
    1262        1615 :     bool Replace = false;
    1263        1615 :     SDValue N0 = Op.getOperand(0);
    1264        1615 :     SDValue N1 = Op.getOperand(1);
    1265        1615 :     if (Opc == ISD::SRA)
    1266           0 :       N0 = SExtPromoteOperand(N0, PVT);
    1267        1615 :     else if (Opc == ISD::SRL)
    1268         496 :       N0 = ZExtPromoteOperand(N0, PVT);
    1269             :     else
    1270        1119 :       N0 = PromoteOperand(N0, PVT, Replace);
    1271             : 
    1272        1615 :     if (!N0.getNode())
    1273           0 :       return SDValue();
    1274             : 
    1275             :     SDLoc DL(Op);
    1276             :     SDValue RV =
    1277        3230 :         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
    1278             : 
    1279        1615 :     AddToWorklist(N0.getNode());
    1280        1615 :     if (Replace)
    1281          54 :       ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
    1282             : 
    1283             :     // Deal with Op being deleted.
    1284        1615 :     if (Op && Op.getOpcode() != ISD::DELETED_NODE)
    1285        1615 :       return RV;
    1286             :   }
    1287        3904 :   return SDValue();
    1288             : }
    1289             : 
    1290      189609 : SDValue DAGCombiner::PromoteExtend(SDValue Op) {
    1291      189609 :   if (!LegalOperations)
    1292      117783 :     return SDValue();
    1293             : 
    1294       71826 :   EVT VT = Op.getValueType();
    1295       71826 :   if (VT.isVector() || !VT.isInteger())
    1296        1469 :     return SDValue();
    1297             : 
    1298             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1299             :   // promoting it.
    1300             :   unsigned Opc = Op.getOpcode();
    1301       70357 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1302       70090 :     return SDValue();
    1303             : 
    1304         267 :   EVT PVT = VT;
    1305             :   // Consult target whether it is a good idea to promote this operation and
    1306             :   // what's the right type to promote it to.
    1307         267 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1308             :     assert(PVT != VT && "Don't know what type to promote to!");
    1309             :     // fold (aext (aext x)) -> (aext x)
    1310             :     // fold (aext (zext x)) -> (zext x)
    1311             :     // fold (aext (sext x)) -> (sext x)
    1312             :     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
    1313         506 :     return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
    1314             :   }
    1315          14 :   return SDValue();
    1316             : }
    1317             : 
    1318     6143598 : bool DAGCombiner::PromoteLoad(SDValue Op) {
    1319     6143598 :   if (!LegalOperations)
    1320             :     return false;
    1321             : 
    1322             :   if (!ISD::isUNINDEXEDLoad(Op.getNode()))
    1323             :     return false;
    1324             : 
    1325     2619814 :   EVT VT = Op.getValueType();
    1326     2619814 :   if (VT.isVector() || !VT.isInteger())
    1327             :     return false;
    1328             : 
    1329             :   // If operation type is 'undesirable', e.g. i16 on x86, consider
    1330             :   // promoting it.
    1331             :   unsigned Opc = Op.getOpcode();
    1332     2338745 :   if (TLI.isTypeDesirableForOp(Opc, VT))
    1333             :     return false;
    1334             : 
    1335        4662 :   EVT PVT = VT;
    1336             :   // Consult target whether it is a good idea to promote this operation and
    1337             :   // what's the right type to promote it to.
    1338        4662 :   if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    1339             :     assert(PVT != VT && "Don't know what type to promote to!");
    1340             : 
    1341             :     SDLoc DL(Op);
    1342             :     SDNode *N = Op.getNode();
    1343             :     LoadSDNode *LD = cast<LoadSDNode>(N);
    1344           0 :     EVT MemVT = LD->getMemoryVT();
    1345             :     ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
    1346             :                                                       : LD->getExtensionType();
    1347           0 :     SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
    1348             :                                    LD->getChain(), LD->getBasePtr(),
    1349           0 :                                    MemVT, LD->getMemOperand());
    1350           0 :     SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
    1351             : 
    1352             :     LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
    1353             :                Result.getNode()->dump(&DAG); dbgs() << '\n');
    1354             :     WorklistRemover DeadNodes(*this);
    1355           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
    1356           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
    1357           0 :     deleteAndRecombine(N);
    1358           0 :     AddToWorklist(Result.getNode());
    1359             :     return true;
    1360             :   }
    1361             :   return false;
    1362             : }
    1363             : 
    1364             : /// Recursively delete a node which has no uses and any operands for
    1365             : /// which it is the only use.
    1366             : ///
    1367             : /// Note that this both deletes the nodes and removes them from the worklist.
    1368             : /// It also adds any nodes who have had a user deleted to the worklist as they
    1369             : /// may now have only one use and subject to other combines.
    1370    92486438 : bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
    1371    92486438 :   if (!N->use_empty())
    1372             :     return false;
    1373             : 
    1374             :   SmallSetVector<SDNode *, 16> Nodes;
    1375     3949125 :   Nodes.insert(N);
    1376             :   do {
    1377    12725499 :     N = Nodes.pop_back_val();
    1378    12725499 :     if (!N)
    1379             :       continue;
    1380             : 
    1381    12725499 :     if (N->use_empty()) {
    1382    15971072 :       for (const SDValue &ChildN : N->op_values())
    1383     9588653 :         Nodes.insert(ChildN.getNode());
    1384             : 
    1385     6382419 :       removeFromWorklist(N);
    1386     6382419 :       DAG.DeleteNode(N);
    1387             :     } else {
    1388     6343080 :       AddToWorklist(N);
    1389             :     }
    1390    12725498 :   } while (!Nodes.empty());
    1391             :   return true;
    1392             : }
    1393             : 
    1394             : //===----------------------------------------------------------------------===//
    1395             : //  Main DAG Combiner implementation
    1396             : //===----------------------------------------------------------------------===//
    1397             : 
    1398     2767992 : void DAGCombiner::Run(CombineLevel AtLevel) {
    1399             :   // set the instance variables, so that the various visit routines may use it.
    1400     2767992 :   Level = AtLevel;
    1401     2767992 :   LegalOperations = Level >= AfterLegalizeVectorOps;
    1402     2767992 :   LegalTypes = Level >= AfterLegalizeTypes;
    1403             : 
    1404             :   // Add all the dag nodes to the worklist.
    1405    85262820 :   for (SDNode &Node : DAG.allnodes())
    1406    82494828 :     AddToWorklist(&Node);
    1407             : 
    1408             :   // Create a dummy node (which is not added to allnodes), that adds a reference
    1409             :   // to the root node, preventing it from being deleted, and tracking any
    1410             :   // changes of the root.
    1411     8303976 :   HandleSDNode Dummy(DAG.getRoot());
    1412             : 
    1413             :   // While the worklist isn't empty, find a node and try to combine it.
    1414    93347489 :   while (!WorklistMap.empty()) {
    1415             :     SDNode *N;
    1416             :     // The Worklist holds the SDNodes in order, but it may contain null entries.
    1417             :     do {
    1418    93035690 :       N = Worklist.pop_back_val();
    1419    93035690 :     } while (!N);
    1420             : 
    1421    90579497 :     bool GoodWorklistEntry = WorklistMap.erase(N);
    1422             :     (void)GoodWorklistEntry;
    1423             :     assert(GoodWorklistEntry &&
    1424             :            "Found a worklist entry without a corresponding map entry!");
    1425             : 
    1426             :     // If N has no uses, it is dead.  Make sure to revisit all N's operands once
    1427             :     // N is deleted from the DAG, since they too may now be dead or may have a
    1428             :     // reduced number of uses, allowing other xforms.
    1429    90579498 :     if (recursivelyDeleteUnusedNodes(N))
    1430    88673233 :       continue;
    1431             : 
    1432             :     WorklistRemover DeadNodes(*this);
    1433             : 
    1434             :     // If this combine is running after legalizing the DAG, re-legalize any
    1435             :     // nodes pulled off the worklist.
    1436    88536543 :     if (Level == AfterLegalizeDAG) {
    1437             :       SmallSetVector<SDNode *, 16> UpdatedNodes;
    1438    39041762 :       bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
    1439             : 
    1440    39066393 :       for (SDNode *LN : UpdatedNodes) {
    1441       24631 :         AddToWorklist(LN);
    1442             :         AddUsersToWorklist(LN);
    1443             :       }
    1444    39041762 :       if (!NIsValid)
    1445             :         continue;
    1446             :     }
    1447             : 
    1448             :     LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
    1449             : 
    1450             :     // Add any operands of the new node which have not yet been combined to the
    1451             :     // worklist as well. Because the worklist uniques things already, this
    1452             :     // won't repeatedly process the same operand.
    1453    88525903 :     CombinedNodes.insert(N);
    1454   250096475 :     for (const SDValue &ChildN : N->op_values())
    1455   161570572 :       if (!CombinedNodes.count(ChildN.getNode()))
    1456   149634654 :         AddToWorklist(ChildN.getNode());
    1457             : 
    1458    88525903 :     SDValue RV = combine(N);
    1459             : 
    1460    88525902 :     if (!RV.getNode())
    1461             :       continue;
    1462             : 
    1463             :     ++NodesCombined;
    1464             : 
    1465             :     // If we get back the same node we passed in, rather than a new node or
    1466             :     // zero, we know that the node must have defined multiple values and
    1467             :     // CombineTo was used.  Since CombineTo takes care of the worklist
    1468             :     // mechanics for us, we have no work to do in this case.
    1469     3305870 :     if (RV.getNode() == N)
    1470             :       continue;
    1471             : 
    1472             :     assert(N->getOpcode() != ISD::DELETED_NODE &&
    1473             :            RV.getOpcode() != ISD::DELETED_NODE &&
    1474             :            "Node was deleted but visit returned new node!");
    1475             : 
    1476             :     LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG));
    1477             : 
    1478     1906264 :     if (N->getNumValues() == RV.getNode()->getNumValues())
    1479     1718196 :       DAG.ReplaceAllUsesWith(N, RV.getNode());
    1480             :     else {
    1481             :       assert(N->getValueType(0) == RV.getValueType() &&
    1482             :              N->getNumValues() == 1 && "Type mismatch");
    1483      188068 :       DAG.ReplaceAllUsesWith(N, &RV);
    1484             :     }
    1485             : 
    1486             :     // Push the new node and any users onto the worklist
    1487     1906264 :     AddToWorklist(RV.getNode());
    1488     1906264 :     AddUsersToWorklist(RV.getNode());
    1489             : 
    1490             :     // Finally, if the node is now dead, remove it from the graph.  The node
    1491             :     // may not be dead if the replacement process recursively simplified to
    1492             :     // something else needing this node. This will also take care of adding any
    1493             :     // operands which have lost a user to the worklist.
    1494     1906264 :     recursivelyDeleteUnusedNodes(N);
    1495             :   }
    1496             : 
    1497             :   // If the root changed (e.g. it was a dead load, update the root).
    1498     2767992 :   DAG.setRoot(Dummy.getValue());
    1499     2767992 :   DAG.RemoveDeadNodes();
    1500     2767992 : }
    1501             : 
    1502    88526629 : SDValue DAGCombiner::visit(SDNode *N) {
    1503   177053258 :   switch (N->getOpcode()) {
    1504             :   default: break;
    1505     4725134 :   case ISD::TokenFactor:        return visitTokenFactor(N);
    1506      436984 :   case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
    1507     3332748 :   case ISD::ADD:                return visitADD(N);
    1508      532128 :   case ISD::SUB:                return visitSUB(N);
    1509         522 :   case ISD::ADDC:               return visitADDC(N);
    1510      130962 :   case ISD::UADDO:              return visitUADDO(N);
    1511          89 :   case ISD::SUBC:               return visitSUBC(N);
    1512        1913 :   case ISD::USUBO:              return visitUSUBO(N);
    1513        2544 :   case ISD::ADDE:               return visitADDE(N);
    1514      156807 :   case ISD::ADDCARRY:           return visitADDCARRY(N);
    1515          79 :   case ISD::SUBE:               return visitSUBE(N);
    1516         839 :   case ISD::SUBCARRY:           return visitSUBCARRY(N);
    1517       39729 :   case ISD::MUL:                return visitMUL(N);
    1518        5883 :   case ISD::SDIV:               return visitSDIV(N);
    1519        4531 :   case ISD::UDIV:               return visitUDIV(N);
    1520        4801 :   case ISD::SREM:
    1521        4801 :   case ISD::UREM:               return visitREM(N);
    1522        6357 :   case ISD::MULHU:              return visitMULHU(N);
    1523        1513 :   case ISD::MULHS:              return visitMULHS(N);
    1524         723 :   case ISD::SMUL_LOHI:          return visitSMUL_LOHI(N);
    1525        5613 :   case ISD::UMUL_LOHI:          return visitUMUL_LOHI(N);
    1526          99 :   case ISD::SMULO:              return visitSMULO(N);
    1527         228 :   case ISD::UMULO:              return visitUMULO(N);
    1528       25437 :   case ISD::SMIN:
    1529             :   case ISD::SMAX:
    1530             :   case ISD::UMIN:
    1531       25437 :   case ISD::UMAX:               return visitIMINMAX(N);
    1532      400002 :   case ISD::AND:                return visitAND(N);
    1533      146434 :   case ISD::OR:                 return visitOR(N);
    1534       87480 :   case ISD::XOR:                return visitXOR(N);
    1535      123493 :   case ISD::SHL:                return visitSHL(N);
    1536       30600 :   case ISD::SRA:                return visitSRA(N);
    1537      164810 :   case ISD::SRL:                return visitSRL(N);
    1538        2760 :   case ISD::ROTR:
    1539        2760 :   case ISD::ROTL:               return visitRotate(N);
    1540        1036 :   case ISD::ABS:                return visitABS(N);
    1541        1476 :   case ISD::BSWAP:              return visitBSWAP(N);
    1542         559 :   case ISD::BITREVERSE:         return visitBITREVERSE(N);
    1543        1221 :   case ISD::CTLZ:               return visitCTLZ(N);
    1544        1564 :   case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
    1545         500 :   case ISD::CTTZ:               return visitCTTZ(N);
    1546         652 :   case ISD::CTTZ_ZERO_UNDEF:    return visitCTTZ_ZERO_UNDEF(N);
    1547        2141 :   case ISD::CTPOP:              return visitCTPOP(N);
    1548       32115 :   case ISD::SELECT:             return visitSELECT(N);
    1549       39265 :   case ISD::VSELECT:            return visitVSELECT(N);
    1550       16226 :   case ISD::SELECT_CC:          return visitSELECT_CC(N);
    1551      298267 :   case ISD::SETCC:              return visitSETCC(N);
    1552         289 :   case ISD::SETCCCARRY:         return visitSETCCCARRY(N);
    1553       51003 :   case ISD::SIGN_EXTEND:        return visitSIGN_EXTEND(N);
    1554      142725 :   case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
    1555       88162 :   case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
    1556      187971 :   case ISD::AssertSext:
    1557      187971 :   case ISD::AssertZext:         return visitAssertExt(N);
    1558       46974 :   case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
    1559        3049 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
    1560        5864 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
    1561      858614 :   case ISD::TRUNCATE:           return visitTRUNCATE(N);
    1562      821241 :   case ISD::BITCAST:            return visitBITCAST(N);
    1563             :   case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
    1564       43573 :   case ISD::FADD:               return visitFADD(N);
    1565       12783 :   case ISD::FSUB:               return visitFSUB(N);
    1566       27386 :   case ISD::FMUL:               return visitFMUL(N);
    1567        9771 :   case ISD::FMA:                return visitFMA(N);
    1568       10233 :   case ISD::FDIV:               return visitFDIV(N);
    1569         283 :   case ISD::FREM:               return visitFREM(N);
    1570        2134 :   case ISD::FSQRT:              return visitFSQRT(N);
    1571        1440 :   case ISD::FCOPYSIGN:          return visitFCOPYSIGN(N);
    1572         250 :   case ISD::FPOW:               return visitFPOW(N);
    1573       21933 :   case ISD::SINT_TO_FP:         return visitSINT_TO_FP(N);
    1574       12560 :   case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
    1575        8957 :   case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
    1576        6297 :   case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
    1577        4291 :   case ISD::FP_ROUND:           return visitFP_ROUND(N);
    1578           0 :   case ISD::FP_ROUND_INREG:     return visitFP_ROUND_INREG(N);
    1579        9165 :   case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
    1580        6789 :   case ISD::FNEG:               return visitFNEG(N);
    1581        5148 :   case ISD::FABS:               return visitFABS(N);
    1582        1642 :   case ISD::FFLOOR:             return visitFFLOOR(N);
    1583        3382 :   case ISD::FMINNUM:            return visitFMINNUM(N);
    1584        3298 :   case ISD::FMAXNUM:            return visitFMAXNUM(N);
    1585        4301 :   case ISD::FCEIL:              return visitFCEIL(N);
    1586        1548 :   case ISD::FTRUNC:             return visitFTRUNC(N);
    1587      258065 :   case ISD::BRCOND:             return visitBRCOND(N);
    1588        6356 :   case ISD::BR_CC:              return visitBR_CC(N);
    1589     6474249 :   case ISD::LOAD:               return visitLOAD(N);
    1590     8005537 :   case ISD::STORE:              return visitSTORE(N);
    1591       52048 :   case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
    1592      416312 :   case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
    1593      698113 :   case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
    1594       30650 :   case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
    1595       74221 :   case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
    1596       70837 :   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
    1597       23174 :   case ISD::SCALAR_TO_VECTOR:   return visitSCALAR_TO_VECTOR(N);
    1598       19322 :   case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
    1599         994 :   case ISD::MGATHER:            return visitMGATHER(N);
    1600        1291 :   case ISD::MLOAD:              return visitMLOAD(N);
    1601         287 :   case ISD::MSCATTER:           return visitMSCATTER(N);
    1602         761 :   case ISD::MSTORE:             return visitMSTORE(N);
    1603             :   case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
    1604        6740 :   case ISD::FP16_TO_FP:         return visitFP16_TO_FP(N);
    1605             :   }
    1606    59162749 :   return SDValue();
    1607             : }
    1608             : 
    1609    88526629 : SDValue DAGCombiner::combine(SDNode *N) {
    1610    88526629 :   SDValue RV = visit(N);
    1611             : 
    1612             :   // If nothing happened, try a target-specific DAG combine.
    1613    88526629 :   if (!RV.getNode()) {
    1614             :     assert(N->getOpcode() != ISD::DELETED_NODE &&
    1615             :            "Node was deleted but visit returned NULL!");
    1616             : 
    1617   171056554 :     if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
    1618    80949613 :         TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
    1619             : 
    1620             :       // Expose the DAG combiner to the target combiner impls.
    1621             :       TargetLowering::DAGCombinerInfo
    1622    24931877 :         DagCombineInfo(DAG, Level, false, this);
    1623             : 
    1624    24931877 :       RV = TLI.PerformDAGCombine(N, DagCombineInfo);
    1625             :     }
    1626             :   }
    1627             : 
    1628             :   // If nothing happened still, try promoting the operation.
    1629    88526629 :   if (!RV.getNode()) {
    1630   170452332 :     switch (N->getOpcode()) {
    1631             :     default: break;
    1632             :     case ISD::ADD:
    1633             :     case ISD::SUB:
    1634             :     case ISD::MUL:
    1635             :     case ISD::AND:
    1636             :     case ISD::OR:
    1637             :     case ISD::XOR:
    1638     3975682 :       RV = PromoteIntBinOp(SDValue(N, 0));
    1639     3975682 :       break;
    1640             :     case ISD::SHL:
    1641             :     case ISD::SRA:
    1642             :     case ISD::SRL:
    1643      259809 :       RV = PromoteIntShiftOp(SDValue(N, 0));
    1644      259809 :       break;
    1645             :     case ISD::SIGN_EXTEND:
    1646             :     case ISD::ZERO_EXTEND:
    1647             :     case ISD::ANY_EXTEND:
    1648      189609 :       RV = PromoteExtend(SDValue(N, 0));
    1649      189609 :       break;
    1650             :     case ISD::LOAD:
    1651     6143598 :       if (PromoteLoad(SDValue(N, 0)))
    1652           0 :         RV = SDValue(N, 0);
    1653             :       break;
    1654             :     }
    1655             :   }
    1656             : 
    1657             :   // If N is a commutative binary node, try eliminate it if the commuted
    1658             :   // version is already present in the DAG.
    1659    88526629 :   if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
    1660     3695149 :       N->getNumValues() == 1) {
    1661     3556035 :     SDValue N0 = N->getOperand(0);
    1662     3556035 :     SDValue N1 = N->getOperand(1);
    1663             : 
    1664             :     // Constant operands are canonicalized to RHS.
    1665             :     if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
    1666      635593 :       SDValue Ops[] = {N1, N0};
    1667     2542372 :       SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
    1668             :                                             N->getFlags());
    1669      635593 :       if (CSENode)
    1670          20 :         return SDValue(CSENode, 0);
    1671             :     }
    1672             :   }
    1673             : 
    1674    88526609 :   return RV;
    1675             : }
    1676             : 
    1677             : /// Given a node, return its input chain if it has one, otherwise return a null
    1678             : /// sd operand.
    1679     7171608 : static SDValue getInputChainForNode(SDNode *N) {
    1680    14343216 :   if (unsigned NumOps = N->getNumOperands()) {
    1681     7150080 :     if (N->getOperand(0).getValueType() == MVT::Other)
    1682     7100244 :       return N->getOperand(0);
    1683       99672 :     if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
    1684       49836 :       return N->getOperand(NumOps-1);
    1685           0 :     for (unsigned i = 1; i < NumOps-1; ++i)
    1686           0 :       if (N->getOperand(i).getValueType() == MVT::Other)
    1687           0 :         return N->getOperand(i);
    1688             :   }
    1689       21528 :   return SDValue();
    1690             : }
    1691             : 
    1692     4725134 : SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
    1693             :   // If N has two operands, where one has an input chain equal to the other,
    1694             :   // the 'other' chain is redundant.
    1695     4725134 :   if (N->getNumOperands() == 2) {
    1696     3593192 :     if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
    1697       14776 :       return N->getOperand(0);
    1698     3578416 :     if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
    1699       87635 :       return N->getOperand(1);
    1700             :   }
    1701             : 
    1702             :   // Don't simplify token factors if optnone.
    1703     4622723 :   if (OptLevel == CodeGenOpt::None)
    1704     2513835 :     return SDValue();
    1705             : 
    1706             :   SmallVector<SDNode *, 8> TFs;     // List of token factors to visit.
    1707             :   SmallVector<SDValue, 8> Ops;      // Ops for replacing token factor.
    1708             :   SmallPtrSet<SDNode*, 16> SeenOps;
    1709     2108888 :   bool Changed = false;             // If we should replace this token factor.
    1710             : 
    1711             :   // Start out with this token factor.
    1712     2108888 :   TFs.push_back(N);
    1713             : 
    1714             :   // Iterate through token factors.  The TFs grows when new token factors are
    1715             :   // encountered.
    1716     4950900 :   for (unsigned i = 0; i < TFs.size(); ++i) {
    1717     2842012 :     SDNode *TF = TFs[i];
    1718             : 
    1719             :     // Check each of the operands.
    1720    11192010 :     for (const SDValue &Op : TF->op_values()) {
    1721    16699996 :       switch (Op.getOpcode()) {
    1722       32291 :       case ISD::EntryToken:
    1723             :         // Entry tokens don't need to be added to the list. They are
    1724             :         // redundant.
    1725       32291 :         Changed = true;
    1726       32291 :         break;
    1727             : 
    1728     1288988 :       case ISD::TokenFactor:
    1729     1288988 :         if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
    1730             :           // Queue up for processing.
    1731      733124 :           TFs.push_back(Op.getNode());
    1732             :           // Clean up in case the token factor is removed.
    1733      733124 :           AddToWorklist(Op.getNode());
    1734      733124 :           Changed = true;
    1735      733124 :           break;
    1736             :         }
    1737             :         LLVM_FALLTHROUGH;
    1738             : 
    1739             :       default:
    1740             :         // Only add if it isn't already in the list.
    1741     7584583 :         if (SeenOps.insert(Op.getNode()).second)
    1742     7460977 :           Ops.push_back(Op);
    1743             :         else
    1744      123606 :           Changed = true;
    1745             :         break;
    1746             :       }
    1747             :     }
    1748             :   }
    1749             : 
    1750             :   // Remove Nodes that are chained to another node in the list. Do so
    1751             :   // by walking up chains breath-first stopping when we've seen
    1752             :   // another operand. In general we must climb to the EntryNode, but we can exit
    1753             :   // early if we find all remaining work is associated with just one operand as
    1754             :   // no further pruning is possible.
    1755             : 
    1756             :   // List of nodes to search through and original Ops from which they originate.
    1757             :   SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
    1758             :   SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
    1759             :   SmallPtrSet<SDNode *, 16> SeenChains;
    1760     2108888 :   bool DidPruneOps = false;
    1761             : 
    1762     2108888 :   unsigned NumLeftToConsider = 0;
    1763     9569865 :   for (const SDValue &Op : Ops) {
    1764     7460977 :     Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
    1765     7460977 :     OpWorkCount.push_back(1);
    1766             :   }
    1767             : 
    1768             :   auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
    1769             :     // If this is an Op, we can remove the op from the list. Remark any
    1770             :     // search associated with it as from the current OpNumber.
    1771             :     if (SeenOps.count(Op) != 0) {
    1772             :       Changed = true;
    1773             :       DidPruneOps = true;
    1774             :       unsigned OrigOpNumber = 0;
    1775             :       while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
    1776             :         OrigOpNumber++;
    1777             :       assert((OrigOpNumber != Ops.size()) &&
    1778             :              "expected to find TokenFactor Operand");
    1779             :       // Re-mark worklist from OrigOpNumber to OpNumber
    1780             :       for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
    1781             :         if (Worklist[i].second == OrigOpNumber) {
    1782             :           Worklist[i].second = OpNumber;
    1783             :         }
    1784             :       }
    1785             :       OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
    1786             :       OpWorkCount[OrigOpNumber] = 0;
    1787             :       NumLeftToConsider--;
    1788             :     }
    1789             :     // Add if it's a new chain
    1790             :     if (SeenChains.insert(Op).second) {
    1791             :       OpWorkCount[OpNumber]++;
    1792             :       Worklist.push_back(std::make_pair(Op, OpNumber));
    1793             :     }
    1794     2108888 :   };
    1795             : 
    1796    14624216 :   for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
    1797             :     // We need at least be consider at least 2 Ops to prune.
    1798    14153618 :     if (NumLeftToConsider <= 1)
    1799             :       break;
    1800    12515328 :     auto CurNode = Worklist[i].first;
    1801    12515328 :     auto CurOpNumber = Worklist[i].second;
    1802             :     assert((OpWorkCount[CurOpNumber] > 0) &&
    1803             :            "Node should not appear in worklist");
    1804    25030656 :     switch (CurNode->getOpcode()) {
    1805      489576 :     case ISD::EntryToken:
    1806             :       // Hitting EntryToken is the only way for the search to terminate without
    1807             :       // hitting
    1808             :       // another operand's search. Prevent us from marking this operand
    1809             :       // considered.
    1810      489576 :       NumLeftToConsider++;
    1811      489576 :       break;
    1812             :     case ISD::TokenFactor:
    1813     4264940 :       for (const SDValue &Op : CurNode->op_values())
    1814     3051420 :         AddToWorklist(i, Op.getNode(), CurOpNumber);
    1815             :       break;
    1816      978552 :     case ISD::CopyFromReg:
    1817             :     case ISD::CopyToReg:
    1818      978552 :       AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
    1819      978552 :       break;
    1820             :     default:
    1821             :       if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
    1822     9063493 :         AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
    1823             :       break;
    1824             :     }
    1825    12515328 :     OpWorkCount[CurOpNumber]--;
    1826    12515328 :     if (OpWorkCount[CurOpNumber] == 0)
    1827     5305132 :       NumLeftToConsider--;
    1828             :   }
    1829             : 
    1830             :   // If we've changed things around then replace token factor.
    1831     2108888 :   if (Changed) {
    1832             :     SDValue Result;
    1833      578983 :     if (Ops.empty()) {
    1834             :       // The entry token is the only possible outcome.
    1835         326 :       Result = DAG.getEntryNode();
    1836             :     } else {
    1837      578657 :       if (DidPruneOps) {
    1838             :         SmallVector<SDValue, 8> PrunedOps;
    1839             :         //
    1840     1811057 :         for (const SDValue &Op : Ops) {
    1841     1473956 :           if (SeenChains.count(Op.getNode()) == 0)
    1842      914705 :             PrunedOps.push_back(Op);
    1843             :         }
    1844     1287894 :         Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
    1845             :       } else {
    1846      798626 :         Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
    1847             :       }
    1848             :     }
    1849      578983 :     return Result;
    1850             :   }
    1851     1529905 :   return SDValue();
    1852             : }
    1853             : 
    1854             : /// MERGE_VALUES can always be eliminated.
    1855      436984 : SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
    1856             :   WorklistRemover DeadNodes(*this);
    1857             :   // Replacing results may cause a different MERGE_VALUES to suddenly
    1858             :   // be CSE'd with N, and carry its uses with it. Iterate until no
    1859             :   // uses remain, to ensure that the node can be safely deleted.
    1860             :   // First add the users of this node to the work list so that they
    1861             :   // can be tried again once they have new operands.
    1862             :   AddUsersToWorklist(N);
    1863             :   do {
    1864             :     // Do as a single replacement to avoid rewalking use lists.
    1865             :     SmallVector<SDValue, 8> Ops;
    1866     1316835 :     for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
    1867     1759702 :       Ops.push_back(N->getOperand(i));
    1868      436984 :     DAG.ReplaceAllUsesWith(N, Ops.data());
    1869      436984 :   } while (!N->use_empty());
    1870      436984 :   deleteAndRecombine(N);
    1871      873968 :   return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    1872             : }
    1873             : 
    1874             : /// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
    1875             : /// ConstantSDNode pointer else nullptr.
    1876           0 : static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
    1877             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
    1878      112653 :   return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
    1879             : }
    1880             : 
    1881           0 : SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
    1882             :   assert(ISD::isBinaryOp(BO) && "Unexpected binary operator");
    1883             : 
    1884             :   // Don't do this unless the old select is going away. We want to eliminate the
    1885             :   // binary operator, not replace a binop with a select.
    1886             :   // TODO: Handle ISD::SELECT_CC.
    1887             :   unsigned SelOpNo = 0;
    1888           0 :   SDValue Sel = BO->getOperand(0);
    1889           0 :   if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
    1890             :     SelOpNo = 1;
    1891           0 :     Sel = BO->getOperand(1);
    1892             :   }
    1893             : 
    1894           0 :   if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
    1895           0 :     return SDValue();
    1896             : 
    1897           0 :   SDValue CT = Sel.getOperand(1);
    1898           0 :   if (!isConstantOrConstantVector(CT, true) &&
    1899             :       !isConstantFPBuildVectorOrConstantFP(CT))
    1900           0 :     return SDValue();
    1901             : 
    1902           0 :   SDValue CF = Sel.getOperand(2);
    1903           0 :   if (!isConstantOrConstantVector(CF, true) &&
    1904             :       !isConstantFPBuildVectorOrConstantFP(CF))
    1905           0 :     return SDValue();
    1906             : 
    1907             :   // Bail out if any constants are opaque because we can't constant fold those.
    1908             :   // The exception is "and" and "or" with either 0 or -1 in which case we can
    1909             :   // propagate non constant operands into select. I.e.:
    1910             :   // and (select Cond, 0, -1), X --> select Cond, 0, X
    1911             :   // or X, (select Cond, -1, 0) --> select Cond, -1, X
    1912           0 :   auto BinOpcode = BO->getOpcode();
    1913           0 :   bool CanFoldNonConst = (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
    1914           0 :                          (isNullConstantOrNullSplatConstant(CT) ||
    1915           0 :                           isAllOnesConstantOrAllOnesSplatConstant(CT)) &&
    1916           0 :                          (isNullConstantOrNullSplatConstant(CF) ||
    1917           0 :                           isAllOnesConstantOrAllOnesSplatConstant(CF));
    1918             : 
    1919           0 :   SDValue CBO = BO->getOperand(SelOpNo ^ 1);
    1920           0 :   if (!CanFoldNonConst &&
    1921           0 :       !isConstantOrConstantVector(CBO, true) &&
    1922             :       !isConstantFPBuildVectorOrConstantFP(CBO))
    1923           0 :     return SDValue();
    1924             : 
    1925           0 :   EVT VT = Sel.getValueType();
    1926             : 
    1927             :   // In case of shift value and shift amount may have different VT. For instance
    1928             :   // on x86 shift amount is i8 regardles of LHS type. Bail out if we have
    1929             :   // swapped operands and value types do not match. NB: x86 is fine if operands
    1930             :   // are not swapped with shift amount VT being not bigger than shifted value.
    1931             :   // TODO: that is possible to check for a shift operation, correct VTs and
    1932             :   // still perform optimization on x86 if needed.
    1933           0 :   if (SelOpNo && VT != CBO.getValueType())
    1934           0 :     return SDValue();
    1935             : 
    1936             :   // We have a select-of-constants followed by a binary operator with a
    1937             :   // constant. Eliminate the binop by pulling the constant math into the select.
    1938             :   // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
    1939           0 :   SDLoc DL(Sel);
    1940           0 :   SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
    1941           0 :                           : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
    1942           0 :   if (!CanFoldNonConst && !NewCT.isUndef() &&
    1943           0 :       !isConstantOrConstantVector(NewCT, true) &&
    1944             :       !isConstantFPBuildVectorOrConstantFP(NewCT))
    1945           0 :     return SDValue();
    1946             : 
    1947           0 :   SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
    1948           0 :                           : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
    1949           0 :   if (!CanFoldNonConst && !NewCF.isUndef() &&
    1950           0 :       !isConstantOrConstantVector(NewCF, true) &&
    1951             :       !isConstantFPBuildVectorOrConstantFP(NewCF))
    1952           0 :     return SDValue();
    1953             : 
    1954           0 :   return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
    1955             : }
    1956             : 
    1957     3626163 : static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
    1958             :   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
    1959             :          "Expecting add or sub");
    1960             : 
    1961             :   // Match a constant operand and a zext operand for the math instruction:
    1962             :   // add Z, C
    1963             :   // sub C, Z
    1964     3626163 :   bool IsAdd = N->getOpcode() == ISD::ADD;
    1965     3626163 :   SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
    1966     3626163 :   SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
    1967             :   auto *CN = dyn_cast<ConstantSDNode>(C);
    1968     2641274 :   if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
    1969     3625337 :     return SDValue();
    1970             : 
    1971             :   // Match the zext operand as a setcc of a boolean.
    1972        1652 :   if (Z.getOperand(0).getOpcode() != ISD::SETCC ||
    1973         143 :       Z.getOperand(0).getValueType() != MVT::i1)
    1974         702 :     return SDValue();
    1975             : 
    1976             :   // Match the compare as: setcc (X & 1), 0, eq.
    1977             :   SDValue SetCC = Z.getOperand(0);
    1978         124 :   ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
    1979          37 :   if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) ||
    1980         175 :       SetCC.getOperand(0).getOpcode() != ISD::AND ||
    1981          23 :       !isOneConstant(SetCC.getOperand(0).getOperand(1)))
    1982         102 :     return SDValue();
    1983             : 
    1984             :   // We are adding/subtracting a constant and an inverted low bit. Turn that
    1985             :   // into a subtract/add of the low bit with incremented/decremented constant:
    1986             :   // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
    1987             :   // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
    1988          22 :   EVT VT = C.getValueType();
    1989             :   SDLoc DL(N);
    1990          22 :   SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
    1991          61 :   SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
    1992          62 :                        DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
    1993          31 :   return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
    1994             : }
    1995             : 
    1996             : /// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
    1997             : /// a shift and add with a different constant.
    1998     3626141 : static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
    1999             :   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
    2000             :          "Expecting add or sub");
    2001             : 
    2002             :   // We need a constant operand for the add/sub, and the other operand is a
    2003             :   // logical shift right: add (srl), C or sub C, (srl).
    2004     3626141 :   bool IsAdd = N->getOpcode() == ISD::ADD;
    2005     3626141 :   SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
    2006     3626141 :   SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
    2007     3626141 :   ConstantSDNode *C = isConstOrConstSplat(ConstantOp);
    2008     3626141 :   if (!C || ShiftOp.getOpcode() != ISD::SRL)
    2009     3624375 :     return SDValue();
    2010             : 
    2011             :   // The shift must be of a 'not' value.
    2012        1766 :   SDValue Not = ShiftOp.getOperand(0);
    2013        1766 :   if (!Not.hasOneUse() || !isBitwiseNot(Not))
    2014        1747 :     return SDValue();
    2015             : 
    2016             :   // The shift must be moving the sign bit to the least-significant-bit.
    2017          19 :   EVT VT = ShiftOp.getValueType();
    2018          19 :   SDValue ShAmt = ShiftOp.getOperand(1);
    2019          19 :   ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
    2020          38 :   if (!ShAmtC || ShAmtC->getZExtValue() != VT.getScalarSizeInBits() - 1)
    2021           0 :     return SDValue();
    2022             : 
    2023             :   // Eliminate the 'not' by adjusting the shift and add/sub constant:
    2024             :   // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
    2025             :   // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
    2026             :   SDLoc DL(N);
    2027          19 :   auto ShOpcode = IsAdd ? ISD::SRA : ISD::SRL;
    2028          19 :   SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
    2029          48 :   APInt NewC = IsAdd ? C->getAPIntValue() + 1 : C->getAPIntValue() - 1;
    2030          19 :   return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT));
    2031             : }
    2032             : 
    2033     3332748 : SDValue DAGCombiner::visitADD(SDNode *N) {
    2034     3332748 :   SDValue N0 = N->getOperand(0);
    2035     3332748 :   SDValue N1 = N->getOperand(1);
    2036     3332748 :   EVT VT = N0.getValueType();
    2037             :   SDLoc DL(N);
    2038             : 
    2039             :   // fold vector ops
    2040     3332748 :   if (VT.isVector()) {
    2041      141605 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2042          17 :       return FoldedVOp;
    2043             : 
    2044             :     // fold (add x, 0) -> x, vector edition
    2045      141588 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    2046         137 :       return N0;
    2047      141451 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    2048         158 :       return N1;
    2049             :   }
    2050             : 
    2051             :   // fold (add x, undef) -> undef
    2052     3332436 :   if (N0.isUndef())
    2053           3 :     return N0;
    2054             : 
    2055     3332433 :   if (N1.isUndef())
    2056           9 :     return N1;
    2057             : 
    2058     3332424 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
    2059             :     // canonicalize constant to RHS
    2060        2098 :     if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
    2061        2926 :       return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
    2062             :     // fold (add c1, c2) -> c1+c2
    2063         635 :     return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
    2064         635 :                                       N1.getNode());
    2065             :   }
    2066             : 
    2067             :   // fold (add x, 0) -> x
    2068     3330326 :   if (isNullConstant(N1))
    2069         327 :     return N0;
    2070             : 
    2071     3329999 :   if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
    2072             :     // fold ((c1-A)+c2) -> (c1+c2)-A
    2073     2956591 :     if (N0.getOpcode() == ISD::SUB &&
    2074         644 :         isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
    2075             :       // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
    2076          31 :       return DAG.getNode(ISD::SUB, DL, VT,
    2077             :                          DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
    2078          31 :                          N0.getOperand(1));
    2079             :     }
    2080             : 
    2081             :     // add (sext i1 X), 1 -> zext (not i1 X)
    2082             :     // We don't transform this pattern:
    2083             :     //   add (zext i1 X), -1 -> sext (not i1 X)
    2084             :     // because most (?) targets generate better code for the zext form.
    2085     2957988 :     if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
    2086        1002 :         isOneConstantOrOneSplatConstant(N1)) {
    2087          41 :       SDValue X = N0.getOperand(0);
    2088          41 :       if ((!LegalOperations ||
    2089           3 :            (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
    2090          81 :             TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
    2091          40 :           X.getScalarValueSizeInBits() == 1) {
    2092          34 :         SDValue Not = DAG.getNOT(DL, X, X.getValueType());
    2093          34 :         return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
    2094             :       }
    2095             :     }
    2096             : 
    2097             :     // Undo the add -> or combine to merge constant offsets from a frame index.
    2098             :     if (N0.getOpcode() == ISD::OR &&
    2099             :         isa<FrameIndexSDNode>(N0.getOperand(0)) &&
    2100     2958723 :         isa<ConstantSDNode>(N0.getOperand(1)) &&
    2101        5648 :         DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
    2102        5648 :       SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
    2103        5648 :       return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
    2104             :     }
    2105             :   }
    2106             : 
    2107     3327127 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2108           8 :     return NewSel;
    2109             : 
    2110             :   // reassociate add
    2111     3327119 :   if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
    2112      223488 :     return RADD;
    2113             : 
    2114             :   // fold ((0-A) + B) -> B-A
    2115     3106447 :   if (N0.getOpcode() == ISD::SUB &&
    2116        2816 :       isNullConstantOrNullSplatConstant(N0.getOperand(0)))
    2117          54 :     return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
    2118             : 
    2119             :   // fold (A + (0-B)) -> A-B
    2120     3109784 :   if (N1.getOpcode() == ISD::SUB &&
    2121        6180 :       isNullConstantOrNullSplatConstant(N1.getOperand(0)))
    2122         620 :     return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
    2123             : 
    2124             :   // fold (A+(B-A)) -> B
    2125     3103294 :   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
    2126           4 :     return N1.getOperand(0);
    2127             : 
    2128             :   // fold ((B-A)+A) -> B
    2129     3103290 :   if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
    2130          31 :     return N0.getOperand(0);
    2131             : 
    2132             :   // fold (A+(B-(A+C))) to (B-C)
    2133     3103259 :   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
    2134          63 :       N0 == N1.getOperand(1).getOperand(0))
    2135           2 :     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
    2136           2 :                        N1.getOperand(1).getOperand(1));
    2137             : 
    2138             :   // fold (A+(B-(C+A))) to (B-C)
    2139     3103257 :   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
    2140          61 :       N0 == N1.getOperand(1).getOperand(1))
    2141           3 :     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
    2142           3 :                        N1.getOperand(1).getOperand(0));
    2143             : 
    2144             :   // fold (A+((B-A)+or-C)) to (B+or-C)
    2145     3103254 :   if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
    2146     3107248 :       N1.getOperand(0).getOpcode() == ISD::SUB &&
    2147         103 :       N0 == N1.getOperand(0).getOperand(1))
    2148           5 :     return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
    2149           5 :                        N1.getOperand(1));
    2150             : 
    2151             :   // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
    2152     3103249 :   if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
    2153          18 :     SDValue N00 = N0.getOperand(0);
    2154          18 :     SDValue N01 = N0.getOperand(1);
    2155          18 :     SDValue N10 = N1.getOperand(0);
    2156          18 :     SDValue N11 = N1.getOperand(1);
    2157             : 
    2158          18 :     if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
    2159           5 :       return DAG.getNode(ISD::SUB, DL, VT,
    2160           5 :                          DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
    2161          15 :                          DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
    2162             :   }
    2163             : 
    2164     3103244 :   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
    2165          13 :     return V;
    2166             : 
    2167     3103231 :   if (SDValue V = foldAddSubOfSignBit(N, DAG))
    2168          10 :     return V;
    2169             : 
    2170     3103221 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    2171        5103 :     return SDValue(N, 0);
    2172             : 
    2173             :   // fold (a+b) -> (a|b) iff a and b share no bits.
    2174     6190470 :   if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
    2175     3092352 :       DAG.haveNoCommonBitsSet(N0, N1))
    2176       61872 :     return DAG.getNode(ISD::OR, DL, VT, N0, N1);
    2177             : 
    2178             :   // fold (add (xor a, -1), 1) -> (sub 0, a)
    2179     3067182 :   if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1))
    2180           9 :     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
    2181           9 :                        N0.getOperand(0));
    2182             : 
    2183     3067173 :   if (SDValue Combined = visitADDLike(N0, N1, N))
    2184         804 :     return Combined;
    2185             : 
    2186     3066369 :   if (SDValue Combined = visitADDLike(N1, N0, N))
    2187          73 :     return Combined;
    2188             : 
    2189     3066296 :   return SDValue();
    2190             : }
    2191             : 
    2192     5826665 : static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
    2193             :   bool Masked = false;
    2194             : 
    2195             :   // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
    2196             :   while (true) {
    2197     5837978 :     if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
    2198        9114 :       V = V.getOperand(0);
    2199        9114 :       continue;
    2200             :     }
    2201             : 
    2202     5828864 :     if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
    2203             :       Masked = true;
    2204        2199 :       V = V.getOperand(0);
    2205        2199 :       continue;
    2206             :     }
    2207             : 
    2208             :     break;
    2209             :   }
    2210             : 
    2211             :   // If this is not a carry, return.
    2212     5826665 :   if (V.getResNo() != 1)
    2213     5813747 :     return SDValue();
    2214             : 
    2215       12646 :   if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
    2216       24932 :       V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
    2217       12014 :     return SDValue();
    2218             : 
    2219             :   // If the result is masked, then no matter what kind of bool it is we can
    2220             :   // return. If it isn't, then we need to make sure the bool type is either 0 or
    2221             :   // 1 and not other values.
    2222         904 :   if (Masked ||
    2223         926 :       TLI.getBooleanContents(V.getValueType()) ==
    2224             :           TargetLoweringBase::ZeroOrOneBooleanContent)
    2225         904 :     return V;
    2226             : 
    2227           0 :   return SDValue();
    2228             : }
    2229             : 
    2230           0 : SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
    2231           0 :   EVT VT = N0.getValueType();
    2232             :   SDLoc DL(LocReference);
    2233             : 
    2234             :   // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
    2235           0 :   if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
    2236           0 :       isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
    2237           0 :     return DAG.getNode(ISD::SUB, DL, VT, N0,
    2238             :                        DAG.getNode(ISD::SHL, DL, VT,
    2239           0 :                                    N1.getOperand(0).getOperand(1),
    2240           0 :                                    N1.getOperand(1)));
    2241             : 
    2242           0 :   if (N1.getOpcode() == ISD::AND) {
    2243           0 :     SDValue AndOp0 = N1.getOperand(0);
    2244           0 :     unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
    2245             :     unsigned DestBits = VT.getScalarSizeInBits();
    2246             : 
    2247             :     // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
    2248             :     // and similar xforms where the inner op is either ~0 or 0.
    2249           0 :     if (NumSignBits == DestBits &&
    2250           0 :         isOneConstantOrOneSplatConstant(N1->getOperand(1)))
    2251           0 :       return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
    2252             :   }
    2253             : 
    2254             :   // add (sext i1), X -> sub X, (zext i1)
    2255             :   if (N0.getOpcode() == ISD::SIGN_EXTEND &&
    2256           0 :       N0.getOperand(0).getValueType() == MVT::i1 &&
    2257           0 :       !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
    2258           0 :     SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
    2259           0 :     return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
    2260             :   }
    2261             : 
    2262             :   // add X, (sextinreg Y i1) -> sub X, (and Y 1)
    2263           0 :   if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2264             :     VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    2265           0 :     if (TN->getVT() == MVT::i1) {
    2266           0 :       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
    2267           0 :                                  DAG.getConstant(1, DL, VT));
    2268           0 :       return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
    2269             :     }
    2270             :   }
    2271             : 
    2272             :   // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
    2273           0 :   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
    2274           0 :       N1.getResNo() == 0)
    2275           0 :     return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
    2276           0 :                        N0, N1.getOperand(0), N1.getOperand(2));
    2277             : 
    2278             :   // (add X, Carry) -> (addcarry X, 0, Carry)
    2279           0 :   if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
    2280           0 :     if (SDValue Carry = getAsCarry(TLI, N1))
    2281           0 :       return DAG.getNode(ISD::ADDCARRY, DL,
    2282           0 :                          DAG.getVTList(VT, Carry.getValueType()), N0,
    2283           0 :                          DAG.getConstant(0, DL, VT), Carry);
    2284             : 
    2285           0 :   return SDValue();
    2286             : }
    2287             : 
    2288         522 : SDValue DAGCombiner::visitADDC(SDNode *N) {
    2289         522 :   SDValue N0 = N->getOperand(0);
    2290         522 :   SDValue N1 = N->getOperand(1);
    2291         522 :   EVT VT = N0.getValueType();
    2292             :   SDLoc DL(N);
    2293             : 
    2294             :   // If the flag result is dead, turn this into an ADD.
    2295         522 :   if (!N->hasAnyUseOfValue(1))
    2296          15 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2297          45 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2298             : 
    2299             :   // canonicalize constant to RHS.
    2300             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2301             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2302         507 :   if (N0C && !N1C)
    2303           0 :     return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
    2304             : 
    2305             :   // fold (addc x, 0) -> x + no carry out
    2306         507 :   if (isNullConstant(N1))
    2307          11 :     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
    2308          22 :                                         DL, MVT::Glue));
    2309             : 
    2310             :   // If it cannot overflow, transform into an add.
    2311         496 :   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
    2312          11 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2313          33 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2314             : 
    2315         485 :   return SDValue();
    2316             : }
    2317             : 
    2318           7 : static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT,
    2319             :                            SelectionDAG &DAG, const TargetLowering &TLI) {
    2320           7 :   SDValue Cst;
    2321           7 :   switch (TLI.getBooleanContents(VT)) {
    2322           7 :   case TargetLowering::ZeroOrOneBooleanContent:
    2323             :   case TargetLowering::UndefinedBooleanContent:
    2324           7 :     Cst = DAG.getConstant(1, DL, VT);
    2325           7 :     break;
    2326           0 :   case TargetLowering::ZeroOrNegativeOneBooleanContent:
    2327           0 :     Cst = DAG.getConstant(-1, DL, VT);
    2328           0 :     break;
    2329             :   }
    2330             : 
    2331           7 :   return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
    2332             : }
    2333             : 
    2334           0 : static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) {
    2335           0 :   if (V.getOpcode() != ISD::XOR) return false;
    2336             :   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V.getOperand(1));
    2337           0 :   if (!Const) return false;
    2338             : 
    2339           0 :   switch(TLI.getBooleanContents(VT)) {
    2340           0 :     case TargetLowering::ZeroOrOneBooleanContent:
    2341           0 :       return Const->isOne();
    2342           0 :     case TargetLowering::ZeroOrNegativeOneBooleanContent:
    2343           0 :       return Const->isAllOnesValue();
    2344           0 :     case TargetLowering::UndefinedBooleanContent:
    2345           0 :       return (Const->getAPIntValue() & 0x01) == 1;
    2346             :   }
    2347           0 :   llvm_unreachable("Unsupported boolean content");
    2348             : }
    2349             : 
    2350      130962 : SDValue DAGCombiner::visitUADDO(SDNode *N) {
    2351      130962 :   SDValue N0 = N->getOperand(0);
    2352      130962 :   SDValue N1 = N->getOperand(1);
    2353      130962 :   EVT VT = N0.getValueType();
    2354      130962 :   if (VT.isVector())
    2355           0 :     return SDValue();
    2356             : 
    2357      261924 :   EVT CarryVT = N->getValueType(1);
    2358             :   SDLoc DL(N);
    2359             : 
    2360             :   // If the flag result is dead, turn this into an ADD.
    2361      130962 :   if (!N->hasAnyUseOfValue(1))
    2362         405 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2363         810 :                      DAG.getUNDEF(CarryVT));
    2364             : 
    2365             :   // canonicalize constant to RHS.
    2366             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2367             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2368      130557 :   if (N0C && !N1C)
    2369          10 :     return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
    2370             : 
    2371             :   // fold (uaddo x, 0) -> x + no carry out
    2372      130552 :   if (isNullConstant(N1))
    2373         904 :     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
    2374             : 
    2375             :   // If it cannot overflow, transform into an add.
    2376      130100 :   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
    2377         103 :     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
    2378         206 :                      DAG.getConstant(0, DL, CarryVT));
    2379             : 
    2380             :   // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
    2381      129997 :   if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1)) {
    2382           4 :     SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
    2383             :                               DAG.getConstant(0, DL, VT),
    2384           4 :                               N0.getOperand(0));
    2385             :     return CombineTo(N, Sub,
    2386           4 :                      flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
    2387             :   }
    2388             : 
    2389      129993 :   if (SDValue Combined = visitUADDOLike(N0, N1, N))
    2390         231 :     return Combined;
    2391             : 
    2392      129762 :   if (SDValue Combined = visitUADDOLike(N1, N0, N))
    2393           9 :     return Combined;
    2394             : 
    2395      129753 :   return SDValue();
    2396             : }
    2397             : 
    2398           0 : SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
    2399           0 :   auto VT = N0.getValueType();
    2400             : 
    2401             :   // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
    2402             :   // If Y + 1 cannot overflow.
    2403           0 :   if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
    2404           0 :     SDValue Y = N1.getOperand(0);
    2405           0 :     SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
    2406           0 :     if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
    2407           0 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
    2408           0 :                          N1.getOperand(2));
    2409             :   }
    2410             : 
    2411             :   // (uaddo X, Carry) -> (addcarry X, 0, Carry)
    2412           0 :   if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
    2413           0 :     if (SDValue Carry = getAsCarry(TLI, N1))
    2414           0 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
    2415           0 :                          DAG.getConstant(0, SDLoc(N), VT), Carry);
    2416             : 
    2417           0 :   return SDValue();
    2418             : }
    2419             : 
    2420           0 : SDValue DAGCombiner::visitADDE(SDNode *N) {
    2421           0 :   SDValue N0 = N->getOperand(0);
    2422           0 :   SDValue N1 = N->getOperand(1);
    2423           0 :   SDValue CarryIn = N->getOperand(2);
    2424             : 
    2425             :   // canonicalize constant to RHS
    2426             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2427             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2428           0 :   if (N0C && !N1C)
    2429           0 :     return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
    2430           0 :                        N1, N0, CarryIn);
    2431             : 
    2432             :   // fold (adde x, y, false) -> (addc x, y)
    2433           0 :   if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    2434           0 :     return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
    2435             : 
    2436           0 :   return SDValue();
    2437             : }
    2438             : 
    2439      156807 : SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
    2440      156807 :   SDValue N0 = N->getOperand(0);
    2441      156807 :   SDValue N1 = N->getOperand(1);
    2442      156807 :   SDValue CarryIn = N->getOperand(2);
    2443             :   SDLoc DL(N);
    2444             : 
    2445             :   // canonicalize constant to RHS
    2446             :   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
    2447             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    2448      156807 :   if (N0C && !N1C)
    2449         720 :     return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
    2450             : 
    2451             :   // fold (addcarry x, y, false) -> (uaddo x, y)
    2452      156447 :   if (isNullConstant(CarryIn)) {
    2453         565 :     if (!LegalOperations ||
    2454          76 :         TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
    2455        1130 :       return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
    2456             :   }
    2457             : 
    2458      155882 :   EVT CarryVT = CarryIn.getValueType();
    2459             : 
    2460             :   // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
    2461      155882 :   if (isNullConstant(N0) && isNullConstant(N1)) {
    2462         510 :     EVT VT = N0.getValueType();
    2463         510 :     SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
    2464         510 :     AddToWorklist(CarryExt.getNode());
    2465         510 :     return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
    2466             :                                     DAG.getConstant(1, DL, VT)),
    2467        1020 :                      DAG.getConstant(0, DL, CarryVT));
    2468             :   }
    2469             : 
    2470             :   // fold (addcarry (xor a, -1), 0, !b) -> (subcarry 0, a, b) and flip carry.
    2471      155380 :   if (isBitwiseNot(N0) && isNullConstant(N1) &&
    2472           8 :       isBooleanFlip(CarryIn, CarryVT, TLI)) {
    2473           3 :     SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(),
    2474             :                               DAG.getConstant(0, DL, N0.getValueType()),
    2475           3 :                               N0.getOperand(0), CarryIn.getOperand(0));
    2476             :     return CombineTo(N, Sub,
    2477           3 :                      flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
    2478             :   }
    2479             : 
    2480      155369 :   if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
    2481         234 :     return Combined;
    2482             : 
    2483      155135 :   if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
    2484           0 :     return Combined;
    2485             : 
    2486      155135 :   return SDValue();
    2487             : }
    2488             : 
    2489      310504 : SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
    2490             :                                        SDNode *N) {
    2491             :   // Iff the flag result is dead:
    2492             :   // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
    2493      309060 :   if ((N0.getOpcode() == ISD::ADD ||
    2494       16535 :        (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
    2495      324322 :       isNullConstant(N1) && !N->hasAnyUseOfValue(1))
    2496         464 :     return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
    2497         232 :                        N0.getOperand(0), N0.getOperand(1), CarryIn);
    2498             : 
    2499             :   /**
    2500             :    * When one of the addcarry argument is itself a carry, we may be facing
    2501             :    * a diamond carry propagation. In which case we try to transform the DAG
    2502             :    * to ensure linear carry propagation if that is possible.
    2503             :    *
    2504             :    * We are trying to get:
    2505             :    *   (addcarry X, 0, (addcarry A, B, Z):Carry)
    2506             :    */
    2507      310272 :   if (auto Y = getAsCarry(TLI, N1)) {
    2508             :     /**
    2509             :      *            (uaddo A, B)
    2510             :      *             /       \
    2511             :      *          Carry      Sum
    2512             :      *            |          \
    2513             :      *            | (addcarry *, 0, Z)
    2514             :      *            |       /
    2515             :      *             \   Carry
    2516             :      *              |   /
    2517             :      * (addcarry X, *, *)
    2518             :      */
    2519         621 :     if (Y.getOpcode() == ISD::UADDO &&
    2520         621 :         CarryIn.getResNo() == 1 &&
    2521          12 :         CarryIn.getOpcode() == ISD::ADDCARRY &&
    2522         900 :         isNullConstant(CarryIn.getOperand(1)) &&
    2523           4 :         CarryIn.getOperand(0) == Y.getValue(0)) {
    2524           2 :       auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
    2525             :                               Y.getOperand(0), Y.getOperand(1),
    2526           2 :                               CarryIn.getOperand(2));
    2527           2 :       AddToWorklist(NewY.getNode());
    2528           4 :       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
    2529           2 :                          DAG.getConstant(0, SDLoc(N), N0.getValueType()),
    2530           2 :                          NewY.getValue(1));
    2531             :     }
    2532             :   }
    2533             : 
    2534      310270 :   return SDValue();
    2535             : }
    2536             : 
    2537             : // Since it may not be valid to emit a fold to zero for vector initializers
    2538             : // check if we can before folding.
    2539          51 : static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
    2540             :                              SelectionDAG &DAG, bool LegalOperations,
    2541             :                              bool LegalTypes) {
    2542          51 :   if (!VT.isVector())
    2543          38 :     return DAG.getConstant(0, DL, VT);
    2544          13 :   if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
    2545          13 :     return DAG.getConstant(0, DL, VT);
    2546           0 :   return SDValue();
    2547             : }
    2548             : 
    2549      532128 : SDValue DAGCombiner::visitSUB(SDNode *N) {
    2550      532128 :   SDValue N0 = N->getOperand(0);
    2551      532128 :   SDValue N1 = N->getOperand(1);
    2552      532128 :   EVT VT = N0.getValueType();
    2553             :   SDLoc DL(N);
    2554             : 
    2555             :   // fold vector ops
    2556      532128 :   if (VT.isVector()) {
    2557      483137 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2558           4 :       return FoldedVOp;
    2559             : 
    2560             :     // fold (sub x, 0) -> x, vector edition
    2561      483133 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    2562           7 :       return N0;
    2563             :   }
    2564             : 
    2565             :   // fold (sub x, x) -> 0
    2566             :   // FIXME: Refactor this and xor and other similar operations together.
    2567      532117 :   if (N0 == N1)
    2568          32 :     return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
    2569      542117 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    2570       10032 :       DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
    2571             :     // fold (sub c1, c2) -> c1-c2
    2572           3 :     return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
    2573           3 :                                       N1.getNode());
    2574             :   }
    2575             : 
    2576      532082 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2577          11 :     return NewSel;
    2578             : 
    2579      532071 :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    2580             : 
    2581             :   // fold (sub x, c) -> (add x, -c)
    2582             :   if (N1C) {
    2583        8287 :     return DAG.getNode(ISD::ADD, DL, VT, N0,
    2584       24861 :                        DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
    2585             :   }
    2586             : 
    2587      523784 :   if (isNullConstantOrNullSplatConstant(N0)) {
    2588             :     unsigned BitWidth = VT.getScalarSizeInBits();
    2589             :     // Right-shifting everything out but the sign bit followed by negation is
    2590             :     // the same as flipping arithmetic/logical shift type without the negation:
    2591             :     // -(X >>u 31) -> (X >>s 31)
    2592             :     // -(X >>s 31) -> (X >>u 31)
    2593       10050 :     if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
    2594         135 :       ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
    2595         267 :       if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
    2596          11 :         auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
    2597          11 :         if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
    2598          22 :           return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
    2599             :       }
    2600             :     }
    2601             : 
    2602             :     // 0 - X --> 0 if the sub is NUW.
    2603        5014 :     if (N->getFlags().hasNoUnsignedWrap())
    2604           2 :       return N0;
    2605             : 
    2606       15036 :     if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
    2607             :       // N1 is either 0 or the minimum signed value. If the sub is NSW, then
    2608             :       // N1 must be 0 because negating the minimum signed value is undefined.
    2609           8 :       if (N->getFlags().hasNoSignedWrap())
    2610           2 :         return N0;
    2611             : 
    2612             :       // 0 - X --> X if X is 0 or the minimum signed value.
    2613           6 :       return N1;
    2614             :     }
    2615             :   }
    2616             : 
    2617             :   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
    2618      523763 :   if (isAllOnesConstantOrAllOnesSplatConstant(N0))
    2619         134 :     return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
    2620             : 
    2621             :   // fold (A - (0-B)) -> A+B
    2622     1048146 :   if (N1.getOpcode() == ISD::SUB &&
    2623         754 :       isNullConstantOrNullSplatConstant(N1.getOperand(0)))
    2624          87 :     return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
    2625             : 
    2626             :   // fold A-(A-B) -> B
    2627     1047334 :   if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
    2628         212 :     return N1.getOperand(1);
    2629             : 
    2630             :   // fold (A+B)-A -> B
    2631      523455 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
    2632          22 :     return N0.getOperand(1);
    2633             : 
    2634             :   // fold (A+B)-B -> A
    2635      523433 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
    2636           3 :     return N0.getOperand(0);
    2637             : 
    2638             :   // fold C2-(A+C1) -> (C2-C1)-A
    2639      523430 :   if (N1.getOpcode() == ISD::ADD) {
    2640         597 :     SDValue N11 = N1.getOperand(1);
    2641         784 :     if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
    2642         187 :         isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
    2643         288 :       SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
    2644         432 :       return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
    2645             :     }
    2646             :   }
    2647             : 
    2648             :   // fold ((A+(B+or-C))-B) -> A+or-C
    2649        1777 :   if (N0.getOpcode() == ISD::ADD &&
    2650        1777 :       (N0.getOperand(1).getOpcode() == ISD::SUB ||
    2651      523286 :        N0.getOperand(1).getOpcode() == ISD::ADD) &&
    2652          46 :       N0.getOperand(1).getOperand(0) == N1)
    2653           7 :     return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
    2654           7 :                        N0.getOperand(1).getOperand(1));
    2655             : 
    2656             :   // fold ((A+(C+B))-B) -> A+C
    2657      523279 :   if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
    2658           7 :       N0.getOperand(1).getOperand(1) == N1)
    2659           1 :     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
    2660           1 :                        N0.getOperand(1).getOperand(0));
    2661             : 
    2662             :   // fold ((A-(B-C))-C) -> A-B
    2663      523278 :   if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
    2664           3 :       N0.getOperand(1).getOperand(1) == N1)
    2665           3 :     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
    2666           3 :                        N0.getOperand(1).getOperand(0));
    2667             : 
    2668             :   // fold (A-(B-C)) -> A+(C-B)
    2669     1046550 :   if (N1.getOpcode() == ISD::SUB && N1.hasOneUse())
    2670         347 :     return DAG.getNode(ISD::ADD, DL, VT, N0,
    2671             :                        DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
    2672         694 :                                    N1.getOperand(0)));
    2673             : 
    2674             :   // fold (X - (-Y * Z)) -> (X + (Y * Z))
    2675     1045856 :   if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) {
    2676        6864 :     if (N1.getOperand(0).getOpcode() == ISD::SUB &&
    2677           9 :         isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0))) {
    2678           9 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
    2679           9 :                                 N1.getOperand(0).getOperand(1),
    2680          18 :                                 N1.getOperand(1));
    2681          18 :       return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
    2682             :     }
    2683        6828 :     if (N1.getOperand(1).getOpcode() == ISD::SUB &&
    2684           0 :         isNullConstantOrNullSplatConstant(N1.getOperand(1).getOperand(0))) {
    2685           0 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
    2686             :                                 N1.getOperand(0),
    2687           0 :                                 N1.getOperand(1).getOperand(1));
    2688           0 :       return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
    2689             :     }
    2690             :   }
    2691             : 
    2692             :   // If either operand of a sub is undef, the result is undef
    2693      522919 :   if (N0.isUndef())
    2694           0 :     return N0;
    2695     1045838 :   if (N1.isUndef())
    2696           0 :     return N1;
    2697             : 
    2698      522919 :   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
    2699           9 :     return V;
    2700             : 
    2701      522910 :   if (SDValue V = foldAddSubOfSignBit(N, DAG))
    2702           9 :     return V;
    2703             : 
    2704             :   // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
    2705      522901 :   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
    2706       30009 :     if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
    2707           2 :       SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
    2708           2 :       SDValue S0 = N1.getOperand(0);
    2709             :       if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
    2710             :         unsigned OpSizeInBits = VT.getScalarSizeInBits();
    2711           4 :         if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
    2712           4 :           if (C->getAPIntValue() == (OpSizeInBits - 1))
    2713           4 :             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
    2714             :       }
    2715             :     }
    2716             :   }
    2717             : 
    2718             :   // If the relocation model supports it, consider symbol offsets.
    2719             :   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
    2720          11 :     if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
    2721             :       // fold (sub Sym, c) -> Sym-c
    2722             :       if (N1C && GA->getOpcode() == ISD::GlobalAddress)
    2723             :         return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
    2724             :                                     GA->getOffset() -
    2725             :                                         (uint64_t)N1C->getSExtValue());
    2726             :       // fold (sub Sym+c1, Sym+c2) -> c1-c2
    2727             :       if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
    2728           0 :         if (GA->getGlobal() == GB->getGlobal())
    2729           0 :           return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
    2730           0 :                                  DL, VT);
    2731             :     }
    2732             : 
    2733             :   // sub X, (sextinreg Y i1) -> add X, (and Y 1)
    2734     1045798 :   if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2735             :     VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    2736             :     if (TN->getVT() == MVT::i1) {
    2737          58 :       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
    2738          58 :                                  DAG.getConstant(1, DL, VT));
    2739         116 :       return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
    2740             :     }
    2741             :   }
    2742             : 
    2743             :   // Prefer an add for more folding potential and possibly better codegen:
    2744             :   // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
    2745      522841 :   if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
    2746          75 :     SDValue ShAmt = N1.getOperand(1);
    2747          75 :     ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
    2748         150 :     if (ShAmtC && ShAmtC->getZExtValue() == N1.getScalarValueSizeInBits() - 1) {
    2749          45 :       SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
    2750          30 :       return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
    2751             :     }
    2752             :   }
    2753             : 
    2754      522826 :   return SDValue();
    2755             : }
    2756             : 
    2757          89 : SDValue DAGCombiner::visitSUBC(SDNode *N) {
    2758          89 :   SDValue N0 = N->getOperand(0);
    2759          89 :   SDValue N1 = N->getOperand(1);
    2760          89 :   EVT VT = N0.getValueType();
    2761             :   SDLoc DL(N);
    2762             : 
    2763             :   // If the flag result is dead, turn this into an SUB.
    2764          89 :   if (!N->hasAnyUseOfValue(1))
    2765          24 :     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
    2766          72 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2767             : 
    2768             :   // fold (subc x, x) -> 0 + no borrow
    2769             :   if (N0 == N1)
    2770           0 :     return CombineTo(N, DAG.getConstant(0, DL, VT),
    2771           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2772             : 
    2773             :   // fold (subc x, 0) -> x + no borrow
    2774          65 :   if (isNullConstant(N1))
    2775           0 :     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2776             : 
    2777             :   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
    2778          65 :   if (isAllOnesConstant(N0))
    2779           0 :     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
    2780           0 :                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
    2781             : 
    2782          65 :   return SDValue();
    2783             : }
    2784             : 
    2785        1913 : SDValue DAGCombiner::visitUSUBO(SDNode *N) {
    2786        1913 :   SDValue N0 = N->getOperand(0);
    2787        1913 :   SDValue N1 = N->getOperand(1);
    2788        1913 :   EVT VT = N0.getValueType();
    2789        1913 :   if (VT.isVector())
    2790           0 :     return SDValue();
    2791             : 
    2792        3826 :   EVT CarryVT = N->getValueType(1);
    2793             :   SDLoc DL(N);
    2794             : 
    2795             :   // If the flag result is dead, turn this into an SUB.
    2796        1913 :   if (!N->hasAnyUseOfValue(1))
    2797         106 :     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
    2798         212 :                      DAG.getUNDEF(CarryVT));
    2799             : 
    2800             :   // fold (usubo x, x) -> 0 + no borrow
    2801             :   if (N0 == N1)
    2802           7 :     return CombineTo(N, DAG.getConstant(0, DL, VT),
    2803          14 :                      DAG.getConstant(0, DL, CarryVT));
    2804             : 
    2805             :   // fold (usubo x, 0) -> x + no borrow
    2806        1800 :   if (isNullConstant(N1))
    2807          32 :     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
    2808             : 
    2809             :   // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
    2810        1784 :   if (isAllOnesConstant(N0))
    2811           4 :     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
    2812           8 :                      DAG.getConstant(0, DL, CarryVT));
    2813             : 
    2814        1780 :   return SDValue();
    2815             : }
    2816             : 
    2817           0 : SDValue DAGCombiner::visitSUBE(SDNode *N) {
    2818           0 :   SDValue N0 = N->getOperand(0);
    2819           0 :   SDValue N1 = N->getOperand(1);
    2820           0 :   SDValue CarryIn = N->getOperand(2);
    2821             : 
    2822             :   // fold (sube x, y, false) -> (subc x, y)
    2823           0 :   if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    2824           0 :     return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
    2825             : 
    2826           0 :   return SDValue();
    2827             : }
    2828             : 
    2829         839 : SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
    2830         839 :   SDValue N0 = N->getOperand(0);
    2831         839 :   SDValue N1 = N->getOperand(1);
    2832         839 :   SDValue CarryIn = N->getOperand(2);
    2833             : 
    2834             :   // fold (subcarry x, y, false) -> (usubo x, y)
    2835         839 :   if (isNullConstant(CarryIn)) {
    2836         120 :     if (!LegalOperations ||
    2837         109 :         TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
    2838         240 :       return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
    2839             :   }
    2840             : 
    2841         719 :   return SDValue();
    2842             : }
    2843             : 
    2844       39729 : SDValue DAGCombiner::visitMUL(SDNode *N) {
    2845       39729 :   SDValue N0 = N->getOperand(0);
    2846       39729 :   SDValue N1 = N->getOperand(1);
    2847       79458 :   EVT VT = N0.getValueType();
    2848             : 
    2849             :   // fold (mul x, undef) -> 0
    2850       39729 :   if (N0.isUndef() || N1.isUndef())
    2851          56 :     return DAG.getConstant(0, SDLoc(N), VT);
    2852             : 
    2853             :   bool N0IsConst = false;
    2854             :   bool N1IsConst = false;
    2855             :   bool N1IsOpaqueConst = false;
    2856             :   bool N0IsOpaqueConst = false;
    2857             :   APInt ConstValue0, ConstValue1;
    2858             :   // fold vector ops
    2859       39701 :   if (VT.isVector()) {
    2860        9791 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    2861          12 :       return FoldedVOp;
    2862             : 
    2863        9779 :     N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
    2864        9779 :     N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
    2865             :     assert((!N0IsConst ||
    2866             :             ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&
    2867             :            "Splat APInt should be element width");
    2868             :     assert((!N1IsConst ||
    2869             :             ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&
    2870             :            "Splat APInt should be element width");
    2871             :   } else {
    2872             :     N0IsConst = isa<ConstantSDNode>(N0);
    2873             :     if (N0IsConst) {
    2874          12 :       ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
    2875           6 :       N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
    2876             :     }
    2877             :     N1IsConst = isa<ConstantSDNode>(N1);
    2878             :     if (N1IsConst) {
    2879       35058 :       ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
    2880       17529 :       N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
    2881             :     }
    2882             :   }
    2883             : 
    2884             :   // fold (mul c1, c2) -> c1*c2
    2885       39689 :   if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
    2886           6 :     return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
    2887           6 :                                       N0.getNode(), N1.getNode());
    2888             : 
    2889             :   // canonicalize constant to RHS (vector doesn't have to splat)
    2890       39709 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    2891          26 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    2892          52 :     return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
    2893             :   // fold (mul x, 0) -> 0
    2894       58438 :   if (N1IsConst && ConstValue1.isNullValue())
    2895          25 :     return N1;
    2896             :   // fold (mul x, 1) -> x
    2897       58388 :   if (N1IsConst && ConstValue1.isOneValue())
    2898        2143 :     return N0;
    2899             : 
    2900       37489 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    2901           3 :     return NewSel;
    2902             : 
    2903             :   // fold (mul x, -1) -> 0-x
    2904       54096 :   if (N1IsConst && ConstValue1.isAllOnesValue()) {
    2905             :     SDLoc DL(N);
    2906          97 :     return DAG.getNode(ISD::SUB, DL, VT,
    2907          97 :                        DAG.getConstant(0, DL, VT), N0);
    2908             :   }
    2909             :   // fold (mul x, (1 << c)) -> x << c
    2910       54572 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    2911       41555 :       DAG.isKnownToBeAPowerOfTwo(N1) &&
    2912         286 :       (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
    2913             :     SDLoc DL(N);
    2914        4166 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    2915        8332 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    2916        4166 :     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
    2917        8332 :     return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
    2918             :   }
    2919             :   // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
    2920       82682 :   if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
    2921          85 :     unsigned Log2Val = (-ConstValue1).logBase2();
    2922             :     SDLoc DL(N);
    2923             :     // FIXME: If the input is something that is easily negated (e.g. a
    2924             :     // single-use add), we should put the negate there.
    2925          85 :     return DAG.getNode(ISD::SUB, DL, VT,
    2926          85 :                        DAG.getConstant(0, DL, VT),
    2927             :                        DAG.getNode(ISD::SHL, DL, VT, N0,
    2928             :                             DAG.getConstant(Log2Val, DL,
    2929         170 :                                       getShiftAmountTy(N0.getValueType()))));
    2930             :   }
    2931             : 
    2932             :   // Try to transform multiply-by-(power-of-2 +/- 1) into shift and add/sub.
    2933             :   // mul x, (2^N + 1) --> add (shl x, N), x
    2934             :   // mul x, (2^N - 1) --> sub (shl x, N), x
    2935             :   // Examples: x * 33 --> (x << 5) + x
    2936             :   //           x * 15 --> (x << 4) - x
    2937             :   //           x * -33 --> -((x << 5) + x)
    2938             :   //           x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
    2939       33138 :   if (N1IsConst && TLI.decomposeMulByConstant(VT, N1)) {
    2940             :     // TODO: We could handle more general decomposition of any constant by
    2941             :     //       having the target set a limit on number of ops and making a
    2942             :     //       callback to determine that sequence (similar to sqrt expansion).
    2943             :     unsigned MathOp = ISD::DELETED_NODE;
    2944          66 :     APInt MulC = ConstValue1.abs();
    2945          66 :     if ((MulC - 1).isPowerOf2())
    2946             :       MathOp = ISD::ADD;
    2947          44 :     else if ((MulC + 1).isPowerOf2())
    2948             :       MathOp = ISD::SUB;
    2949             : 
    2950             :     if (MathOp != ISD::DELETED_NODE) {
    2951         154 :       unsigned ShAmt = MathOp == ISD::ADD ? (MulC - 1).logBase2()
    2952         154 :                                           : (MulC + 1).logBase2();
    2953             :       assert(ShAmt > 0 && ShAmt < VT.getScalarSizeInBits() &&
    2954             :              "Not expecting multiply-by-constant that could have simplified");
    2955             :       SDLoc DL(N);
    2956          66 :       SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, N0,
    2957          66 :                                 DAG.getConstant(ShAmt, DL, VT));
    2958         132 :       SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0);
    2959          66 :       if (ConstValue1.isNegative())
    2960          32 :         R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
    2961          66 :       return R;
    2962             :     }
    2963             :   }
    2964             : 
    2965             :   // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
    2966       33094 :   if (N0.getOpcode() == ISD::SHL &&
    2967       33077 :       isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
    2968          10 :       isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
    2969          10 :     SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
    2970           5 :     if (isConstantOrConstantVector(C3))
    2971          15 :       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
    2972             :   }
    2973             : 
    2974             :   // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
    2975             :   // use.
    2976             :   {
    2977             :     SDValue Sh(nullptr, 0), Y(nullptr, 0);
    2978             : 
    2979             :     // Check for both (mul (shl X, C), Y)  and  (mul Y, (shl X, C)).
    2980       33084 :     if (N0.getOpcode() == ISD::SHL &&
    2981       33067 :         isConstantOrConstantVector(N0.getOperand(1)) &&
    2982          15 :         N0.getNode()->hasOneUse()) {
    2983          12 :       Sh = N0; Y = N1;
    2984       33064 :     } else if (N1.getOpcode() == ISD::SHL &&
    2985       33055 :                isConstantOrConstantVector(N1.getOperand(1)) &&
    2986           6 :                N1.getNode()->hasOneUse()) {
    2987           3 :       Sh = N1; Y = N0;
    2988             :     }
    2989             : 
    2990       33067 :     if (Sh.getNode()) {
    2991          16 :       SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
    2992          31 :       return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
    2993             :     }
    2994             :   }
    2995             : 
    2996             :   // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
    2997       46132 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
    2998       13581 :       N0.getOpcode() == ISD::ADD &&
    2999       34245 :       DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
    3000         191 :       isMulAddWithConstProfitable(N, N0, N1))
    3001         184 :       return DAG.getNode(ISD::ADD, SDLoc(N), VT,
    3002          92 :                          DAG.getNode(ISD::MUL, SDLoc(N0), VT,
    3003          92 :                                      N0.getOperand(0), N1),
    3004          92 :                          DAG.getNode(ISD::MUL, SDLoc(N1), VT,
    3005         368 :                                      N0.getOperand(1), N1));
    3006             : 
    3007             :   // reassociate mul
    3008       41538 :   if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags()))
    3009         542 :     return RMUL;
    3010             : 
    3011       32418 :   return SDValue();
    3012             : }
    3013             : 
    3014             : /// Return true if divmod libcall is available.
    3015        1431 : static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
    3016             :                                      const TargetLowering &TLI) {
    3017             :   RTLIB::Libcall LC;
    3018        1431 :   EVT NodeType = Node->getValueType(0);
    3019        1431 :   if (!NodeType.isSimple())
    3020             :     return false;
    3021        1431 :   switch (NodeType.getSimpleVT().SimpleTy) {
    3022             :   default: return false; // No libcall for vector types.
    3023           0 :   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
    3024           4 :   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
    3025        1232 :   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
    3026         188 :   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
    3027           7 :   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
    3028             :   }
    3029             : 
    3030        1431 :   return TLI.getLibcallName(LC) != nullptr;
    3031             : }
    3032             : 
    3033             : /// Issue divrem if both quotient and remainder are needed.
    3034        5726 : SDValue DAGCombiner::useDivRem(SDNode *Node) {
    3035        5726 :   if (Node->use_empty())
    3036           0 :     return SDValue(); // This is a dead node, leave it alone.
    3037             : 
    3038        5726 :   unsigned Opcode = Node->getOpcode();
    3039        5726 :   bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
    3040        5726 :   unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
    3041             : 
    3042             :   // DivMod lib calls can still work on non-legal types if using lib-calls.
    3043       11452 :   EVT VT = Node->getValueType(0);
    3044        5726 :   if (VT.isVector() || !VT.isInteger())
    3045         499 :     return SDValue();
    3046             : 
    3047        6059 :   if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
    3048         801 :     return SDValue();
    3049             : 
    3050             :   // If DIVREM is going to get expanded into a libcall,
    3051             :   // but there is no libcall available, then don't combine.
    3052        4426 :   if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
    3053        1431 :       !isDivRemLibcallAvailable(Node, isSigned, TLI))
    3054        1309 :     return SDValue();
    3055             : 
    3056             :   // If div is legal, it's better to do the normal expansion
    3057             :   unsigned OtherOpcode = 0;
    3058        3117 :   if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
    3059        1634 :     OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
    3060        1634 :     if (TLI.isOperationLegalOrCustom(Opcode, VT))
    3061         211 :       return SDValue();
    3062             :   } else {
    3063        1483 :     OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
    3064        1483 :     if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
    3065         176 :       return SDValue();
    3066             :   }
    3067             : 
    3068        2730 :   SDValue Op0 = Node->getOperand(0);
    3069        2730 :   SDValue Op1 = Node->getOperand(1);
    3070             :   SDValue combined;
    3071        2730 :   for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
    3072        6389 :          UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
    3073             :     SDNode *User = *UI;
    3074        3659 :     if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
    3075         929 :         User->use_empty())
    3076             :       continue;
    3077             :     // Convert the other matching node(s), too;
    3078             :     // otherwise, the DIVREM may get target-legalized into something
    3079             :     // target-specific that we won't be able to recognize.
    3080             :     unsigned UserOpc = User->getOpcode();
    3081         410 :     if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
    3082         948 :         User->getOperand(0) == Op0 &&
    3083         384 :         User->getOperand(1) == Op1) {
    3084         168 :       if (!combined) {
    3085         168 :         if (UserOpc == OtherOpcode) {
    3086         168 :           SDVTList VTs = DAG.getVTList(VT, VT);
    3087         182 :           combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
    3088           0 :         } else if (UserOpc == DivRemOpc) {
    3089             :           combined = SDValue(User, 0);
    3090             :         } else {
    3091             :           assert(UserOpc == Opcode);
    3092             :           continue;
    3093             :         }
    3094             :       }
    3095         168 :       if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
    3096         138 :         CombineTo(User, combined);
    3097          30 :       else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
    3098          30 :         CombineTo(User, combined.getValue(1));
    3099             :     }
    3100             :   }
    3101        2730 :   return combined;
    3102             : }
    3103             : 
    3104       15164 : static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
    3105       15164 :   SDValue N0 = N->getOperand(0);
    3106       15164 :   SDValue N1 = N->getOperand(1);
    3107       30328 :   EVT VT = N->getValueType(0);
    3108             :   SDLoc DL(N);
    3109             : 
    3110       15164 :   unsigned Opc = N->getOpcode();
    3111       15164 :   bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
    3112       15164 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3113             : 
    3114             :   // X / undef -> undef
    3115             :   // X % undef -> undef
    3116             :   // X / 0 -> undef
    3117             :   // X % 0 -> undef
    3118             :   // NOTE: This includes vectors where any divisor element is zero/undef.
    3119       30328 :   if (DAG.isUndef(Opc, {N0, N1}))
    3120           1 :     return DAG.getUNDEF(VT);
    3121             : 
    3122             :   // undef / X -> 0
    3123             :   // undef % X -> 0
    3124       15163 :   if (N0.isUndef())
    3125           0 :     return DAG.getConstant(0, DL, VT);
    3126             : 
    3127             :   // TODO: 0 / X -> 0
    3128             :   // TODO: 0 % X -> 0
    3129             : 
    3130             :   // X / X -> 1
    3131             :   // X % X -> 0
    3132             :   if (N0 == N1)
    3133          62 :     return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
    3134             : 
    3135             :   // X / 1 -> X
    3136             :   // X % 1 -> 0
    3137       24497 :   if (N1C && N1C->isOne())
    3138          48 :     return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
    3139             :   // If this is a boolean op (single-bit element type), we can't have
    3140             :   // division-by-zero or remainder-by-zero, so assume the divisor is 1.
    3141             :   // Similarly, if we're zero-extending a boolean divisor, then assume it's a 1.
    3142             : 
    3143       15071 :   return SDValue();
    3144             : }
    3145             : 
    3146        5883 : SDValue DAGCombiner::visitSDIV(SDNode *N) {
    3147        5883 :   SDValue N0 = N->getOperand(0);
    3148        5883 :   SDValue N1 = N->getOperand(1);
    3149        5883 :   EVT VT = N->getValueType(0);
    3150        5883 :   EVT CCVT = getSetCCResultType(VT);
    3151             : 
    3152             :   // fold vector ops
    3153        5883 :   if (VT.isVector())
    3154         425 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3155           1 :       return FoldedVOp;
    3156             : 
    3157             :   SDLoc DL(N);
    3158             : 
    3159             :   // fold (sdiv c1, c2) -> c1/c2
    3160        5882 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3161        5882 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3162        5882 :   if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
    3163           0 :     return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
    3164             :   // fold (sdiv X, -1) -> 0-X
    3165       10084 :   if (N1C && N1C->isAllOnesValue())
    3166          28 :     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
    3167             :   // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
    3168        5868 :   if (N1C && N1C->getAPIntValue().isMinSignedValue())
    3169          28 :     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
    3170          14 :                          DAG.getConstant(1, DL, VT),
    3171          14 :                          DAG.getConstant(0, DL, VT));
    3172             : 
    3173        5854 :   if (SDValue V = simplifyDivRem(N, DAG))
    3174          29 :     return V;
    3175             : 
    3176        5825 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3177           7 :     return NewSel;
    3178             : 
    3179             :   // If we know the sign bits of both operands are zero, strength reduce to a
    3180             :   // udiv instead.  Handles (X&15) /s 4 -> X&15 >> 2
    3181        5818 :   if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    3182          50 :     return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
    3183             : 
    3184        5793 :   if (SDValue V = visitSDIVLike(N0, N1, N))
    3185        4190 :     return V;
    3186             : 
    3187             :   // sdiv, srem -> sdivrem
    3188             :   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
    3189             :   // true.  Otherwise, we break the simplification logic in visitREM().
    3190        1603 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3191        1603 :   if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    3192        1531 :     if (SDValue DivRem = useDivRem(N))
    3193          14 :         return DivRem;
    3194             : 
    3195        1589 :   return SDValue();
    3196             : }
    3197             : 
    3198        6155 : SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
    3199             :   SDLoc DL(N);
    3200        6155 :   EVT VT = N->getValueType(0);
    3201        6155 :   EVT CCVT = getSetCCResultType(VT);
    3202             :   unsigned BitWidth = VT.getScalarSizeInBits();
    3203             : 
    3204             :   // Helper for determining whether a value is a power-2 constant scalar or a
    3205             :   // vector of such elements.
    3206             :   auto IsPowerOfTwo = [](ConstantSDNode *C) {
    3207             :     if (C->isNullValue() || C->isOpaque())
    3208             :       return false;
    3209             :     if (C->getAPIntValue().isPowerOf2())
    3210             :       return true;
    3211             :     if ((-C->getAPIntValue()).isPowerOf2())
    3212             :       return true;
    3213             :     return false;
    3214             :   };
    3215             : 
    3216             :   // fold (sdiv X, pow2) -> simple ops after legalize
    3217             :   // FIXME: We check for the exact bit here because the generic lowering gives
    3218             :   // better results in that case. The target-specific lowering should learn how
    3219             :   // to handle exact sdivs efficiently.
    3220        8822 :   if (!N->getFlags().hasExact() && ISD::matchUnaryPredicate(N1, IsPowerOfTwo)) {
    3221             :     // Target-specific implementation of sdiv x, pow2.
    3222         595 :     if (SDValue Res = BuildSDIVPow2(N))
    3223          32 :       return Res;
    3224             : 
    3225             :     // Create constants that are functions of the shift amount value.
    3226        1126 :     EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
    3227         563 :     SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
    3228        1126 :     SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
    3229         563 :     C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
    3230        1126 :     SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
    3231         563 :     if (!isConstantOrConstantVector(Inexact))
    3232           0 :       return SDValue();
    3233             : 
    3234             :     // Splat the sign bit into the register
    3235         563 :     SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
    3236         563 :                                DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
    3237         563 :     AddToWorklist(Sign.getNode());
    3238             : 
    3239             :     // Add (N0 < 0) ? abs2 - 1 : 0;
    3240        1126 :     SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
    3241         563 :     AddToWorklist(Srl.getNode());
    3242        1126 :     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
    3243         563 :     AddToWorklist(Add.getNode());
    3244        1126 :     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
    3245         563 :     AddToWorklist(Sra.getNode());
    3246             : 
    3247             :     // Special case: (sdiv X, 1) -> X
    3248             :     // Special Case: (sdiv X, -1) -> 0-X
    3249         563 :     SDValue One = DAG.getConstant(1, DL, VT);
    3250         563 :     SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
    3251         563 :     SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
    3252         563 :     SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
    3253        1126 :     SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
    3254         563 :     Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
    3255             : 
    3256             :     // If dividing by a positive value, we're done. Otherwise, the result must
    3257             :     // be negated.
    3258         563 :     SDValue Zero = DAG.getConstant(0, DL, VT);
    3259        1126 :     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
    3260             : 
    3261             :     // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
    3262         563 :     SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
    3263         563 :     SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
    3264         563 :     return Res;
    3265             :   }
    3266             : 
    3267             :   // If integer divide is expensive and we satisfy the requirements, emit an
    3268             :   // alternate sequence.  Targets may check function attributes for size/speed
    3269             :   // trade-offs.
    3270        5560 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3271        9629 :   if (isConstantOrConstantVector(N1) &&
    3272        8138 :       !TLI.isIntDivCheap(N->getValueType(0), Attr))
    3273        4025 :     if (SDValue Op = BuildSDIV(N))
    3274        3895 :       return Op;
    3275             : 
    3276        1665 :   return SDValue();
    3277             : }
    3278             : 
    3279        4531 : SDValue DAGCombiner::visitUDIV(SDNode *N) {
    3280        4531 :   SDValue N0 = N->getOperand(0);
    3281        4531 :   SDValue N1 = N->getOperand(1);
    3282        4531 :   EVT VT = N->getValueType(0);
    3283        4531 :   EVT CCVT = getSetCCResultType(VT);
    3284             : 
    3285             :   // fold vector ops
    3286        4531 :   if (VT.isVector())
    3287         268 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3288           0 :       return FoldedVOp;
    3289             : 
    3290             :   SDLoc DL(N);
    3291             : 
    3292             :   // fold (udiv c1, c2) -> c1/c2
    3293        4531 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3294        4531 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3295        4531 :   if (N0C && N1C)
    3296           0 :     if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
    3297           0 :                                                     N0C, N1C))
    3298           0 :       return Folded;
    3299             :   // fold (udiv X, -1) -> select(X == -1, 1, 0)
    3300        7414 :   if (N1C && N1C->getAPIntValue().isAllOnesValue())
    3301          22 :     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
    3302          11 :                          DAG.getConstant(1, DL, VT),
    3303          11 :                          DAG.getConstant(0, DL, VT));
    3304             : 
    3305        4520 :   if (SDValue V = simplifyDivRem(N, DAG))
    3306          21 :     return V;
    3307             : 
    3308        4499 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3309           7 :     return NewSel;
    3310             : 
    3311        4492 :   if (SDValue V = visitUDIVLike(N0, N1, N))
    3312        2805 :     return V;
    3313             : 
    3314             :   // sdiv, srem -> sdivrem
    3315             :   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
    3316             :   // true.  Otherwise, we break the simplification logic in visitREM().
    3317        1687 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3318        1687 :   if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    3319        1623 :     if (SDValue DivRem = useDivRem(N))
    3320          16 :         return DivRem;
    3321             : 
    3322        1671 :   return SDValue();
    3323             : }
    3324             : 
    3325        5033 : SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
    3326             :   SDLoc DL(N);
    3327        5033 :   EVT VT = N->getValueType(0);
    3328             : 
    3329             :   // fold (udiv x, (1 << c)) -> x >>u c
    3330        8471 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    3331        3438 :       DAG.isKnownToBeAPowerOfTwo(N1)) {
    3332        2438 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    3333        2438 :     AddToWorklist(LogBase2.getNode());
    3334             : 
    3335        4876 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    3336        2438 :     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
    3337        2438 :     AddToWorklist(Trunc.getNode());
    3338        4876 :     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    3339             :   }
    3340             : 
    3341             :   // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
    3342        2595 :   if (N1.getOpcode() == ISD::SHL) {
    3343          15 :     SDValue N10 = N1.getOperand(0);
    3344          30 :     if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
    3345          15 :         DAG.isKnownToBeAPowerOfTwo(N10)) {
    3346          15 :       SDValue LogBase2 = BuildLogBase2(N10, DL);
    3347          15 :       AddToWorklist(LogBase2.getNode());
    3348             : 
    3349          15 :       EVT ADDVT = N1.getOperand(1).getValueType();
    3350          15 :       SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
    3351          15 :       AddToWorklist(Trunc.getNode());
    3352          30 :       SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
    3353          15 :       AddToWorklist(Add.getNode());
    3354          30 :       return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
    3355             :     }
    3356             :   }
    3357             : 
    3358             :   // fold (udiv x, c) -> alternate
    3359        2580 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3360        3580 :   if (isConstantOrConstantVector(N1) &&
    3361        2000 :       !TLI.isIntDivCheap(N->getValueType(0), Attr))
    3362         957 :     if (SDValue Op = BuildUDIV(N))
    3363         812 :       return Op;
    3364             : 
    3365        1768 :   return SDValue();
    3366             : }
    3367             : 
    3368             : // handles ISD::SREM and ISD::UREM
    3369        4801 : SDValue DAGCombiner::visitREM(SDNode *N) {
    3370        4801 :   unsigned Opcode = N->getOpcode();
    3371        4801 :   SDValue N0 = N->getOperand(0);
    3372        4801 :   SDValue N1 = N->getOperand(1);
    3373        4801 :   EVT VT = N->getValueType(0);
    3374        4801 :   EVT CCVT = getSetCCResultType(VT);
    3375             : 
    3376             :   bool isSigned = (Opcode == ISD::SREM);
    3377             :   SDLoc DL(N);
    3378             : 
    3379             :   // fold (rem c1, c2) -> c1%c2
    3380        4801 :   ConstantSDNode *N0C = isConstOrConstSplat(N0);
    3381        4801 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    3382        4801 :   if (N0C && N1C)
    3383           2 :     if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
    3384           2 :       return Folded;
    3385             :   // fold (urem X, -1) -> select(X == -1, 0, x)
    3386        6732 :   if (!isSigned && N1C && N1C->getAPIntValue().isAllOnesValue())
    3387          18 :     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
    3388           9 :                          DAG.getConstant(0, DL, VT), N0);
    3389             : 
    3390        4790 :   if (SDValue V = simplifyDivRem(N, DAG))
    3391          43 :     return V;
    3392             : 
    3393        4747 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    3394          14 :     return NewSel;
    3395             : 
    3396        4733 :   if (isSigned) {
    3397             :     // If we know the sign bits of both operands are zero, strength reduce to a
    3398             :     // urem instead.  Handles (X & 0x0FFFFFFF) %s 16 -> X&15
    3399        1577 :     if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    3400          40 :       return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
    3401             :   } else {
    3402        3156 :     SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
    3403        3156 :     if (DAG.isKnownToBeAPowerOfTwo(N1)) {
    3404             :       // fold (urem x, pow2) -> (and x, pow2-1)
    3405        2750 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
    3406        1375 :       AddToWorklist(Add.getNode());
    3407        2750 :       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    3408             :     }
    3409        1789 :     if (N1.getOpcode() == ISD::SHL &&
    3410          16 :         DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
    3411             :       // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
    3412          12 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
    3413           6 :       AddToWorklist(Add.getNode());
    3414          12 :       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    3415             :     }
    3416             :   }
    3417             : 
    3418        3332 :   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    3419             : 
    3420             :   // If X/C can be simplified by the division-by-constant logic, lower
    3421             :   // X%C to the equivalent of X-X/C*C.
    3422             :   // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
    3423             :   // speculative DIV must not cause a DIVREM conversion.  We guard against this
    3424             :   // by skipping the simplification if isIntDivCheap().  When div is not cheap,
    3425             :   // combine will not return a DIVREM.  Regardless, checking cheapness here
    3426             :   // makes sense since the simplification results in fatter code.
    3427        3332 :   if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
    3428             :     SDValue OptimizedDiv =
    3429         903 :         isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
    3430         903 :     if (OptimizedDiv.getNode()) {
    3431        1520 :       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
    3432        1520 :       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
    3433         760 :       AddToWorklist(OptimizedDiv.getNode());
    3434         760 :       AddToWorklist(Mul.getNode());
    3435         760 :       return Sub;
    3436             :     }
    3437             :   }
    3438             : 
    3439             :   // sdiv, srem -> sdivrem
    3440        2572 :   if (SDValue DivRem = useDivRem(N))
    3441         138 :     return DivRem.getValue(1);
    3442             : 
    3443        2434 :   return SDValue();
    3444             : }
    3445             : 
    3446        1513 : SDValue DAGCombiner::visitMULHS(SDNode *N) {
    3447        1513 :   SDValue N0 = N->getOperand(0);
    3448        1513 :   SDValue N1 = N->getOperand(1);
    3449        3026 :   EVT VT = N->getValueType(0);
    3450             :   SDLoc DL(N);
    3451             : 
    3452        1513 :   if (VT.isVector()) {
    3453             :     // fold (mulhs x, 0) -> 0
    3454         808 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    3455           0 :       return N1;
    3456         808 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    3457           2 :       return N0;
    3458             :   }
    3459             : 
    3460             :   // fold (mulhs x, 0) -> 0
    3461        1511 :   if (isNullConstant(N1))
    3462           0 :     return N1;
    3463             :   // fold (mulhs x, 1) -> (sra x, size(x)-1)
    3464        1511 :   if (isOneConstant(N1))
    3465           0 :     return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
    3466           0 :                        DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
    3467           0 :                                        getShiftAmountTy(N0.getValueType())));
    3468             : 
    3469             :   // fold (mulhs x, undef) -> 0
    3470        3022 :   if (N0.isUndef() || N1.isUndef())
    3471           0 :     return DAG.getConstant(0, DL, VT);
    3472             : 
    3473             :   // If the type twice as wide is legal, transform the mulhs to a wider multiply
    3474             :   // plus a shift.
    3475        1511 :   if (VT.isSimple() && !VT.isVector()) {
    3476         705 :     MVT Simple = VT.getSimpleVT();
    3477         705 :     unsigned SimpleSize = Simple.getSizeInBits();
    3478         705 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3479         705 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3480         138 :       N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
    3481         138 :       N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
    3482         138 :       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
    3483          69 :       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
    3484             :             DAG.getConstant(SimpleSize, DL,
    3485          69 :                             getShiftAmountTy(N1.getValueType())));
    3486         138 :       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    3487             :     }
    3488             :   }
    3489             : 
    3490        1442 :   return SDValue();
    3491             : }
    3492             : 
    3493        6357 : SDValue DAGCombiner::visitMULHU(SDNode *N) {
    3494        6357 :   SDValue N0 = N->getOperand(0);
    3495        6357 :   SDValue N1 = N->getOperand(1);
    3496       12714 :   EVT VT = N->getValueType(0);
    3497             :   SDLoc DL(N);
    3498             : 
    3499        6357 :   if (VT.isVector()) {
    3500             :     // fold (mulhu x, 0) -> 0
    3501        1177 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    3502           0 :       return N1;
    3503        1177 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    3504           0 :       return N0;
    3505             :   }
    3506             : 
    3507             :   // fold (mulhu x, 0) -> 0
    3508        6357 :   if (isNullConstant(N1))
    3509           0 :     return N1;
    3510             :   // fold (mulhu x, 1) -> 0
    3511        6357 :   if (isOneConstant(N1))
    3512           0 :     return DAG.getConstant(0, DL, N0.getValueType());
    3513             :   // fold (mulhu x, undef) -> 0
    3514        6357 :   if (N0.isUndef() || N1.isUndef())
    3515           0 :     return DAG.getConstant(0, DL, VT);
    3516             : 
    3517             :   // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
    3518        8279 :   if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
    3519        6495 :       DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
    3520             :     SDLoc DL(N);
    3521             :     unsigned NumEltBits = VT.getScalarSizeInBits();
    3522          79 :     SDValue LogBase2 = BuildLogBase2(N1, DL);
    3523          79 :     SDValue SRLAmt = DAG.getNode(
    3524          79 :         ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
    3525          79 :     EVT ShiftVT = getShiftAmountTy(N0.getValueType());
    3526          79 :     SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
    3527         158 :     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    3528             :   }
    3529             : 
    3530             :   // If the type twice as wide is legal, transform the mulhu to a wider multiply
    3531             :   // plus a shift.
    3532        6278 :   if (VT.isSimple() && !VT.isVector()) {
    3533        5180 :     MVT Simple = VT.getSimpleVT();
    3534        5180 :     unsigned SimpleSize = Simple.getSizeInBits();
    3535        5180 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3536        5180 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3537         484 :       N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
    3538         484 :       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
    3539         484 :       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
    3540         242 :       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
    3541             :             DAG.getConstant(SimpleSize, DL,
    3542         242 :                             getShiftAmountTy(N1.getValueType())));
    3543         484 :       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    3544             :     }
    3545             :   }
    3546             : 
    3547        6036 :   return SDValue();
    3548             : }
    3549             : 
    3550             : /// Perform optimizations common to nodes that compute two values. LoOp and HiOp
    3551             : /// give the opcodes for the two computations that are being performed. Return
    3552             : /// true if a simplification was made.
    3553        6336 : SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
    3554             :                                                 unsigned HiOp) {
    3555             :   // If the high half is not needed, just compute the low half.
    3556        6336 :   bool HiExists = N->hasAnyUseOfValue(1);
    3557        6336 :   if (!HiExists &&
    3558          36 :       (!LegalOperations ||
    3559          20 :        TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
    3560          48 :     SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    3561             :     return CombineTo(N, Res, Res);
    3562             :   }
    3563             : 
    3564             :   // If the low half is not needed, just compute the high half.
    3565        6320 :   bool LoExists = N->hasAnyUseOfValue(0);
    3566        6320 :   if (!LoExists &&
    3567        1411 :       (!LegalOperations ||
    3568         716 :        TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
    3569        2276 :     SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    3570             :     return CombineTo(N, Res, Res);
    3571             :   }
    3572             : 
    3573             :   // If both halves are used, return as it is.
    3574        5625 :   if (LoExists && HiExists)
    3575        4899 :     return SDValue();
    3576             : 
    3577             :   // If the two computed results can be simplified separately, separate them.
    3578         726 :   if (LoExists) {
    3579          20 :     SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    3580          10 :     AddToWorklist(Lo.getNode());
    3581          10 :     SDValue LoOpt = combine(Lo.getNode());
    3582          10 :     if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
    3583           0 :         (!LegalOperations ||
    3584           0 :          TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
    3585           0 :       return CombineTo(N, LoOpt, LoOpt);
    3586             :   }
    3587             : 
    3588         726 :   if (HiExists) {
    3589        1580 :     SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    3590         716 :     AddToWorklist(Hi.getNode());
    3591         716 :     SDValue HiOpt = combine(Hi.getNode());
    3592         716 :     if (HiOpt.getNode() && HiOpt != Hi &&
    3593           0 :         (!LegalOperations ||
    3594           0 :          TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
    3595           0 :       return CombineTo(N, HiOpt, HiOpt);
    3596             :   }
    3597             : 
    3598         726 :   return SDValue();
    3599             : }
    3600             : 
    3601         723 : SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
    3602         723 :   if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
    3603         179 :     return Res;
    3604             : 
    3605        1088 :   EVT VT = N->getValueType(0);
    3606             :   SDLoc DL(N);
    3607             : 
    3608             :   // If the type is twice as wide is legal, transform the mulhu to a wider
    3609             :   // multiply plus a shift.
    3610         544 :   if (VT.isSimple() && !VT.isVector()) {
    3611         544 :     MVT Simple = VT.getSimpleVT();
    3612         544 :     unsigned SimpleSize = Simple.getSizeInBits();
    3613         544 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3614         544 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3615           0 :       SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
    3616           0 :       SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
    3617           0 :       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
    3618             :       // Compute the high part as N1.
    3619           0 :       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
    3620             :             DAG.getConstant(SimpleSize, DL,
    3621           0 :                             getShiftAmountTy(Lo.getValueType())));
    3622           0 :       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
    3623             :       // Compute the low part as N0.
    3624           0 :       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
    3625             :       return CombineTo(N, Lo, Hi);
    3626             :     }
    3627             :   }
    3628             : 
    3629         544 :   return SDValue();
    3630             : }
    3631             : 
    3632        5613 : SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
    3633        5613 :   if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
    3634         532 :     return Res;
    3635             : 
    3636       10162 :   EVT VT = N->getValueType(0);
    3637             :   SDLoc DL(N);
    3638             : 
    3639             :   // If the type is twice as wide is legal, transform the mulhu to a wider
    3640             :   // multiply plus a shift.
    3641        5081 :   if (VT.isSimple() && !VT.isVector()) {
    3642        5081 :     MVT Simple = VT.getSimpleVT();
    3643        5081 :     unsigned SimpleSize = Simple.getSizeInBits();
    3644        5081 :     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    3645        5081 :     if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
    3646           0 :       SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
    3647           0 :       SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
    3648           0 :       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
    3649             :       // Compute the high part as N1.
    3650           0 :       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
    3651             :             DAG.getConstant(SimpleSize, DL,
    3652           0 :                             getShiftAmountTy(Lo.getValueType())));
    3653           0 :       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
    3654             :       // Compute the low part as N0.
    3655           0 :       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
    3656             :       return CombineTo(N, Lo, Hi);
    3657             :     }
    3658             :   }
    3659             : 
    3660        5081 :   return SDValue();
    3661             : }
    3662             : 
    3663           0 : SDValue DAGCombiner::visitSMULO(SDNode *N) {
    3664             :   // (smulo x, 2) -> (saddo x, x)
    3665           0 :   if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    3666           0 :     if (C2->getAPIntValue() == 2)
    3667           0 :       return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
    3668           0 :                          N->getOperand(0), N->getOperand(0));
    3669             : 
    3670           0 :   return SDValue();
    3671             : }
    3672             : 
    3673           0 : SDValue DAGCombiner::visitUMULO(SDNode *N) {
    3674             :   // (umulo x, 2) -> (uaddo x, x)
    3675           0 :   if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
    3676           0 :     if (C2->getAPIntValue() == 2)
    3677           0 :       return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
    3678           0 :                          N->getOperand(0), N->getOperand(0));
    3679             : 
    3680           0 :   return SDValue();
    3681             : }
    3682             : 
    3683       25437 : SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
    3684       25437 :   SDValue N0 = N->getOperand(0);
    3685       25437 :   SDValue N1 = N->getOperand(1);
    3686       25437 :   EVT VT = N0.getValueType();
    3687             : 
    3688             :   // fold vector ops
    3689       25437 :   if (VT.isVector())
    3690       20293 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    3691         392 :       return FoldedVOp;
    3692             : 
    3693             :   // fold operation with constant operands.
    3694             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    3695             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    3696       25045 :   if (N0C && N1C)
    3697           0 :     return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
    3698             : 
    3699             :   // canonicalize constant to RHS
    3700       25055 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    3701          10 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    3702          20 :     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
    3703             : 
    3704             :   // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
    3705             :   // Only do this if the current op isn't legal and the flipped is.
    3706       25035 :   unsigned Opcode = N->getOpcode();
    3707       25035 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    3708        5455 :   if (!TLI.isOperationLegal(Opcode, VT) &&
    3709        5455 :       (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
    3710          52 :       (N1.isUndef() || DAG.SignBitIsZero(N1))) {
    3711             :     unsigned AltOpcode;
    3712             :     switch (Opcode) {
    3713             :     case ISD::SMIN: AltOpcode = ISD::UMIN; break;
    3714             :     case ISD::SMAX: AltOpcode = ISD::UMAX; break;
    3715             :     case ISD::UMIN: AltOpcode = ISD::SMIN; break;
    3716             :     case ISD::UMAX: AltOpcode = ISD::SMAX; break;
    3717           0 :     default: llvm_unreachable("Unknown MINMAX opcode");
    3718             :     }
    3719             :     if (TLI.isOperationLegal(AltOpcode, VT))
    3720          10 :       return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
    3721             :   }
    3722             : 
    3723       25030 :   return SDValue();
    3724             : }
    3725             : 
    3726             : /// If this is a binary operator with two operands of the same opcode, try to
    3727             : /// simplify it.
    3728       66791 : SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
    3729       66791 :   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
    3730       66791 :   EVT VT = N0.getValueType();
    3731             :   assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
    3732             : 
    3733             :   // Bail early if none of these transforms apply.
    3734       66791 :   if (N0.getNumOperands() == 0) return SDValue();
    3735             : 
    3736             :   // For each of OP in AND/OR/XOR:
    3737             :   // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
    3738             :   // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
    3739             :   // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
    3740             :   // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
    3741             :   // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
    3742             :   //
    3743             :   // do not sink logical op inside of a vector extend, since it may combine
    3744             :   // into a vsetcc.
    3745      133546 :   EVT Op0VT = N0.getOperand(0).getValueType();
    3746       66611 :   if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
    3747       66530 :        N0.getOpcode() == ISD::SIGN_EXTEND ||
    3748       66530 :        N0.getOpcode() == ISD::BSWAP ||
    3749             :        // Avoid infinite looping with PromoteIntBinOp.
    3750        1255 :        (N0.getOpcode() == ISD::ANY_EXTEND &&
    3751       66530 :         (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
    3752        1093 :        (N0.getOpcode() == ISD::TRUNCATE &&
    3753        1374 :         (!TLI.isZExtFree(VT, Op0VT) ||
    3754         281 :          !TLI.isTruncateFree(Op0VT, VT)) &&
    3755        1859 :         TLI.isTypeLegal(Op0VT))) &&
    3756           9 :       !VT.isVector() &&
    3757       67629 :       Op0VT == N1.getOperand(0).getValueType() &&
    3758         834 :       (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
    3759         834 :     SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
    3760             :                                  N0.getOperand(0).getValueType(),
    3761         852 :                                  N0.getOperand(0), N1.getOperand(0));
    3762         834 :     AddToWorklist(ORNode.getNode());
    3763        1681 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
    3764             :   }
    3765             : 
    3766             :   // For each of OP in SHL/SRL/SRA/AND...
    3767             :   //   fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
    3768             :   //   fold (or  (OP x, z), (OP y, z)) -> (OP (or  x, y), z)
    3769             :   //   fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
    3770       65690 :   if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
    3771      131344 :        N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
    3772        8717 :       N0.getOperand(1) == N1.getOperand(1)) {
    3773          99 :     SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
    3774             :                                  N0.getOperand(0).getValueType(),
    3775         103 :                                  N0.getOperand(0), N1.getOperand(0));
    3776          99 :     AddToWorklist(ORNode.getNode());
    3777          99 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
    3778         198 :                        ORNode, N0.getOperand(1));
    3779             :   }
    3780             : 
    3781             :   // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
    3782             :   // Only perform this optimization up until type legalization, before
    3783             :   // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
    3784             :   // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
    3785             :   // we don't want to undo this promotion.
    3786             :   // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
    3787             :   // on scalars.
    3788       37630 :   if ((N0.getOpcode() == ISD::BITCAST ||
    3789       65840 :        N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
    3790       28215 :        Level <= AfterLegalizeTypes) {
    3791         697 :     SDValue In0 = N0.getOperand(0);
    3792         697 :     SDValue In1 = N1.getOperand(0);
    3793         697 :     EVT In0Ty = In0.getValueType();
    3794         697 :     EVT In1Ty = In1.getValueType();
    3795             :     SDLoc DL(N);
    3796             :     // If both incoming values are integers, and the original types are the
    3797             :     // same.
    3798         727 :     if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
    3799         354 :       SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
    3800         236 :       SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
    3801         118 :       AddToWorklist(Op.getNode());
    3802         118 :       return BC;
    3803             :     }
    3804             :   }
    3805             : 
    3806             :   // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
    3807             :   // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
    3808             :   // If both shuffles use the same mask, and both shuffle within a single
    3809             :   // vector, then it is worthwhile to move the swizzle after the operation.
    3810             :   // The type-legalizer generates this pattern when loading illegal
    3811             :   // vector types from memory. In many cases this allows additional shuffle
    3812             :   // optimizations.
    3813             :   // There are other cases where moving the shuffle after the xor/and/or
    3814             :   // is profitable even if shuffles don't perform a swizzle.
    3815             :   // If both shuffles use the same mask, and both shuffles have the same first
    3816             :   // or second operand, then it might still be profitable to move the shuffle
    3817             :   // after the xor/and/or operation.
    3818       65722 :   if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
    3819             :     ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
    3820             :     ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
    3821             : 
    3822             :     assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
    3823             :            "Inputs to shuffles are not the same type");
    3824             : 
    3825             :     // Check that both shuffles use the same mask. The masks are known to be of
    3826             :     // the same length because the result vector type is the same.
    3827             :     // Check also that shuffles have only one use to avoid introducing extra
    3828             :     // instructions.
    3829        1065 :     if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
    3830        1124 :         SVN0->getMask().equals(SVN1->getMask())) {
    3831         166 :       SDValue ShOp = N0->getOperand(1);
    3832             : 
    3833             :       // Don't try to fold this node if it requires introducing a
    3834             :       // build vector of all zeros that might be illegal at this stage.
    3835         166 :       if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
    3836          26 :         if (!LegalTypes)
    3837          24 :           ShOp = DAG.getConstant(0, SDLoc(N), VT);
    3838             :         else
    3839           2 :           ShOp = SDValue();
    3840             :       }
    3841             : 
    3842             :       // (AND (shuf (A, C), shuf (B, C))) -> shuf (AND (A, B), C)
    3843             :       // (OR  (shuf (A, C), shuf (B, C))) -> shuf (OR  (A, B), C)
    3844             :       // (XOR (shuf (A, C), shuf (B, C))) -> shuf (XOR (A, B), V_0)
    3845         240 :       if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
    3846          74 :         SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
    3847         148 :                                       N0->getOperand(0), N1->getOperand(0));
    3848          74 :         AddToWorklist(NewNode.getNode());
    3849          74 :         return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
    3850         148 :                                     SVN0->getMask());
    3851             :       }
    3852             : 
    3853             :       // Don't try to fold this node if it requires introducing a
    3854             :       // build vector of all zeros that might be illegal at this stage.
    3855          92 :       ShOp = N0->getOperand(0);
    3856          92 :       if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
    3857          14 :         if (!LegalTypes)
    3858          12 :           ShOp = DAG.getConstant(0, SDLoc(N), VT);
    3859             :         else
    3860           2 :           ShOp = SDValue();
    3861             :       }
    3862             : 
    3863             :       // (AND (shuf (C, A), shuf (C, B))) -> shuf (C, AND (A, B))
    3864             :       // (OR  (shuf (C, A), shuf (C, B))) -> shuf (C, OR  (A, B))
    3865             :       // (XOR (shuf (C, A), shuf (C, B))) -> shuf (V_0, XOR (A, B))
    3866         128 :       if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
    3867          36 :         SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
    3868          72 :                                       N0->getOperand(1), N1->getOperand(1));
    3869          36 :         AddToWorklist(NewNode.getNode());
    3870          36 :         return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
    3871          72 :                                     SVN0->getMask());
    3872             :       }
    3873             :     }
    3874             :   }
    3875             : 
    3876       65612 :   return SDValue();
    3877             : }
    3878             : 
    3879             : /// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
    3880      521342 : SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
    3881             :                                        const SDLoc &DL) {
    3882      521342 :   SDValue LL, LR, RL, RR, N0CC, N1CC;
    3883      609702 :   if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
    3884       88360 :       !isSetCCEquivalent(N1, RL, RR, N1CC))
    3885      518381 :     return SDValue();
    3886             : 
    3887             :   assert(N0.getValueType() == N1.getValueType() &&
    3888             :          "Unexpected operand types for bitwise logic op");
    3889             :   assert(LL.getValueType() == LR.getValueType() &&
    3890             :          RL.getValueType() == RR.getValueType() &&
    3891             :          "Unexpected operand types for setcc");
    3892             : 
    3893             :   // If we're here post-legalization or the logic op type is not i1, the logic
    3894             :   // op type must match a setcc result type. Also, all folds require new
    3895             :   // operations on the left and right operands, so those types must match.
    3896        2961 :   EVT VT = N0.getValueType();
    3897        2961 :   EVT OpVT = LL.getValueType();
    3898        2961 :   if (LegalOperations || VT.getScalarType() != MVT::i1)
    3899        1500 :     if (VT != getSetCCResultType(OpVT))
    3900          24 :       return SDValue();
    3901        5882 :   if (OpVT != RL.getValueType())
    3902         109 :     return SDValue();
    3903             : 
    3904        2828 :   ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
    3905        2828 :   ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
    3906        2828 :   bool IsInteger = OpVT.isInteger();
    3907        3337 :   if (LR == RR && CC0 == CC1 && IsInteger) {
    3908         271 :     bool IsZero = isNullConstantOrNullSplatConstant(LR);
    3909         271 :     bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
    3910             : 
    3911             :     // All bits clear?
    3912         271 :     bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
    3913             :     // All sign bits clear?
    3914         271 :     bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
    3915             :     // Any bits set?
    3916         271 :     bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
    3917             :     // Any sign bits set?
    3918         271 :     bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
    3919             : 
    3920             :     // (and (seteq X,  0), (seteq Y,  0)) --> (seteq (or X, Y),  0)
    3921             :     // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
    3922             :     // (or  (setne X,  0), (setne Y,  0)) --> (setne (or X, Y),  0)
    3923             :     // (or  (setlt X,  0), (setlt Y,  0)) --> (setlt (or X, Y),  0)
    3924         271 :     if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
    3925          72 :       SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
    3926          69 :       AddToWorklist(Or.getNode());
    3927          69 :       return DAG.getSetCC(DL, VT, Or, LR, CC1);
    3928             :     }
    3929             : 
    3930             :     // All bits set?
    3931         202 :     bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
    3932             :     // All sign bits set?
    3933         202 :     bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
    3934             :     // Any bits clear?
    3935         202 :     bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
    3936             :     // Any sign bits clear?
    3937         202 :     bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
    3938             : 
    3939             :     // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
    3940             :     // (and (setlt X,  0), (setlt Y,  0)) --> (setlt (and X, Y),  0)
    3941             :     // (or  (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
    3942             :     // (or  (setgt X, -1), (setgt Y  -1)) --> (setgt (and X, Y), -1)
    3943         202 :     if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
    3944          22 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
    3945          22 :       AddToWorklist(And.getNode());
    3946          22 :       return DAG.getSetCC(DL, VT, And, LR, CC1);
    3947             :     }
    3948             :   }
    3949             : 
    3950             :   // TODO: What is the 'or' equivalent of this fold?
    3951             :   // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
    3952        2115 :   if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
    3953        2824 :       IsInteger && CC0 == ISD::SETNE &&
    3954          70 :       ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
    3955          38 :        (isAllOnesConstant(LR) && isNullConstant(RR)))) {
    3956           3 :     SDValue One = DAG.getConstant(1, DL, OpVT);
    3957           3 :     SDValue Two = DAG.getConstant(2, DL, OpVT);
    3958           3 :     SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
    3959           3 :     AddToWorklist(Add.getNode());
    3960           3 :     return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
    3961             :   }
    3962             : 
    3963             :   // Try more general transforms if the predicates match and the only user of
    3964             :   // the compares is the 'and' or 'or'.
    3965        2252 :   if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
    3966        2966 :       N0.hasOneUse() && N1.hasOneUse()) {
    3967             :     // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
    3968             :     // or  (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
    3969         232 :     if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
    3970          81 :       SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
    3971          81 :       SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
    3972         158 :       SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
    3973          79 :       SDValue Zero = DAG.getConstant(0, DL, OpVT);
    3974          79 :       return DAG.getSetCC(DL, VT, Or, Zero, CC1);
    3975             :     }
    3976             :   }
    3977             : 
    3978             :   // Canonicalize equivalent operands to LL == RL.
    3979        2695 :   if (LL == RR && LR == RL) {
    3980           0 :     CC1 = ISD::getSetCCSwappedOperands(CC1);
    3981             :     std::swap(RL, RR);
    3982             :   }
    3983             : 
    3984             :   // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
    3985             :   // (or  (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
    3986        3292 :   if (LL == RL && LR == RR) {
    3987          86 :     ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
    3988          43 :                                 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
    3989          86 :     if (NewCC != ISD::SETCC_INVALID &&
    3990          71 :         (!LegalOperations ||
    3991          39 :          (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
    3992             :           TLI.isOperationLegal(ISD::SETCC, OpVT))))
    3993          32 :       return DAG.getSetCC(DL, VT, LL, LR, NewCC);
    3994             :   }
    3995             : 
    3996        2623 :   return SDValue();
    3997             : }
    3998             : 
    3999             : /// This contains all DAGCombine rules which reduce two values combined by
    4000             : /// an And operation to a single value. This makes them reusable in the context
    4001             : /// of visitSELECT(). Rules involving constants are not included as
    4002             : /// visitSELECT() already handles those cases.
    4003      377699 : SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
    4004      755398 :   EVT VT = N1.getValueType();
    4005             :   SDLoc DL(N);
    4006             : 
    4007             :   // fold (and x, undef) -> 0
    4008      377699 :   if (N0.isUndef() || N1.isUndef())
    4009           2 :     return DAG.getConstant(0, DL, VT);
    4010             : 
    4011      377697 :   if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
    4012          93 :     return V;
    4013             : 
    4014      377604 :   if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
    4015         280 :       VT.getSizeInBits() <= 64) {
    4016             :     if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    4017             :       if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
    4018             :         // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
    4019             :         // immediate for an add, but it is legal if its top c2 bits are set,
    4020             :         // transform the ADD so the immediate doesn't need to be materialized
    4021             :         // in a register.
    4022          62 :         APInt ADDC = ADDI->getAPIntValue();
    4023          62 :         APInt SRLC = SRLI->getAPIntValue();
    4024         124 :         if (ADDC.getMinSignedBits() <= 64 &&
    4025         123 :             SRLC.ult(VT.getSizeInBits()) &&
    4026         122 :             !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
    4027             :           APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
    4028          13 :                                              SRLC.getZExtValue());
    4029          26 :           if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
    4030             :             ADDC |= Mask;
    4031          26 :             if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
    4032             :               SDLoc DL0(N0);
    4033             :               SDValue NewAdd =
    4034          13 :                 DAG.getNode(ISD::ADD, DL0, VT,
    4035          13 :                             N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
    4036          13 :               CombineTo(N0.getNode(), NewAdd);
    4037             :               // Return N so it doesn't get rechecked!
    4038          13 :               return SDValue(N, 0);
    4039             :             }
    4040             :           }
    4041             :         }
    4042             :       }
    4043             :     }
    4044             :   }
    4045             : 
    4046             :   // Reduce bit extract of low half of an integer to the narrower type.
    4047             :   // (and (srl i64:x, K), KMask) ->
    4048             :   //   (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
    4049      405727 :   if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    4050             :     if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
    4051             :       if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    4052       19572 :         unsigned Size = VT.getSizeInBits();
    4053       19572 :         const APInt &AndMask = CAnd->getAPIntValue();
    4054       19572 :         unsigned ShiftBits = CShift->getZExtValue();
    4055             : 
    4056             :         // Bail out, this node will probably disappear anyway.
    4057       19572 :         if (ShiftBits == 0)
    4058           2 :           return SDValue();
    4059             : 
    4060             :         unsigned MaskBits = AndMask.countTrailingOnes();
    4061       19570 :         EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
    4062             : 
    4063       36135 :         if (AndMask.isMask() &&
    4064             :             // Required bits must not span the two halves of the integer and
    4065             :             // must fit in the half size type.
    4066       25745 :             (ShiftBits + MaskBits <= Size / 2) &&
    4067        9303 :             TLI.isNarrowingProfitable(VT, HalfVT) &&
    4068         246 :             TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
    4069         246 :             TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
    4070       19816 :             TLI.isTruncateFree(VT, HalfVT) &&
    4071         123 :             TLI.isZExtFree(HalfVT, VT)) {
    4072             :           // The isNarrowingProfitable is to avoid regressions on PPC and
    4073             :           // AArch64 which match a few 64-bit bit insert / bit extract patterns
    4074             :           // on downstream users of this. Those patterns could probably be
    4075             :           // extended to handle extensions mixed in.
    4076             : 
    4077             :           SDValue SL(N0);
    4078             :           assert(MaskBits <= Size);
    4079             : 
    4080             :           // Extracting the highest bit of the low half.
    4081         116 :           EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
    4082         116 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
    4083         116 :                                       N0.getOperand(0));
    4084             : 
    4085         278 :           SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
    4086         162 :           SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
    4087         162 :           SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
    4088         162 :           SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
    4089         278 :           return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
    4090             :         }
    4091             :       }
    4092             :     }
    4093             :   }
    4094             : 
    4095      377473 :   return SDValue();
    4096             : }
    4097             : 
    4098           0 : bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
    4099             :                                    EVT LoadResultTy, EVT &ExtVT) {
    4100           0 :   if (!AndC->getAPIntValue().isMask())
    4101           0 :     return false;
    4102             : 
    4103             :   unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
    4104             : 
    4105           0 :   ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    4106           0 :   EVT LoadedVT = LoadN->getMemoryVT();
    4107             : 
    4108           0 :   if (ExtVT == LoadedVT &&
    4109           0 :       (!LegalOperations ||
    4110           0 :        TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
    4111             :     // ZEXTLOAD will match without needing to change the size of the value being
    4112             :     // loaded.
    4113           0 :     return true;
    4114             :   }
    4115             : 
    4116             :   // Do not change the width of a volatile load.
    4117           0 :   if (LoadN->isVolatile())
    4118           0 :     return false;
    4119             : 
    4120             :   // Do not generate loads of non-round integer types since these can
    4121             :   // be expensive (and would be wrong if the type is not byte sized).
    4122           0 :   if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
    4123           0 :     return false;
    4124             : 
    4125           0 :   if (LegalOperations &&
    4126           0 :       !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
    4127           0 :     return false;
    4128             : 
    4129           0 :   if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
    4130           0 :     return false;
    4131             : 
    4132             :   return true;
    4133             : }
    4134             : 
    4135      149739 : bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
    4136             :                                     ISD::LoadExtType ExtType, EVT &MemVT,
    4137             :                                     unsigned ShAmt) {
    4138      149739 :   if (!LDST)
    4139             :     return false;
    4140             :   // Only allow byte offsets.
    4141      149739 :   if (ShAmt % 8)
    4142             :     return false;
    4143             : 
    4144             :   // Do not generate loads of non-round integer types since these can
    4145             :   // be expensive (and would be wrong if the type is not byte sized).
    4146             :   if (!MemVT.isRound())
    4147             :     return false;
    4148             : 
    4149             :   // Don't change the width of a volatile load.
    4150       54685 :   if (LDST->isVolatile())
    4151             :     return false;
    4152             : 
    4153             :   // Verify that we are actually reducing a load width here.
    4154       53804 :   if (LDST->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits())
    4155             :     return false;
    4156             : 
    4157             :   // Ensure that this isn't going to produce an unsupported unaligned access.
    4158       65082 :   if (ShAmt &&
    4159       11741 :       !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
    4160             :                               LDST->getAddressSpace(), ShAmt / 8))
    4161             :     return false;
    4162             : 
    4163             :   // It's not possible to generate a constant of extended or untyped type.
    4164       53335 :   EVT PtrType = LDST->getBasePtr().getValueType();
    4165       53335 :   if (PtrType == MVT::Untyped || PtrType.isExtended())
    4166             :     return false;
    4167             : 
    4168       53335 :   if (isa<LoadSDNode>(LDST)) {
    4169             :     LoadSDNode *Load = cast<LoadSDNode>(LDST);
    4170             :     // Don't transform one with multiple uses, this would require adding a new
    4171             :     // load.
    4172       53335 :     if (!SDValue(Load, 0).hasOneUse())
    4173             :       return false;
    4174             : 
    4175       22761 :     if (LegalOperations &&
    4176       18448 :         !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
    4177             :       return false;
    4178             : 
    4179             :     // For the transform to be legal, the load must produce only two values
    4180             :     // (the value loaded and the chain).  Don't transform a pre-increment
    4181             :     // load, for example, which produces an extra value.  Otherwise the
    4182             :     // transformation is not equivalent, and the downstream logic to replace
    4183             :     // uses gets things wrong.
    4184        5154 :     if (Load->getNumValues() > 2)
    4185             :       return false;
    4186             : 
    4187             :     // If the load that we're shrinking is an extload and we're not just
    4188             :     // discarding the extension we can't simply shrink the load. Bail.
    4189             :     // TODO: It would be possible to merge the extensions in some cases.
    4190        5154 :     if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
    4191         848 :         Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
    4192           1 :       return false;
    4193             : 
    4194        5153 :     if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT))
    4195        1874 :       return false;
    4196             :   } else {
    4197             :     assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode");
    4198             :     StoreSDNode *Store = cast<StoreSDNode>(LDST);
    4199             :     // Can't write outside the original store
    4200           0 :     if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
    4201             :       return false;
    4202             : 
    4203           0 :     if (LegalOperations &&
    4204           0 :         !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT))
    4205           0 :       return false;
    4206             :   }
    4207             :   return true;
    4208             : }
    4209             : 
    4210      200374 : bool DAGCombiner::SearchForAndLoads(SDNode *N,
    4211             :                                     SmallPtrSetImpl<LoadSDNode*> &Loads,
    4212             :                                     SmallPtrSetImpl<SDNode*> &NodesWithConsts,
    4213             :                                     ConstantSDNode *Mask,
    4214             :                                     SDNode *&NodeToMask) {
    4215             :   // Recursively search for the operands, looking for loads which can be
    4216             :   // narrowed.
    4217      553516 :   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
    4218      754452 :     SDValue Op = N->getOperand(i);
    4219             : 
    4220      754452 :     if (Op.getValueType().isVector())
    4221             :       return false;
    4222             : 
    4223             :     // Some constants may need fixing up later if they are too large.
    4224             :     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
    4225      353170 :       if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
    4226      179238 :           (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
    4227          89 :         NodesWithConsts.insert(N);
    4228      176206 :       continue;
    4229             :     }
    4230             : 
    4231      201020 :     if (!Op.hasOneUse())
    4232             :       return false;
    4233             : 
    4234      182599 :     switch(Op.getOpcode()) {
    4235             :     case ISD::LOAD: {
    4236             :       auto *Load = cast<LoadSDNode>(Op);
    4237        1278 :       EVT ExtVT;
    4238        2726 :       if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
    4239         170 :           isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
    4240             : 
    4241             :         // ZEXTLOAD is already small enough.
    4242         159 :         if (Load->getExtensionType() == ISD::ZEXTLOAD &&
    4243          13 :             ExtVT.bitsGE(Load->getMemoryVT()))
    4244         159 :           continue;
    4245             : 
    4246             :         // Use LE to convert equal sized loads to zext.
    4247         150 :         if (ExtVT.bitsLE(Load->getMemoryVT()))
    4248         150 :           Loads.insert(Load);
    4249             : 
    4250         150 :         continue;
    4251             :       }
    4252        1119 :       return false;
    4253             :     }
    4254        1730 :     case ISD::ZERO_EXTEND:
    4255             :     case ISD::AssertZext: {
    4256        1730 :       unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
    4257        1730 :       EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    4258             :       EVT VT = Op.getOpcode() == ISD::AssertZext ?
    4259        1168 :         cast<VTSDNode>(Op.getOperand(1))->getVT() :
    4260        1730 :         Op.getOperand(0).getValueType();
    4261             : 
    4262             :       // We can accept extending nodes if the mask is wider or an equal
    4263             :       // width to the original type.
    4264        1730 :       if (ExtVT.bitsGE(VT))
    4265        1162 :         continue;
    4266         568 :       break;
    4267             :     }
    4268        4418 :     case ISD::OR:
    4269             :     case ISD::XOR:
    4270             :     case ISD::AND:
    4271        4418 :       if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
    4272             :                              NodeToMask))
    4273             :         return false;
    4274             :       continue;
    4275             :     }
    4276             : 
    4277             :     // Allow one node which will masked along with any loads found.
    4278      175741 :     if (NodeToMask)
    4279             :       return false;
    4280             : 
    4281             :     // Also ensure that the node to be masked only produces one data result.
    4282      175345 :     NodeToMask = Op.getNode();
    4283      350690 :     if (NodeToMask->getNumValues() > 1) {
    4284             :       bool HasValue = false;
    4285       20927 :       for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
    4286             :         MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
    4287       14747 :         if (VT != MVT::Glue && VT != MVT::Other) {
    4288        7734 :           if (HasValue) {
    4289         777 :             NodeToMask = nullptr;
    4290             :             return false;
    4291             :           }
    4292             :           HasValue = true;
    4293             :         }
    4294             :       }
    4295             :       assert(HasValue && "Node to be masked has no data result?");
    4296             :     }
    4297             :   }
    4298             :   return true;
    4299             : }
    4300             : 
    4301      308037 : bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
    4302      308037 :   auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
    4303             :   if (!Mask)
    4304             :     return false;
    4305             : 
    4306      502786 :   if (!Mask->getAPIntValue().isMask())
    4307             :     return false;
    4308             : 
    4309             :   // No need to do anything if the and directly uses a load.
    4310      228221 :   if (isa<LoadSDNode>(N->getOperand(0)))
    4311             :     return false;
    4312             : 
    4313             :   SmallPtrSet<LoadSDNode*, 8> Loads;
    4314             :   SmallPtrSet<SDNode*, 2> NodesWithConsts;
    4315      195956 :   SDNode *FixupNode = nullptr;
    4316      195956 :   if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
    4317      175243 :     if (Loads.size() == 0)
    4318             :       return false;
    4319             : 
    4320             :     LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump());
    4321          88 :     SDValue MaskOp = N->getOperand(1);
    4322             : 
    4323             :     // If it exists, fixup the single node we allow in the tree that needs
    4324             :     // masking.
    4325          88 :     if (FixupNode) {
    4326             :       LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump());
    4327          26 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
    4328             :                                 FixupNode->getValueType(0),
    4329          26 :                                 SDValue(FixupNode, 0), MaskOp);
    4330          52 :       DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
    4331          26 :       if (And.getOpcode() == ISD ::AND)
    4332          50 :         DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
    4333             :     }
    4334             : 
    4335             :     // Narrow any constants that need it.
    4336         102 :     for (auto *LogicN : NodesWithConsts) {
    4337          14 :       SDValue Op0 = LogicN->getOperand(0);
    4338          14 :       SDValue Op1 = LogicN->getOperand(1);
    4339             : 
    4340             :       if (isa<ConstantSDNode>(Op0))
    4341             :           std::swap(Op0, Op1);
    4342             : 
    4343          14 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
    4344          14 :                                 Op1, MaskOp);
    4345             : 
    4346          14 :       DAG.UpdateNodeOperands(LogicN, Op0, And);
    4347             :     }
    4348             : 
    4349             :     // Create narrow loads.
    4350         233 :     for (auto *Load : Loads) {
    4351             :       LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump());
    4352         145 :       SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
    4353         145 :                                 SDValue(Load, 0), MaskOp);
    4354         145 :       DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
    4355         145 :       if (And.getOpcode() == ISD ::AND)
    4356         145 :         And = SDValue(
    4357             :             DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
    4358         145 :       SDValue NewLoad = ReduceLoadWidth(And.getNode());
    4359             :       assert(NewLoad &&
    4360             :              "Shouldn't be masking the load if it can't be narrowed");
    4361             :       CombineTo(Load, NewLoad, NewLoad.getValue(1));
    4362             :     }
    4363          88 :     DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
    4364          88 :     return true;
    4365             :   }
    4366             :   return false;
    4367             : }
    4368             : 
    4369             : // Unfold
    4370             : //    x &  (-1 'logical shift' y)
    4371             : // To
    4372             : //    (x 'opposite logical shift' y) 'logical shift' y
    4373             : // if it is better for performance.
    4374           0 : SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
    4375             :   assert(N->getOpcode() == ISD::AND);
    4376             : 
    4377           0 :   SDValue N0 = N->getOperand(0);
    4378           0 :   SDValue N1 = N->getOperand(1);
    4379             : 
    4380             :   // Do we actually prefer shifts over mask?
    4381           0 :   if (!TLI.preferShiftsToClearExtremeBits(N0))
    4382           0 :     return SDValue();
    4383             : 
    4384             :   // Try to match  (-1 '[outer] logical shift' y)
    4385             :   unsigned OuterShift;
    4386             :   unsigned InnerShift; // The opposite direction to the OuterShift.
    4387           0 :   SDValue Y;           // Shift amount.
    4388             :   auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
    4389             :     if (!M.hasOneUse())
    4390             :       return false;
    4391             :     OuterShift = M->getOpcode();
    4392             :     if (OuterShift == ISD::SHL)
    4393             :       InnerShift = ISD::SRL;
    4394             :     else if (OuterShift == ISD::SRL)
    4395             :       InnerShift = ISD::SHL;
    4396             :     else
    4397             :       return false;
    4398             :     if (!isAllOnesConstant(M->getOperand(0)))
    4399             :       return false;
    4400             :     Y = M->getOperand(1);
    4401             :     return true;
    4402           0 :   };
    4403             : 
    4404           0 :   SDValue X;
    4405           0 :   if (matchMask(N1))
    4406           0 :     X = N0;
    4407           0 :   else if (matchMask(N0))
    4408           0 :     X = N1;
    4409             :   else
    4410           0 :     return SDValue();
    4411             : 
    4412             :   SDLoc DL(N);
    4413           0 :   EVT VT = N->getValueType(0);
    4414             : 
    4415             :   //     tmp = x   'opposite logical shift' y
    4416           0 :   SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
    4417             :   //     ret = tmp 'logical shift' y
    4418           0 :   SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
    4419             : 
    4420           0 :   return T1;
    4421             : }
    4422             : 
    4423      400002 : SDValue DAGCombiner::visitAND(SDNode *N) {
    4424      400002 :   SDValue N0 = N->getOperand(0);
    4425      400002 :   SDValue N1 = N->getOperand(1);
    4426      400002 :   EVT VT = N1.getValueType();
    4427             : 
    4428             :   // x & x --> x
    4429      400002 :   if (N0 == N1)
    4430          21 :     return N0;
    4431             : 
    4432             :   // fold vector ops
    4433      399981 :   if (VT.isVector()) {
    4434       54116 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    4435          10 :       return FoldedVOp;
    4436             : 
    4437             :     // fold (and x, 0) -> 0, vector edition
    4438       54106 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    4439             :       // do not return N0, because undef node may exist in N0
    4440         126 :       return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
    4441         189 :                              SDLoc(N), N0.getValueType());
    4442       54043 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    4443             :       // do not return N1, because undef node may exist in N1
    4444           4 :       return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
    4445           6 :                              SDLoc(N), N1.getValueType());
    4446             : 
    4447             :     // fold (and x, -1) -> x, vector edition
    4448       54041 :     if (ISD::isBuildVectorAllOnes(N0.getNode()))
    4449          51 :       return N1;
    4450       53990 :     if (ISD::isBuildVectorAllOnes(N1.getNode()))
    4451           7 :       return N0;
    4452             :   }
    4453             : 
    4454             :   // fold (and c1, c2) -> c1&c2
    4455      399848 :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    4456      399848 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    4457      399848 :   if (N0C && N1C && !N1C->isOpaque())
    4458          44 :     return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
    4459             :   // canonicalize constant to RHS
    4460      400440 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    4461         614 :       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    4462        1204 :     return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
    4463             :   // fold (and x, -1) -> x
    4464      399224 :   if (isAllOnesConstant(N1))
    4465          11 :     return N0;
    4466             :   // if (and x, c) is known to be zero, return 0
    4467             :   unsigned BitWidth = VT.getScalarSizeInBits();
    4468      732794 :   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
    4469     1066316 :                                    APInt::getAllOnesValue(BitWidth)))
    4470         118 :     return DAG.getConstant(0, SDLoc(N), VT);
    4471             : 
    4472      399154 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    4473         190 :     return NewSel;
    4474             : 
    4475             :   // reassociate and
    4476      544076 :   if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
    4477        2044 :     return RAND;
    4478             : 
    4479             :   // Try to convert a constant mask AND into a shuffle clear mask.
    4480      396920 :   if (VT.isVector())
    4481       53297 :     if (SDValue Shuffle = XformToShuffleWithZero(N))
    4482        1089 :       return Shuffle;
    4483             : 
    4484             :   // fold (and (or x, C), D) -> D if (C & D) == D
    4485             :   auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    4486             :     return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
    4487             :   };
    4488      802719 :   if (N0.getOpcode() == ISD::OR &&
    4489      415656 :       ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
    4490        2289 :     return N1;
    4491             :   // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
    4492      393542 :   if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    4493       68021 :     SDValue N0Op0 = N0.getOperand(0);
    4494       68021 :     APInt Mask = ~N1C->getAPIntValue();
    4495       68021 :     Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
    4496       68021 :     if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
    4497       13463 :       SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
    4498       27189 :                                  N0.getValueType(), N0Op0);
    4499             : 
    4500             :       // Replace uses of the AND with uses of the Zero extend node.
    4501       13463 :       CombineTo(N, Zext);
    4502             : 
    4503             :       // We actually want to replace all uses of the any_extend with the
    4504             :       // zero_extend, to avoid duplicating things.  This will later cause this
    4505             :       // AND to be folded.
    4506       13463 :       CombineTo(N0.getNode(), Zext);
    4507       13463 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4508             :     }
    4509             :   }
    4510             :   // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
    4511             :   // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
    4512             :   // already be zero by virtue of the width of the base type of the load.
    4513             :   //
    4514             :   // the 'X' node here can either be nothing or an extract_vector_elt to catch
    4515             :   // more cases.
    4516      389425 :   if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
    4517       17636 :        N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
    4518       16580 :        N0.getOperand(0).getOpcode() == ISD::LOAD &&
    4519      389425 :        N0.getOperand(0).getResNo() == 0) ||
    4520      376341 :       (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
    4521      160864 :     LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
    4522             :                                          N0 : N0.getOperand(0) );
    4523             : 
    4524             :     // Get the constant (if applicable) the zero'th operand is being ANDed with.
    4525             :     // This can be a pure constant or a vector splat, in which case we treat the
    4526             :     // vector as a scalar and use the splat value.
    4527             :     APInt Constant = APInt::getNullValue(1);
    4528             :     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
    4529      148002 :       Constant = C->getAPIntValue();
    4530             :     } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
    4531             :       APInt SplatValue, SplatUndef;
    4532             :       unsigned SplatBitSize;
    4533             :       bool HasAnyUndefs;
    4534        1107 :       bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
    4535             :                                              SplatBitSize, HasAnyUndefs);
    4536        1107 :       if (IsSplat) {
    4537             :         // Undef bits can contribute to a possible optimisation if set, so
    4538             :         // set them.
    4539             :         SplatValue |= SplatUndef;
    4540             : 
    4541             :         // The splat value may be something like "0x00FFFFFF", which means 0 for
    4542             :         // the first vector value and FF for the rest, repeating. We need a mask
    4543             :         // that will apply equally to all members of the vector, so AND all the
    4544             :         // lanes of the constant together.
    4545        2210 :         EVT VT = Vector->getValueType(0);
    4546             :         unsigned BitWidth = VT.getScalarSizeInBits();
    4547             : 
    4548             :         // If the splat value has been compressed to a bitlength lower
    4549             :         // than the size of the vector lane, we need to re-expand it to
    4550             :         // the lane size.
    4551        1105 :         if (BitWidth > SplatBitSize)
    4552           2 :           for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
    4553           2 :                SplatBitSize < BitWidth;
    4554           1 :                SplatBitSize = SplatBitSize * 2)
    4555           2 :             SplatValue |= SplatValue.shl(SplatBitSize);
    4556             : 
    4557             :         // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
    4558             :         // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
    4559        1105 :         if (SplatBitSize % BitWidth == 0) {
    4560        1104 :           Constant = APInt::getAllOnesValue(BitWidth);
    4561        2253 :           for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
    4562        2306 :             Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
    4563             :         }
    4564             :       }
    4565             :     }
    4566             : 
    4567             :     // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
    4568             :     // actually legal and isn't going to get expanded, else this is a false
    4569             :     // optimisation.
    4570       80432 :     bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
    4571             :                                                     Load->getValueType(0),
    4572             :                                                     Load->getMemoryVT());
    4573             : 
    4574             :     // Resize the constant to the same size as the original memory access before
    4575             :     // extension. If it is still the AllOnesValue then this AND is completely
    4576             :     // unneeded.
    4577      160864 :     Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
    4578             : 
    4579             :     bool B;
    4580       80432 :     switch (Load->getExtensionType()) {
    4581             :     default: B = false; break;
    4582             :     case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
    4583             :     case ISD::ZEXTLOAD:
    4584             :     case ISD::NON_EXTLOAD: B = true; break;
    4585             :     }
    4586             : 
    4587       93092 :     if (B && Constant.isAllOnesValue()) {
    4588             :       // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
    4589             :       // preserve semantics once we get rid of the AND.
    4590             :       SDValue NewLoad(Load, 0);
    4591             : 
    4592             :       // Fold the AND away. NewLoad may get replaced immediately.
    4593        4444 :       CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
    4594             : 
    4595        2222 :       if (Load->getExtensionType() == ISD::EXTLOAD) {
    4596        1268 :         NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
    4597        1268 :                               Load->getValueType(0), SDLoc(Load),
    4598             :                               Load->getChain(), Load->getBasePtr(),
    4599             :                               Load->getOffset(), Load->getMemoryVT(),
    4600        1276 :                               Load->getMemOperand());
    4601             :         // Replace uses of the EXTLOAD with the new ZEXTLOAD.
    4602        1268 :         if (Load->getNumValues() == 3) {
    4603             :           // PRE/POST_INC loads have 3 values.
    4604           0 :           SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
    4605           0 :                            NewLoad.getValue(2) };
    4606           0 :           CombineTo(Load, To, 3, true);
    4607             :         } else {
    4608        1268 :           CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
    4609             :         }
    4610             :       }
    4611             : 
    4612        2222 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    4613             :     }
    4614             :   }
    4615             : 
    4616             :   // fold (and (load x), 255) -> (zextload x, i8)
    4617             :   // fold (and (extload x, i16), 255) -> (zextload x, i8)
    4618             :   // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
    4619      377857 :   if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
    4620       54529 :                                 (N0.getOpcode() == ISD::ANY_EXTEND &&
    4621       54529 :                                  N0.getOperand(0).getOpcode() == ISD::LOAD))) {
    4622       80073 :     if (SDValue Res = ReduceLoadWidth(N)) {
    4623         138 :       LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
    4624         138 :         ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
    4625             : 
    4626         138 :       AddToWorklist(N);
    4627             :       CombineTo(LN0, Res, Res.getValue(1));
    4628         138 :       return SDValue(N, 0);
    4629             :     }
    4630             :   }
    4631             : 
    4632      377719 :   if (Level >= AfterLegalizeTypes) {
    4633             :     // Attempt to propagate the AND back up to the leaves which, if they're
    4634             :     // loads, can be combined to narrow loads and the AND node can be removed.
    4635             :     // Perform after legalization so that extend nodes will already be
    4636             :     // combined into the loads.
    4637      308037 :     if (BackwardsPropagateMask(N, DAG)) {
    4638          88 :       return SDValue(N, 0);
    4639             :     }
    4640             :   }
    4641             : 
    4642      377631 :   if (SDValue Combined = visitANDLike(N0, N1, N))
    4643         218 :     return Combined;
    4644             : 
    4645             :   // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
    4646     1132239 :   if (N0.getOpcode() == N1.getOpcode())
    4647       27632 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    4648         299 :       return Tmp;
    4649             : 
    4650             :   // Masking the negated extension of a boolean is just the zero-extended
    4651             :   // boolean:
    4652             :   // and (sub 0, zext(bool X)), 1 --> zext(bool X)
    4653             :   // and (sub 0, sext(bool X)), 1 --> zext(bool X)
    4654             :   //
    4655             :   // Note: the SimplifyDemandedBits fold below can make an information-losing
    4656             :   // transform, and then we have no way to find this better fold.
    4657      689439 :   if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
    4658          28 :     if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
    4659          12 :       SDValue SubRHS = N0.getOperand(1);
    4660           9 :       if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
    4661           3 :           SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
    4662           3 :         return SubRHS;
    4663           6 :       if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
    4664           3 :           SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
    4665           6 :         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
    4666             :     }
    4667             :   }
    4668             : 
    4669             :   // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
    4670             :   // fold (and (sra)) -> (and (srl)) when possible.
    4671      377108 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    4672      110614 :     return SDValue(N, 0);
    4673             : 
    4674             :   // fold (zext_inreg (extload x)) -> (zextload x)
    4675      266494 :   if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
    4676             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    4677       14236 :     EVT MemVT = LN0->getMemoryVT();
    4678             :     // If we zero all the possible extended bits, then we can turn this into
    4679             :     // a zextload if we are running before legalize or the operation is legal.
    4680       14236 :     unsigned BitWidth = N1.getScalarValueSizeInBits();
    4681       28472 :     if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
    4682       14236 :                            BitWidth - MemVT.getScalarSizeInBits())) &&
    4683       16727 :         ((!LegalOperations && !LN0->isVolatile()) ||
    4684        2561 :          TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
    4685       24544 :       SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
    4686             :                                        LN0->getChain(), LN0->getBasePtr(),
    4687       12309 :                                        MemVT, LN0->getMemOperand());
    4688       12272 :       AddToWorklist(N);
    4689       12272 :       CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    4690       12272 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4691             :     }
    4692             :   }
    4693             :   // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
    4694      254568 :   if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
    4695         346 :       N0.hasOneUse()) {
    4696             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    4697         301 :     EVT MemVT = LN0->getMemoryVT();
    4698             :     // If we zero all the possible extended bits, then we can turn this into
    4699             :     // a zextload if we are running before legalize or the operation is legal.
    4700         301 :     unsigned BitWidth = N1.getScalarValueSizeInBits();
    4701         602 :     if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
    4702         301 :                            BitWidth - MemVT.getScalarSizeInBits())) &&
    4703         488 :         ((!LegalOperations && !LN0->isVolatile()) ||
    4704         226 :          TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
    4705         492 :       SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
    4706             :                                        LN0->getChain(), LN0->getBasePtr(),
    4707         246 :                                        MemVT, LN0->getMemOperand());
    4708         246 :       AddToWorklist(N);
    4709         246 :       CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    4710         246 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    4711             :     }
    4712             :   }
    4713             :   // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
    4714      253976 :   if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
    4715         274 :     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
    4716         274 :                                            N0.getOperand(1), false))
    4717           1 :       return BSwap;
    4718             :   }
    4719             : 
    4720      253975 :   if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
    4721         636 :     return Shifts;
    4722             : 
    4723      253339 :   return SDValue();
    4724             : }
    4725             : 
    4726             : /// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
    4727      143554 : SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
    4728             :                                         bool DemandHighBits) {
    4729      143554 :   if (!LegalOperations)
    4730       56652 :     return SDValue();
    4731             : 
    4732      173804 :   EVT VT = N->getValueType(0);
    4733             :   if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
    4734       21392 :     return SDValue();
    4735       65510 :   if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    4736       14759 :     return SDValue();
    4737             : 
    4738             :   // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
    4739             :   bool LookPassAnd0 = false;
    4740             :   bool LookPassAnd1 = false;
    4741       50751 :   if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
    4742             :       std::swap(N0, N1);
    4743       50751 :   if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
    4744             :       std::swap(N0, N1);
    4745       50751 :   if (N0.getOpcode() == ISD::AND) {
    4746             :     if (!N0.getNode()->hasOneUse())
    4747         205 :       return SDValue();
    4748             :     ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4749             :     // Also handle 0xffff since the LHS is guaranteed to have zeros there.
    4750             :     // This is needed for X86.
    4751       12808 :     if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
    4752             :                   N01C->getZExtValue() != 0xFFFF))
    4753        5599 :       return SDValue();
    4754         999 :     N0 = N0.getOperand(0);
    4755             :     LookPassAnd0 = true;
    4756             :   }
    4757             : 
    4758       44947 :   if (N1.getOpcode() == ISD::AND) {
    4759             :     if (!N1.getNode()->hasOneUse())
    4760          24 :       return SDValue();
    4761             :     ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    4762        2288 :     if (!N11C || N11C->getZExtValue() != 0xFF)
    4763        1052 :       return SDValue();
    4764         107 :     N1 = N1.getOperand(0);
    4765             :     LookPassAnd1 = true;
    4766             :   }
    4767             : 
    4768       43871 :   if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
    4769             :     std::swap(N0, N1);
    4770       43871 :   if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
    4771       42824 :     return SDValue();
    4772             :   if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
    4773          14 :     return SDValue();
    4774             : 
    4775             :   ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4776             :   ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    4777        1033 :   if (!N01C || !N11C)
    4778         239 :     return SDValue();
    4779        1625 :   if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
    4780         774 :     return SDValue();
    4781             : 
    4782             :   // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
    4783          20 :   SDValue N00 = N0->getOperand(0);
    4784          20 :   if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
    4785             :     if (!N00.getNode()->hasOneUse())
    4786           0 :       return SDValue();
    4787             :     ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
    4788          12 :     if (!N001C || N001C->getZExtValue() != 0xFF)
    4789           2 :       return SDValue();
    4790           4 :     N00 = N00.getOperand(0);
    4791             :     LookPassAnd0 = true;
    4792             :   }
    4793             : 
    4794          18 :   SDValue N10 = N1->getOperand(0);
    4795          18 :   if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
    4796             :     if (!N10.getNode()->hasOneUse())
    4797           0 :       return SDValue();
    4798             :     ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
    4799             :     // Also allow 0xFFFF since the bits will be shifted out. This is needed
    4800             :     // for X86.
    4801          10 :     if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
    4802             :                    N101C->getZExtValue() != 0xFFFF))
    4803           0 :       return SDValue();
    4804           5 :     N10 = N10.getOperand(0);
    4805             :     LookPassAnd1 = true;
    4806             :   }
    4807             : 
    4808             :   if (N00 != N10)
    4809           0 :     return SDValue();
    4810             : 
    4811             :   // Make sure everything beyond the low halfword gets set to zero since the SRL
    4812             :   // 16 will clear the top bits.
    4813          18 :   unsigned OpSizeInBits = VT.getSizeInBits();
    4814          18 :   if (DemandHighBits && OpSizeInBits > 16) {
    4815             :     // If the left-shift isn't masked out then the only way this is a bswap is
    4816             :     // if all bits beyond the low 8 are 0. In that case the entire pattern
    4817             :     // reduces to a left shift anyway: leave it for other parts of the combiner.
    4818           9 :     if (!LookPassAnd0)
    4819           2 :       return SDValue();
    4820             : 
    4821             :     // However, if the right shift isn't masked out then it might be because
    4822             :     // it's not needed. See if we can spot that too.
    4823           9 :     if (!LookPassAnd1 &&
    4824           2 :         !DAG.MaskedValueIsZero(
    4825          11 :             N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
    4826           0 :       return SDValue();
    4827             :   }
    4828             : 
    4829          16 :   SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
    4830          16 :   if (OpSizeInBits > 16) {
    4831             :     SDLoc DL(N);
    4832          16 :     Res = DAG.getNode(ISD::SRL, DL, VT, Res,
    4833          16 :                       DAG.getConstant(OpSizeInBits - 16, DL,
    4834          16 :                                       getShiftAmountTy(VT)));
    4835             :   }
    4836          16 :   return Res;
    4837             : }
    4838             : 
    4839             : /// Return true if the specified node is an element that makes up a 32-bit
    4840             : /// packed halfword byteswap.
    4841             : /// ((x & 0x000000ff) << 8) |
    4842             : /// ((x & 0x0000ff00) >> 8) |
    4843             : /// ((x & 0x00ff0000) << 8) |
    4844             : /// ((x & 0xff000000) >> 8)
    4845           0 : static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
    4846             :   if (!N.getNode()->hasOneUse())
    4847           0 :     return false;
    4848             : 
    4849             :   unsigned Opc = N.getOpcode();
    4850           0 :   if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
    4851           0 :     return false;
    4852             : 
    4853           0 :   SDValue N0 = N.getOperand(0);
    4854             :   unsigned Opc0 = N0.getOpcode();
    4855           0 :   if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
    4856           0 :     return false;
    4857             : 
    4858             :   ConstantSDNode *N1C = nullptr;
    4859             :   // SHL or SRL: look upstream for AND mask operand
    4860           0 :   if (Opc == ISD::AND)
    4861             :     N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4862           0 :   else if (Opc0 == ISD::AND)
    4863             :     N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4864           0 :   if (!N1C)
    4865           0 :     return false;
    4866             : 
    4867             :   unsigned MaskByteOffset;
    4868           0 :   switch (N1C->getZExtValue()) {
    4869             :   default:
    4870             :     return false;
    4871             :   case 0xFF:       MaskByteOffset = 0; break;
    4872           0 :   case 0xFF00:     MaskByteOffset = 1; break;
    4873           0 :   case 0xFFFF:
    4874             :     // In case demanded bits didn't clear the bits that will be shifted out.
    4875             :     // This is needed for X86.
    4876           0 :     if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
    4877             :       MaskByteOffset = 1;
    4878             :       break;
    4879             :     }
    4880             :     return false;
    4881           0 :   case 0xFF0000:   MaskByteOffset = 2; break;
    4882           0 :   case 0xFF000000: MaskByteOffset = 3; break;
    4883             :   }
    4884             : 
    4885             :   // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
    4886           0 :   if (Opc == ISD::AND) {
    4887           0 :     if (MaskByteOffset == 0 || MaskByteOffset == 2) {
    4888             :       // (x >> 8) & 0xff
    4889             :       // (x >> 8) & 0xff0000
    4890           0 :       if (Opc0 != ISD::SRL)
    4891           0 :         return false;
    4892             :       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4893           0 :       if (!C || C->getZExtValue() != 8)
    4894           0 :         return false;
    4895             :     } else {
    4896             :       // (x << 8) & 0xff00
    4897             :       // (x << 8) & 0xff000000
    4898           0 :       if (Opc0 != ISD::SHL)
    4899           0 :         return false;
    4900             :       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    4901           0 :       if (!C || C->getZExtValue() != 8)
    4902           0 :         return false;
    4903             :     }
    4904           0 :   } else if (Opc == ISD::SHL) {
    4905             :     // (x & 0xff) << 8
    4906             :     // (x & 0xff0000) << 8
    4907           0 :     if (MaskByteOffset != 0 && MaskByteOffset != 2)
    4908           0 :       return false;
    4909             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4910           0 :     if (!C || C->getZExtValue() != 8)
    4911           0 :       return false;
    4912             :   } else { // Opc == ISD::SRL
    4913             :     // (x & 0xff00) >> 8
    4914             :     // (x & 0xff000000) >> 8
    4915           0 :     if (MaskByteOffset != 1 && MaskByteOffset != 3)
    4916           0 :       return false;
    4917             :     ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    4918           0 :     if (!C || C->getZExtValue() != 8)
    4919           0 :       return false;
    4920             :   }
    4921             : 
    4922           0 :   if (Parts[MaskByteOffset])
    4923           0 :     return false;
    4924             : 
    4925           0 :   Parts[MaskByteOffset] = N0.getOperand(0).getNode();
    4926           0 :   return true;
    4927             : }
    4928             : 
    4929             : /// Match a 32-bit packed halfword bswap. That is
    4930             : /// ((x & 0x000000ff) << 8) |
    4931             : /// ((x & 0x0000ff00) >> 8) |
    4932             : /// ((x & 0x00ff0000) << 8) |
    4933             : /// ((x & 0xff000000) >> 8)
    4934             : /// => (rotl (bswap x), 16)
    4935           0 : SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
    4936           0 :   if (!LegalOperations)
    4937           0 :     return SDValue();
    4938             : 
    4939           0 :   EVT VT = N->getValueType(0);
    4940           0 :   if (VT != MVT::i32)
    4941           0 :     return SDValue();
    4942           0 :   if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    4943           0 :     return SDValue();
    4944             : 
    4945             :   // Look for either
    4946             :   // (or (or (and), (and)), (or (and), (and)))
    4947             :   // (or (or (or (and), (and)), (and)), (and))
    4948           0 :   if (N0.getOpcode() != ISD::OR)
    4949           0 :     return SDValue();
    4950           0 :   SDValue N00 = N0.getOperand(0);
    4951           0 :   SDValue N01 = N0.getOperand(1);
    4952           0 :   SDNode *Parts[4] = {};
    4953             : 
    4954           0 :   if (N1.getOpcode() == ISD::OR &&
    4955           0 :       N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
    4956             :     // (or (or (and), (and)), (or (and), (and)))
    4957           0 :     if (!isBSwapHWordElement(N00, Parts))
    4958           0 :       return SDValue();
    4959             : 
    4960           0 :     if (!isBSwapHWordElement(N01, Parts))
    4961           0 :       return SDValue();
    4962           0 :     SDValue N10 = N1.getOperand(0);
    4963           0 :     if (!isBSwapHWordElement(N10, Parts))
    4964           0 :       return SDValue();
    4965           0 :     SDValue N11 = N1.getOperand(1);
    4966           0 :     if (!isBSwapHWordElement(N11, Parts))
    4967           0 :       return SDValue();
    4968             :   } else {
    4969             :     // (or (or (or (and), (and)), (and)), (and))
    4970           0 :     if (!isBSwapHWordElement(N1, Parts))
    4971           0 :       return SDValue();
    4972           0 :     if (!isBSwapHWordElement(N01, Parts))
    4973           0 :       return SDValue();
    4974           0 :     if (N00.getOpcode() != ISD::OR)
    4975           0 :       return SDValue();
    4976           0 :     SDValue N000 = N00.getOperand(0);
    4977           0 :     if (!isBSwapHWordElement(N000, Parts))
    4978           0 :       return SDValue();
    4979           0 :     SDValue N001 = N00.getOperand(1);
    4980           0 :     if (!isBSwapHWordElement(N001, Parts))
    4981           0 :       return SDValue();
    4982             :   }
    4983             : 
    4984             :   // Make sure the parts are all coming from the same node.
    4985           0 :   if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
    4986           0 :     return SDValue();
    4987             : 
    4988             :   SDLoc DL(N);
    4989           0 :   SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
    4990           0 :                               SDValue(Parts[0], 0));
    4991             : 
    4992             :   // Result of the bswap should be rotated by 16. If it's not legal, then
    4993             :   // do  (x << 16) | (x >> 16).
    4994           0 :   SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
    4995           0 :   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
    4996           0 :     return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
    4997           0 :   if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
    4998           0 :     return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
    4999           0 :   return DAG.getNode(ISD::OR, DL, VT,
    5000           0 :                      DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
    5001           0 :                      DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
    5002             : }
    5003             : 
    5004             : /// This contains all DAGCombine rules which reduce two values combined by
    5005             : /// an Or operation to a single value \see visitANDLike().
    5006      143660 : SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
    5007      287320 :   EVT VT = N1.getValueType();
    5008             :   SDLoc DL(N);
    5009             : 
    5010             :   // fold (or x, undef) -> -1
    5011      143660 :   if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
    5012          15 :     return DAG.getAllOnesConstant(DL, VT);
    5013             : 
    5014      143645 :   if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
    5015         112 :     return V;
    5016             : 
    5017             :   // (or (and X, C1), (and Y, C2))  -> (and (or X, Y), C3) if possible.
    5018      143533 :   if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
    5019             :       // Don't increase # computations.
    5020             :       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
    5021             :     // We can only do this xform if we know that bits from X that are set in C2
    5022             :     // but not in C1 are already zero.  Likewise for Y.
    5023             :     if (const ConstantSDNode *N0O1C =
    5024        8167 :         getAsNonOpaqueConstant(N0.getOperand(1))) {
    5025             :       if (const ConstantSDNode *N1O1C =
    5026        2980 :           getAsNonOpaqueConstant(N1.getOperand(1))) {
    5027             :         // We can only do this xform if we know that bits from X that are set in
    5028             :         // C2 but not in C1 are already zero.  Likewise for Y.
    5029        2905 :         const APInt &LHSMask = N0O1C->getAPIntValue();
    5030        2905 :         const APInt &RHSMask = N1O1C->getAPIntValue();
    5031             : 
    5032        8954 :         if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
    5033        4100 :             DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
    5034          21 :           SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
    5035          21 :                                   N0.getOperand(0), N1.getOperand(0));
    5036          21 :           return DAG.getNode(ISD::AND, DL, VT, X,
    5037          42 :                              DAG.getConstant(LHSMask | RHSMask, DL, VT));
    5038             :         }
    5039             :       }
    5040             :     }
    5041             :   }
    5042             : 
    5043             :   // (or (and X, M), (and X, N)) -> (and X, (or M, N))
    5044       25558 :   if (N0.getOpcode() == ISD::AND &&
    5045             :       N1.getOpcode() == ISD::AND &&
    5046      151670 :       N0.getOperand(0) == N1.getOperand(0) &&
    5047             :       // Don't increase # computations.
    5048             :       (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
    5049         188 :     SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
    5050         188 :                             N0.getOperand(1), N1.getOperand(1));
    5051         376 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
    5052             :   }
    5053             : 
    5054      143324 :   return SDValue();
    5055             : }
    5056             : 
    5057      146434 : SDValue DAGCombiner::visitOR(SDNode *N) {
    5058      146434 :   SDValue N0 = N->getOperand(0);
    5059      146434 :   SDValue N1 = N->getOperand(1);
    5060      146434 :   EVT VT = N1.getValueType();
    5061             : 
    5062             :   // x | x --> x
    5063             :   if (N0 == N1)
    5064          17 :     return N0;
    5065             : 
    5066             :   // fold vector ops
    5067      146417 :   if (VT.isVector()) {
    5068       24910 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    5069           4 :       return FoldedVOp;
    5070             : 
    5071             :     // fold (or x, 0) -> x, vector edition
    5072       24906 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    5073          22 :       return N1;
    5074       24884 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    5075         127 :       return N0;
    5076             : 
    5077             :     // fold (or x, -1) -> -1, vector edition
    5078       24757 :     if (ISD::isBuildVectorAllOnes(N0.getNode()))
    5079             :       // do not return N0, because undef node may exist in N0
    5080          10 :       return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
    5081       24752 :     if (ISD::isBuildVectorAllOnes(N1.getNode()))
    5082             :       // do not return N1, because undef node may exist in N1
    5083           2 :       return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
    5084             : 
    5085             :     // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
    5086             :     // Do this only if the resulting shuffle is legal.
    5087         170 :     if (isa<ShuffleVectorSDNode>(N0) &&
    5088       24751 :         isa<ShuffleVectorSDNode>(N1) &&
    5089             :         // Avoid folding a node with illegal type.
    5090         143 :         TLI.isTypeLegal(VT)) {
    5091         142 :       bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
    5092         142 :       bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
    5093         142 :       bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
    5094         142 :       bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
    5095             :       // Ensure both shuffles have a zero input.
    5096         142 :       if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
    5097             :         assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!");
    5098             :         assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!");
    5099             :         const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
    5100             :         const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
    5101             :         bool CanFold = true;
    5102          66 :         int NumElts = VT.getVectorNumElements();
    5103          66 :         SmallVector<int, 4> Mask(NumElts);
    5104             : 
    5105         300 :         for (int i = 0; i != NumElts; ++i) {
    5106         239 :           int M0 = SV0->getMaskElt(i);
    5107         239 :           int M1 = SV1->getMaskElt(i);
    5108             : 
    5109             :           // Determine if either index is pointing to a zero vector.
    5110         239 :           bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
    5111         239 :           bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
    5112             : 
    5113             :           // If one element is zero and the otherside is undef, keep undef.
    5114             :           // This also handles the case that both are undef.
    5115         239 :           if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
    5116           1 :             Mask[i] = -1;
    5117           1 :             continue;
    5118             :           }
    5119             : 
    5120             :           // Make sure only one of the elements is zero.
    5121         238 :           if (M0Zero == M1Zero) {
    5122             :             CanFold = false;
    5123             :             break;
    5124             :           }
    5125             : 
    5126             :           assert((M0 >= 0 || M1 >= 0) && "Undef index!");
    5127             : 
    5128             :           // We have a zero and non-zero element. If the non-zero came from
    5129             :           // SV0 make the index a LHS index. If it came from SV1, make it
    5130             :           // a RHS index. We need to mod by NumElts because we don't care
    5131             :           // which operand it came from in the original shuffles.
    5132         233 :           Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
    5133             :         }
    5134             : 
    5135          66 :         if (CanFold) {
    5136          61 :           SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
    5137          61 :           SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
    5138             : 
    5139         122 :           bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
    5140          61 :           if (!LegalMask) {
    5141             :             std::swap(NewLHS, NewRHS);
    5142             :             ShuffleVectorSDNode::commuteMask(Mask);
    5143           0 :             LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
    5144             :           }
    5145             : 
    5146          61 :           if (LegalMask)
    5147         122 :             return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
    5148             :         }
    5149             :       }
    5150             :     }
    5151             :   }
    5152             : 
    5153             :   // fold (or c1, c2) -> c1|c2
    5154             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    5155             :   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
    5156      146197 :   if (N0C && N1C && !N1C->isOpaque())
    5157         182 :     return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
    5158             :   // canonicalize constant to RHS
    5159      147686 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    5160        1580 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    5161        3149 :     return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
    5162             :   // fold (or x, 0) -> x
    5163      144532 :   if (isNullConstant(N1))
    5164         831 :     return N0;
    5165             :   // fold (or x, -1) -> -1
    5166      143701 :   if (isAllOnesConstant(N1))
    5167          85 :     return N1;
    5168             : 
    5169      143616 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    5170          54 :     return NewSel;
    5171             : 
    5172             :   // fold (or x, c) -> c iff (x & ~c) == 0
    5173      402740 :   if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
    5174           2 :     return N1;
    5175             : 
    5176      143560 :   if (SDValue Combined = visitORLike(N0, N1, N))
    5177         330 :     return Combined;
    5178             : 
    5179             :   // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
    5180      143230 :   if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
    5181           8 :     return BSwap;
    5182      143222 :   if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
    5183           7 :     return BSwap;
    5184             : 
    5185             :   // reassociate or
    5186      181815 :   if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags()))
    5187          67 :     return ROR;
    5188             : 
    5189             :   // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
    5190             :   // iff (c1 & c2) != 0.
    5191             :   auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    5192             :     return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
    5193             :   };
    5194      168198 :   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
    5195      193223 :       ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
    5196          25 :     if (SDValue COR = DAG.FoldConstantArithmetic(
    5197          50 :             ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
    5198          25 :       SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
    5199          25 :       AddToWorklist(IOR.getNode());
    5200          50 :       return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
    5201             :     }
    5202             :   }
    5203             : 
    5204             :   // Simplify: (or (op x...), (op y...))  -> (op (or x, y))
    5205      143123 :   if (N0.getOpcode() == N1.getOpcode())
    5206       24813 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    5207         424 :       return Tmp;
    5208             : 
    5209             :   // See if this is some rotate idiom.
    5210      285398 :   if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
    5211        1112 :     return SDValue(Rot, 0);
    5212             : 
    5213      141587 :   if (SDValue Load = MatchLoadCombine(N))
    5214         184 :     return Load;
    5215             : 
    5216             :   // Simplify the operands using demanded-bits information.
    5217      141403 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    5218        6353 :     return SDValue(N, 0);
    5219             : 
    5220      135050 :   return SDValue();
    5221             : }
    5222             : 
    5223      265514 : static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
    5224      297777 :   if (Op.getOpcode() == ISD::AND &&
    5225       32263 :       DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
    5226       27712 :     Mask = Op.getOperand(1);
    5227       27712 :     return Op.getOperand(0);
    5228             :   }
    5229      237802 :   return Op;
    5230             : }
    5231             : 
    5232             : /// Match "(X shl/srl V1) & V2" where V2 may not be present.
    5233      226104 : static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
    5234             :                             SDValue &Mask) {
    5235      226104 :   Op = stripConstantMask(DAG, Op, Mask);
    5236      226104 :   if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
    5237       39402 :     Shift = Op;
    5238       39402 :     return true;
    5239             :   }
    5240             :   return false;
    5241             : }
    5242             : 
    5243             : /// Helper function for visitOR to extract the needed side of a rotate idiom
    5244             : /// from a shl/srl/mul/udiv.  This is meant to handle cases where
    5245             : /// InstCombine merged some outside op with one of the shifts from
    5246             : /// the rotate pattern.
    5247             : /// \returns An empty \c SDValue if the needed shift couldn't be extracted.
    5248             : /// Otherwise, returns an expansion of \p ExtractFrom based on the following
    5249             : /// patterns:
    5250             : ///
    5251             : ///   (or (mul v c0) (shrl (mul v c1) c2)):
    5252             : ///     expands (mul v c0) -> (shl (mul v c1) c3)
    5253             : ///
    5254             : ///   (or (udiv v c0) (shl (udiv v c1) c2)):
    5255             : ///     expands (udiv v c0) -> (shrl (udiv v c1) c3)
    5256             : ///
    5257             : ///   (or (shl v c0) (shrl (shl v c1) c2)):
    5258             : ///     expands (shl v c0) -> (shl (shl v c1) c3)
    5259             : ///
    5260             : ///   (or (shrl v c0) (shl (shrl v c1) c2)):
    5261             : ///     expands (shrl v c0) -> (shrl (shrl v c1) c3)
    5262             : ///
    5263             : /// Such that in all cases, c3+c2==bitwidth(op v c1).
    5264           0 : static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
    5265             :                                      SDValue ExtractFrom, SDValue &Mask,
    5266             :                                      const SDLoc &DL) {
    5267             :   assert(OppShift && ExtractFrom && "Empty SDValue");
    5268             :   assert(
    5269             :       (OppShift.getOpcode() == ISD::SHL || OppShift.getOpcode() == ISD::SRL) &&
    5270             :       "Existing shift must be valid as a rotate half");
    5271             : 
    5272           0 :   ExtractFrom = stripConstantMask(DAG, ExtractFrom, Mask);
    5273             :   // Preconditions:
    5274             :   //    (or (op0 v c0) (shiftl/r (op0 v c1) c2))
    5275             :   //
    5276             :   // Find opcode of the needed shift to be extracted from (op0 v c0).
    5277           0 :   unsigned Opcode = ISD::DELETED_NODE;
    5278           0 :   bool IsMulOrDiv = false;
    5279             :   // Set Opcode and IsMulOrDiv if the extract opcode matches the needed shift
    5280             :   // opcode or its arithmetic (mul or udiv) variant.
    5281             :   auto SelectOpcode = [&](unsigned NeededShift, unsigned MulOrDivVariant) {
    5282           0 :     IsMulOrDiv = ExtractFrom.getOpcode() == MulOrDivVariant;
    5283           0 :     if (!IsMulOrDiv && ExtractFrom.getOpcode() != NeededShift)
    5284             :       return false;
    5285           0 :     Opcode = NeededShift;
    5286             :     return true;
    5287             :   };
    5288             :   // op0 must be either the needed shift opcode or the mul/udiv equivalent
    5289             :   // that the needed shift can be extracted from.
    5290           0 :   if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) &&
    5291             :       (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV)))
    5292           0 :     return SDValue();
    5293             : 
    5294             :   // op0 must be the same opcode on both sides, have the same LHS argument,
    5295             :   // and produce the same value type.
    5296           0 :   SDValue OppShiftLHS = OppShift.getOperand(0);
    5297           0 :   EVT ShiftedVT = OppShiftLHS.getValueType();
    5298           0 :   if (OppShiftLHS.getOpcode() != ExtractFrom.getOpcode() ||
    5299           0 :       OppShiftLHS.getOperand(0) != ExtractFrom.getOperand(0) ||
    5300           0 :       ShiftedVT != ExtractFrom.getValueType())
    5301           0 :     return SDValue();
    5302             : 
    5303             :   // Amount of the existing shift.
    5304           0 :   ConstantSDNode *OppShiftCst = isConstOrConstSplat(OppShift.getOperand(1));
    5305             :   // Constant mul/udiv/shift amount from the RHS of the shift's LHS op.
    5306           0 :   ConstantSDNode *OppLHSCst = isConstOrConstSplat(OppShiftLHS.getOperand(1));
    5307             :   // Constant mul/udiv/shift amount from the RHS of the ExtractFrom op.
    5308             :   ConstantSDNode *ExtractFromCst =
    5309           0 :       isConstOrConstSplat(ExtractFrom.getOperand(1));
    5310             :   // TODO: We should be able to handle non-uniform constant vectors for these values
    5311             :   // Check that we have constant values.
    5312           0 :   if (!OppShiftCst || !OppShiftCst->getAPIntValue() ||
    5313           0 :       !OppLHSCst || !OppLHSCst->getAPIntValue() ||
    5314           0 :       !ExtractFromCst || !ExtractFromCst->getAPIntValue())
    5315           0 :     return SDValue();
    5316             : 
    5317             :   // Compute the shift amount we need to extract to complete the rotate.
    5318             :   const unsigned VTWidth = ShiftedVT.getScalarSizeInBits();
    5319           0 :   if (OppShiftCst->getAPIntValue().ugt(VTWidth))
    5320           0 :     return SDValue();
    5321           0 :   APInt NeededShiftAmt = VTWidth - OppShiftCst->getAPIntValue();
    5322             :   // Normalize the bitwidth of the two mul/udiv/shift constant operands.
    5323           0 :   APInt ExtractFromAmt = ExtractFromCst->getAPIntValue();
    5324           0 :   APInt OppLHSAmt = OppLHSCst->getAPIntValue();
    5325           0 :   zeroExtendToMatch(ExtractFromAmt, OppLHSAmt);
    5326             : 
    5327             :   // Now try extract the needed shift from the ExtractFrom op and see if the
    5328             :   // result matches up with the existing shift's LHS op.
    5329           0 :   if (IsMulOrDiv) {
    5330             :     // Op to extract from is a mul or udiv by a constant.
    5331             :     // Check:
    5332             :     //     c2 / (1 << (bitwidth(op0 v c0) - c1)) == c0
    5333             :     //     c2 % (1 << (bitwidth(op0 v c0) - c1)) == 0
    5334             :     const APInt ExtractDiv = APInt::getOneBitSet(ExtractFromAmt.getBitWidth(),
    5335           0 :                                                  NeededShiftAmt.getZExtValue());
    5336             :     APInt ResultAmt;
    5337             :     APInt Rem;
    5338           0 :     APInt::udivrem(ExtractFromAmt, ExtractDiv, ResultAmt, Rem);
    5339           0 :     if (Rem != 0 || ResultAmt != OppLHSAmt)
    5340           0 :       return SDValue();
    5341             :   } else {
    5342             :     // Op to extract from is a shift by a constant.
    5343             :     // Check:
    5344             :     //      c2 - (bitwidth(op0 v c0) - c1) == c0
    5345           0 :     if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.zextOrTrunc(
    5346             :                                           ExtractFromAmt.getBitWidth()))
    5347           0 :       return SDValue();
    5348             :   }
    5349             : 
    5350             :   // Return the expanded shift op that should allow a rotate to be formed.
    5351           0 :   EVT ShiftVT = OppShift.getOperand(1).getValueType();
    5352           0 :   EVT ResVT = ExtractFrom.getValueType();
    5353           0 :   SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
    5354           0 :   return DAG.getNode(Opcode, DL, ResVT, OppShiftLHS, NewShiftNode);
    5355             : }
    5356             : 
    5357             : // Return true if we can prove that, whenever Neg and Pos are both in the
    5358             : // range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos).  This means that
    5359             : // for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
    5360             : //
    5361             : //     (or (shift1 X, Neg), (shift2 X, Pos))
    5362             : //
    5363             : // reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
    5364             : // in direction shift1 by Neg.  The range [0, EltSize) means that we only need
    5365             : // to consider shift amounts with defined behavior.
    5366         546 : static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
    5367             :                            SelectionDAG &DAG) {
    5368             :   // If EltSize is a power of 2 then:
    5369             :   //
    5370             :   //  (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
    5371             :   //  (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
    5372             :   //
    5373             :   // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
    5374             :   // for the stronger condition:
    5375             :   //
    5376             :   //     Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1)    [A]
    5377             :   //
    5378             :   // for all Neg and Pos.  Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
    5379             :   // we can just replace Neg with Neg' for the rest of the function.
    5380             :   //
    5381             :   // In other cases we check for the even stronger condition:
    5382             :   //
    5383             :   //     Neg == EltSize - Pos                                    [B]
    5384             :   //
    5385             :   // for all Neg and Pos.  Note that the (or ...) then invokes undefined
    5386             :   // behavior if Pos == 0 (and consequently Neg == EltSize).
    5387             :   //
    5388             :   // We could actually use [A] whenever EltSize is a power of 2, but the
    5389             :   // only extra cases that it would match are those uninteresting ones
    5390             :   // where Neg and Pos are never in range at the same time.  E.g. for
    5391             :   // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
    5392             :   // as well as (sub 32, Pos), but:
    5393             :   //
    5394             :   //     (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
    5395             :   //
    5396             :   // always invokes undefined behavior for 32-bit X.
    5397             :   //
    5398             :   // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
    5399             :   unsigned MaskLoBits = 0;
    5400         546 :   if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
    5401          83 :     if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
    5402          83 :       KnownBits Known;
    5403          83 :       DAG.computeKnownBits(Neg.getOperand(0), Known);
    5404             :       unsigned Bits = Log2_64(EltSize);
    5405         246 :       if (NegC->getAPIntValue().getActiveBits() <= Bits &&
    5406         243 :           ((NegC->getAPIntValue() | Known.Zero).countTrailingOnes() >= Bits)) {
    5407          78 :         Neg = Neg.getOperand(0);
    5408             :         MaskLoBits = Bits;
    5409             :       }
    5410             :     }
    5411             :   }
    5412             : 
    5413             :   // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
    5414         546 :   if (Neg.getOpcode() != ISD::SUB)
    5415             :     return false;
    5416         400 :   ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
    5417         400 :   if (!NegC)
    5418             :     return false;
    5419         400 :   SDValue NegOp1 = Neg.getOperand(1);
    5420             : 
    5421             :   // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
    5422             :   // Pos'.  The truncation is redundant for the purpose of the equality.
    5423         400 :   if (MaskLoBits && Pos.getOpcode() == ISD::AND) {
    5424          54 :     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1))) {
    5425          54 :       KnownBits Known;
    5426          54 :       DAG.computeKnownBits(Pos.getOperand(0), Known);
    5427         162 :       if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
    5428         162 :           ((PosC->getAPIntValue() | Known.Zero).countTrailingOnes() >=
    5429             :            MaskLoBits))
    5430          42 :         Pos = Pos.getOperand(0);
    5431             :     }
    5432             :   }
    5433             : 
    5434             :   // The condition we need is now:
    5435             :   //
    5436             :   //     (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
    5437             :   //
    5438             :   // If NegOp1 == Pos then we need:
    5439             :   //
    5440             :   //              EltSize & Mask == NegC & Mask
    5441             :   //
    5442             :   // (because "x & Mask" is a truncation and distributes through subtraction).
    5443             :   APInt Width;
    5444             :   if (Pos == NegOp1)
    5445         780 :     Width = NegC->getAPIntValue();
    5446             : 
    5447             :   // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
    5448             :   // Then the condition we want to prove becomes:
    5449             :   //
    5450             :   //     (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
    5451             :   //
    5452             :   // which, again because "x & Mask" is a truncation, becomes:
    5453             :   //
    5454             :   //                NegC & Mask == (EltSize - PosC) & Mask
    5455             :   //             EltSize & Mask == (NegC + PosC) & Mask
    5456          10 :   else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
    5457           3 :     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
    5458           9 :       Width = PosC->getAPIntValue() + NegC->getAPIntValue();
    5459             :     else
    5460             :       return false;
    5461             :   } else
    5462             :     return false;
    5463             : 
    5464             :   // Now we just need to check that EltSize & Mask == Width & Mask.
    5465         393 :   if (MaskLoBits)
    5466             :     // EltSize & Mask is 0 since Mask is EltSize - 1.
    5467          60 :     return Width.getLoBits(MaskLoBits) == 0;
    5468         333 :   return Width == EltSize;
    5469             : }
    5470             : 
    5471             : // A subroutine of MatchRotate used once we have found an OR of two opposite
    5472             : // shifts of Shifted.  If Neg == <operand size> - Pos then the OR reduces
    5473             : // to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
    5474             : // former being preferred if supported.  InnerPos and InnerNeg are Pos and
    5475             : // Neg with outer conversions stripped away.
    5476           0 : SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
    5477             :                                        SDValue Neg, SDValue InnerPos,
    5478             :                                        SDValue InnerNeg, unsigned PosOpcode,
    5479             :                                        unsigned NegOpcode, const SDLoc &DL) {
    5480             :   // fold (or (shl x, (*ext y)),
    5481             :   //          (srl x, (*ext (sub 32, y)))) ->
    5482             :   //   (rotl x, y) or (rotr x, (sub 32, y))
    5483             :   //
    5484             :   // fold (or (shl x, (*ext (sub 32, y))),
    5485             :   //          (srl x, (*ext y))) ->
    5486             :   //   (rotr x, y) or (rotl x, (sub 32, y))
    5487           0 :   EVT VT = Shifted.getValueType();
    5488           0 :   if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
    5489           0 :     bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
    5490           0 :     return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
    5491           0 :                        HasPos ? Pos : Neg).getNode();
    5492             :   }
    5493             : 
    5494             :   return nullptr;
    5495             : }
    5496             : 
    5497             : // MatchRotate - Handle an 'or' of two operands.  If this is one of the many
    5498             : // idioms for rotate, and if the target supports rotation instructions, generate
    5499             : // a rot[lr].
    5500      142856 : SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
    5501             :   // Must be a legal type.  Expanded 'n promoted things won't work with rotates.
    5502      142856 :   EVT VT = LHS.getValueType();
    5503      142856 :   if (!TLI.isTypeLegal(VT)) return nullptr;
    5504             : 
    5505             :   // The target must have at least one rotate flavor.
    5506      140206 :   bool HasROTL = hasOperation(ISD::ROTL, VT);
    5507      140206 :   bool HasROTR = hasOperation(ISD::ROTR, VT);
    5508      140206 :   if (!HasROTL && !HasROTR) return nullptr;
    5509             : 
    5510             :   // Check for truncated rotate.
    5511      113053 :   if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
    5512         314 :       LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
    5513             :     assert(LHS.getValueType() == RHS.getValueType());
    5514         157 :     if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
    5515           1 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
    5516           1 :                          SDValue(Rot, 0)).getNode();
    5517             :     }
    5518             :   }
    5519             : 
    5520             :   // Match "(X shl/srl V1) & V2" where V2 may not be present.
    5521      113052 :   SDValue LHSShift;   // The shift.
    5522      113052 :   SDValue LHSMask;    // AND value if any.
    5523      113052 :   matchRotateHalf(DAG, LHS, LHSShift, LHSMask);
    5524             : 
    5525      113052 :   SDValue RHSShift;   // The shift.
    5526      113052 :   SDValue RHSMask;    // AND value if any.
    5527      113052 :   matchRotateHalf(DAG, RHS, RHSShift, RHSMask);
    5528             : 
    5529             :   // If neither side matched a rotate half, bail
    5530      113052 :   if (!LHSShift && !RHSShift)
    5531             :     return nullptr;
    5532             : 
    5533             :   // InstCombine may have combined a constant shl, srl, mul, or udiv with one
    5534             :   // side of the rotate, so try to handle that here. In all cases we need to
    5535             :   // pass the matched shift from the opposite side to compute the opcode and
    5536             :   // needed shift amount to extract.  We still want to do this if both sides
    5537             :   // matched a rotate half because one half may be a potential overshift that
    5538             :   // can be broken down (ie if InstCombine merged two shl or srl ops into a
    5539             :   // single one).
    5540             : 
    5541             :   // Have LHS side of the rotate, try to extract the needed shift from the RHS.
    5542       33445 :   if (LHSShift)
    5543       10248 :     if (SDValue NewRHSShift =
    5544       10248 :             extractShiftForRotate(DAG, LHSShift, RHS, RHSMask, DL))
    5545          12 :       RHSShift = NewRHSShift;
    5546             :   // Have RHS side of the rotate, try to extract the needed shift from the LHS.
    5547       33445 :   if (RHSShift)
    5548       29162 :     if (SDValue NewLHSShift =
    5549       29162 :             extractShiftForRotate(DAG, RHSShift, LHS, LHSMask, DL))
    5550          11 :       LHSShift = NewLHSShift;
    5551             : 
    5552             :   // If a side is still missing, nothing else we can do.
    5553       33445 :   if (!RHSShift || !LHSShift)
    5554             :     return nullptr;
    5555             : 
    5556             :   // At this point we've matched or extracted a shift op on each side.
    5557             : 
    5558             :   if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
    5559             :     return nullptr;   // Not shifting the same value.
    5560             : 
    5561        1365 :   if (LHSShift.getOpcode() == RHSShift.getOpcode())
    5562             :     return nullptr;   // Shifts must disagree.
    5563             : 
    5564             :   // Canonicalize shl to left side in a shl/srl pair.
    5565        1186 :   if (RHSShift.getOpcode() == ISD::SHL) {
    5566             :     std::swap(LHS, RHS);
    5567             :     std::swap(LHSShift, RHSShift);
    5568             :     std::swap(LHSMask, RHSMask);
    5569             :   }
    5570             : 
    5571             :   unsigned EltSizeInBits = VT.getScalarSizeInBits();
    5572        1186 :   SDValue LHSShiftArg = LHSShift.getOperand(0);
    5573        1186 :   SDValue LHSShiftAmt = LHSShift.getOperand(1);
    5574        1186 :   SDValue RHSShiftArg = RHSShift.getOperand(0);
    5575        1186 :   SDValue RHSShiftAmt = RHSShift.getOperand(1);
    5576             : 
    5577             :   // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
    5578             :   // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
    5579             :   auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
    5580             :                                         ConstantSDNode *RHS) {
    5581             :     return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
    5582             :   };
    5583        2372 :   if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
    5584         723 :     SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
    5585        1033 :                               LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
    5586             : 
    5587             :     // If there is an AND of either shifted operand, apply it to the result.
    5588         723 :     if (LHSMask.getNode() || RHSMask.getNode()) {
    5589         104 :       SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
    5590         104 :       SDValue Mask = AllOnes;
    5591             : 
    5592         104 :       if (LHSMask.getNode()) {
    5593         208 :         SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
    5594         104 :         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
    5595         104 :                            DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
    5596             :       }
    5597         104 :       if (RHSMask.getNode()) {
    5598         174 :         SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
    5599          87 :         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
    5600          87 :                            DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
    5601             :       }
    5602             : 
    5603         208 :       Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
    5604             :     }
    5605             : 
    5606             :     return Rot.getNode();
    5607             :   }
    5608             : 
    5609             :   // If there is a mask here, and we have a variable shift, we can't be sure
    5610             :   // that we're masking out the right stuff.
    5611         463 :   if (LHSMask.getNode() || RHSMask.getNode())
    5612             :     return nullptr;
    5613             : 
    5614             :   // If the shift amount is sign/zext/any-extended just peel it off.
    5615         424 :   SDValue LExtOp0 = LHSShiftAmt;
    5616         424 :   SDValue RExtOp0 = RHSShiftAmt;
    5617         424 :   if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
    5618         413 :        LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
    5619         413 :        LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
    5620         511 :        LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
    5621          87 :       (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
    5622          76 :        RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
    5623          76 :        RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
    5624             :        RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
    5625          79 :     LExtOp0 = LHSShiftAmt.getOperand(0);
    5626          79 :     RExtOp0 = RHSShiftAmt.getOperand(0);
    5627             :   }
    5628             : 
    5629         424 :   SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
    5630             :                                    LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
    5631         424 :   if (TryL)
    5632             :     return TryL;
    5633             : 
    5634         122 :   SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
    5635             :                                    RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
    5636         122 :   if (TryR)
    5637          87 :     return TryR;
    5638             : 
    5639             :   return nullptr;
    5640             : }
    5641             : 
    5642             : namespace {
    5643             : 
    5644             : /// Represents known origin of an individual byte in load combine pattern. The
    5645             : /// value of the byte is either constant zero or comes from memory.
    5646             : struct ByteProvider {
    5647             :   // For constant zero providers Load is set to nullptr. For memory providers
    5648             :   // Load represents the node which loads the byte from memory.
    5649             :   // ByteOffset is the offset of the byte in the value produced by the load.
    5650             :   LoadSDNode *Load = nullptr;
    5651             :   unsigned ByteOffset = 0;
    5652             : 
    5653             :   ByteProvider() = default;
    5654             : 
    5655             :   static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
    5656             :     return ByteProvider(Load, ByteOffset);
    5657             :   }
    5658             : 
    5659             :   static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
    5660             : 
    5661           0 :   bool isConstantZero() const { return !Load; }
    5662           0 :   bool isMemory() const { return Load; }
    5663             : 
    5664             :   bool operator==(const ByteProvider &Other) const {
    5665             :     return Other.Load == Load && Other.ByteOffset == ByteOffset;
    5666             :   }
    5667             : 
    5668             : private:
    5669             :   ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
    5670             :       : Load(Load), ByteOffset(ByteOffset) {}
    5671             : };
    5672             : 
    5673             : } // end anonymous namespace
    5674             : 
    5675             : /// Recursively traverses the expression calculating the origin of the requested
    5676             : /// byte of the given value. Returns None if the provider can't be calculated.
    5677             : ///
    5678             : /// For all the values except the root of the expression verifies that the value
    5679             : /// has exactly one use and if it's not true return None. This way if the origin
    5680             : /// of the byte is returned it's guaranteed that the values which contribute to
    5681             : /// the byte are not used outside of this expression.
    5682             : ///
    5683             : /// Because the parts of the expression are not allowed to have more than one
    5684             : /// use this function iterates over trees, not DAGs. So it never visits the same
    5685             : /// node more than once.
    5686             : static const Optional<ByteProvider>
    5687      293077 : calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
    5688             :                       bool Root = false) {
    5689             :   // Typical i64 by i8 pattern requires recursion up to 8 calls depth
    5690      293077 :   if (Depth == 10)
    5691             :     return None;
    5692             : 
    5693      289963 :   if (!Root && !Op.hasOneUse())
    5694             :     return None;
    5695             : 
    5696             :   assert(Op.getValueType().isScalarInteger() && "can't handle other types");
    5697      243935 :   unsigned BitWidth = Op.getValueSizeInBits();
    5698      243935 :   if (BitWidth % 8 != 0)
    5699             :     return None;
    5700      243935 :   unsigned ByteWidth = BitWidth / 8;
    5701             :   assert(Index < ByteWidth && "invalid index requested");
    5702             :   (void) ByteWidth;
    5703             : 
    5704      487870 :   switch (Op.getOpcode()) {
    5705      154519 :   case ISD::OR: {
    5706      309038 :     auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
    5707      154519 :     if (!LHS)
    5708             :       return None;
    5709       32604 :     auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
    5710       16302 :     if (!RHS)
    5711             :       return None;
    5712             : 
    5713       12346 :     if (LHS->isConstantZero())
    5714             :       return RHS;
    5715        6219 :     if (RHS->isConstantZero())
    5716             :       return LHS;
    5717             :     return None;
    5718             :   }
    5719       15857 :   case ISD::SHL: {
    5720       15857 :     auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
    5721             :     if (!ShiftOp)
    5722             :       return None;
    5723             : 
    5724       15360 :     uint64_t BitShift = ShiftOp->getZExtValue();
    5725       15360 :     if (BitShift % 8 != 0)
    5726             :       return None;
    5727       12520 :     uint64_t ByteShift = BitShift / 8;
    5728             : 
    5729       12520 :     return Index < ByteShift
    5730             :                ? ByteProvider::getConstantZero()
    5731             :                : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
    5732       12520 :                                        Depth + 1);
    5733             :   }
    5734       12218 :   case ISD::ANY_EXTEND:
    5735             :   case ISD::SIGN_EXTEND:
    5736             :   case ISD::ZERO_EXTEND: {
    5737       12218 :     SDValue NarrowOp = Op->getOperand(0);
    5738       12218 :     unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
    5739       12218 :     if (NarrowBitWidth % 8 != 0)
    5740             :       return None;
    5741       12148 :     uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    5742             : 
    5743       12148 :     if (Index >= NarrowByteWidth)
    5744        1565 :       return Op.getOpcode() == ISD::ZERO_EXTEND
    5745             :                  ? Optional<ByteProvider>(ByteProvider::getConstantZero())
    5746        1565 :                  : None;
    5747       10583 :     return calculateByteProvider(NarrowOp, Index, Depth + 1);
    5748             :   }
    5749          42 :   case ISD::BSWAP:
    5750          84 :     return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
    5751          84 :                                  Depth + 1);
    5752       16760 :   case ISD::LOAD: {
    5753             :     auto L = cast<LoadSDNode>(Op.getNode());
    5754       16760 :     if (L->isVolatile() || L->isIndexed())
    5755             :       return None;
    5756             : 
    5757       16674 :     unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
    5758       16674 :     if (NarrowBitWidth % 8 != 0)
    5759             :       return None;
    5760       16674 :     uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    5761             : 
    5762       16674 :     if (Index >= NarrowByteWidth)
    5763             :       return L->getExtensionType() == ISD::ZEXTLOAD
    5764             :                  ? Optional<ByteProvider>(ByteProvider::getConstantZero())
    5765        4496 :                  : None;
    5766             :     return ByteProvider::getMemory(L, Index);
    5767             :   }
    5768             :   }
    5769             : 
    5770             :   return None;
    5771             : }
    5772             : 
    5773             : /// Match a pattern where a wide type scalar value is loaded by several narrow
    5774             : /// loads and combined by shifts and ors. Fold it into a single load or a load
    5775             : /// and a BSWAP if the targets supports it.
    5776             : ///
    5777             : /// Assuming little endian target:
    5778             : ///  i8 *a = ...
    5779             : ///  i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
    5780             : /// =>
    5781             : ///  i32 val = *((i32)a)
    5782             : ///
    5783             : ///  i8 *a = ...
    5784             : ///  i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
    5785             : /// =>
    5786             : ///  i32 val = BSWAP(*((i32)a))
    5787             : ///
    5788             : /// TODO: This rule matches complex patterns with OR node roots and doesn't
    5789             : /// interact well with the worklist mechanism. When a part of the pattern is
    5790             : /// updated (e.g. one of the loads) its direct users are put into the worklist,
    5791             : /// but the root node of the pattern which triggers the load combine is not
    5792             : /// necessarily a direct user of the changed node. For example, once the address
    5793             : /// of t28 load is reassociated load combine won't be triggered:
    5794             : ///             t25: i32 = add t4, Constant:i32<2>
    5795             : ///           t26: i64 = sign_extend t25
    5796             : ///        t27: i64 = add t2, t26
    5797             : ///       t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
    5798             : ///     t29: i32 = zero_extend t28
    5799             : ///   t32: i32 = shl t29, Constant:i8<8>
    5800             : /// t33: i32 = or t23, t32
    5801             : /// As a possible fix visitLoad can check if the load can be a part of a load
    5802             : /// combine pattern and add corresponding OR roots to the worklist.
    5803           0 : SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
    5804             :   assert(N->getOpcode() == ISD::OR &&
    5805             :          "Can only match load combining against OR nodes");
    5806             : 
    5807             :   // Handles simple types only
    5808           0 :   EVT VT = N->getValueType(0);
    5809           0 :   if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
    5810           0 :     return SDValue();
    5811           0 :   unsigned ByteWidth = VT.getSizeInBits() / 8;
    5812             : 
    5813           0 :   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    5814             :   // Before legalize we can introduce too wide illegal loads which will be later
    5815             :   // split into legal sized loads. This enables us to combine i64 load by i8
    5816             :   // patterns to a couple of i32 loads on 32 bit targets.
    5817           0 :   if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
    5818           0 :     return SDValue();
    5819             : 
    5820             :   std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
    5821             :     unsigned BW, unsigned i) { return i; };
    5822             :   std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
    5823        4274 :     unsigned BW, unsigned i) { return BW - i - 1; };
    5824             : 
    5825           0 :   bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
    5826             :   auto MemoryByteOffset = [&] (ByteProvider P) {
    5827             :     assert(P.isMemory() && "Must be a memory byte provider");
    5828             :     unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
    5829             :     assert(LoadBitWidth % 8 == 0 &&
    5830             :            "can only analyze providers for individual bytes not bit");
    5831             :     unsigned LoadByteWidth = LoadBitWidth / 8;
    5832             :     return IsBigEndianTarget
    5833             :             ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
    5834             :             : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
    5835           0 :   };
    5836             : 
    5837             :   Optional<BaseIndexOffset> Base;
    5838             :   SDValue Chain;
    5839             : 
    5840             :   SmallPtrSet<LoadSDNode *, 8> Loads;
    5841             :   Optional<ByteProvider> FirstByteProvider;
    5842             :   int64_t FirstOffset = INT64_MAX;
    5843             : 
    5844             :   // Check if all the bytes of the OR we are looking at are loaded from the same
    5845             :   // base address. Collect bytes offsets from Base address in ByteOffsets.
    5846           0 :   SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
    5847           0 :   for (unsigned i = 0; i < ByteWidth; i++) {
    5848           0 :     auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
    5849           0 :     if (!P || !P->isMemory()) // All the bytes must be loaded from memory
    5850           0 :       return SDValue();
    5851             : 
    5852             :     LoadSDNode *L = P->Load;
    5853             :     assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&
    5854             :            "Must be enforced by calculateByteProvider");
    5855             :     assert(L->getOffset().isUndef() && "Unindexed load must have undef offset");
    5856             : 
    5857             :     // All loads must share the same chain
    5858           0 :     SDValue LChain = L->getChain();
    5859           0 :     if (!Chain)
    5860           0 :       Chain = LChain;
    5861             :     else if (Chain != LChain)
    5862           0 :       return SDValue();
    5863             : 
    5864             :     // Loads must share the same base address
    5865           0 :     BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
    5866           0 :     int64_t ByteOffsetFromBase = 0;
    5867           0 :     if (!Base)
    5868             :       Base = Ptr;
    5869           0 :     else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
    5870           0 :       return SDValue();
    5871             : 
    5872             :     // Calculate the offset of the current byte from the base address
    5873           0 :     ByteOffsetFromBase += MemoryByteOffset(*P);
    5874           0 :     ByteOffsets[i] = ByteOffsetFromBase;
    5875             : 
    5876             :     // Remember the first byte load
    5877           0 :     if (ByteOffsetFromBase < FirstOffset) {
    5878             :       FirstByteProvider = P;
    5879             :       FirstOffset = ByteOffsetFromBase;
    5880             :     }
    5881             : 
    5882           0 :     Loads.insert(L);
    5883             :   }
    5884             :   assert(!Loads.empty() && "All the bytes of the value must be loaded from "
    5885             :          "memory, so there must be at least one load which produces the value");
    5886             :   assert(Base && "Base address of the accessed memory location must be set");
    5887             :   assert(FirstOffset != INT64_MAX && "First byte offset must be set");
    5888             : 
    5889             :   // Check if the bytes of the OR we are looking at match with either big or
    5890             :   // little endian value load
    5891             :   bool BigEndian = true, LittleEndian = true;
    5892           0 :   for (unsigned i = 0; i < ByteWidth; i++) {
    5893           0 :     int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
    5894           0 :     LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
    5895           0 :     BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
    5896           0 :     if (!BigEndian && !LittleEndian)
    5897           0 :       return SDValue();
    5898             :   }
    5899             :   assert((BigEndian != LittleEndian) && "should be either or");
    5900             :   assert(FirstByteProvider && "must be set");
    5901             : 
    5902             :   // Ensure that the first byte is loaded from zero offset of the first load.
    5903             :   // So the combined value can be loaded from the first load address.
    5904           0 :   if (MemoryByteOffset(*FirstByteProvider) != 0)
    5905           0 :     return SDValue();
    5906             :   LoadSDNode *FirstLoad = FirstByteProvider->Load;
    5907             : 
    5908             :   // The node we are looking at matches with the pattern, check if we can
    5909             :   // replace it with a single load and bswap if needed.
    5910             : 
    5911             :   // If the load needs byte swap check if the target supports it
    5912           0 :   bool NeedsBswap = IsBigEndianTarget != BigEndian;
    5913             : 
    5914             :   // Before legalize we can introduce illegal bswaps which will be later
    5915             :   // converted to an explicit bswap sequence. This way we end up with a single
    5916             :   // load and byte shuffling instead of several loads and byte shuffling.
    5917           0 :   if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
    5918           0 :     return SDValue();
    5919             : 
    5920             :   // Check that a load of the wide type is both allowed and fast on the target
    5921           0 :   bool Fast = false;
    5922           0 :   bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
    5923             :                                         VT, FirstLoad->getAddressSpace(),
    5924             :                                         FirstLoad->getAlignment(), &Fast);
    5925           0 :   if (!Allowed || !Fast)
    5926           0 :     return SDValue();
    5927             : 
    5928             :   SDValue NewLoad =
    5929           0 :       DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
    5930           0 :                   FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
    5931             : 
    5932             :   // Transfer chain users from old loads to the new load.
    5933           0 :   for (LoadSDNode *L : Loads)
    5934           0 :     DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
    5935             : 
    5936           0 :   return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
    5937             : }
    5938             : 
    5939             : // If the target has andn, bsl, or a similar bit-select instruction,
    5940             : // we want to unfold masked merge, with canonical pattern of:
    5941             : //   |        A  |  |B|
    5942             : //   ((x ^ y) & m) ^ y
    5943             : //    |  D  |
    5944             : // Into:
    5945             : //   (x & m) | (y & ~m)
    5946             : // If y is a constant, and the 'andn' does not work with immediates,
    5947             : // we unfold into a different pattern:
    5948             : //   ~(~x & m) & (m | y)
    5949             : // NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
    5950             : //       the very least that breaks andnpd / andnps patterns, and because those
    5951             : //       patterns are simplified in IR and shouldn't be created in the DAG
    5952           0 : SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
    5953             :   assert(N->getOpcode() == ISD::XOR);
    5954             : 
    5955             :   // Don't touch 'not' (i.e. where y = -1).
    5956           0 :   if (isAllOnesConstantOrAllOnesSplatConstant(N->getOperand(1)))
    5957           0 :     return SDValue();
    5958             : 
    5959           0 :   EVT VT = N->getValueType(0);
    5960             : 
    5961             :   // There are 3 commutable operators in the pattern,
    5962             :   // so we have to deal with 8 possible variants of the basic pattern.
    5963           0 :   SDValue X, Y, M;
    5964             :   auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
    5965             :     if (And.getOpcode() != ISD::AND || !And.hasOneUse())
    5966             :       return false;
    5967             :     SDValue Xor = And.getOperand(XorIdx);
    5968             :     if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
    5969             :       return false;
    5970             :     SDValue Xor0 = Xor.getOperand(0);
    5971             :     SDValue Xor1 = Xor.getOperand(1);
    5972             :     // Don't touch 'not' (i.e. where y = -1).
    5973             :     if (isAllOnesConstantOrAllOnesSplatConstant(Xor1))
    5974             :       return false;
    5975             :     if (Other == Xor0)
    5976             :       std::swap(Xor0, Xor1);
    5977             :     if (Other != Xor1)
    5978             :       return false;
    5979             :     X = Xor0;
    5980             :     Y = Xor1;
    5981             :     M = And.getOperand(XorIdx ? 0 : 1);
    5982             :     return true;
    5983           0 :   };
    5984             : 
    5985           0 :   SDValue N0 = N->getOperand(0);
    5986           0 :   SDValue N1 = N->getOperand(1);
    5987           0 :   if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
    5988           0 :       !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
    5989           0 :     return SDValue();
    5990             : 
    5991             :   // Don't do anything if the mask is constant. This should not be reachable.
    5992             :   // InstCombine should have already unfolded this pattern, and DAGCombiner
    5993             :   // probably shouldn't produce it, too.
    5994           0 :   if (isa<ConstantSDNode>(M.getNode()))
    5995           0 :     return SDValue();
    5996             : 
    5997             :   // We can transform if the target has AndNot
    5998           0 :   if (!TLI.hasAndNot(M))
    5999           0 :     return SDValue();
    6000             : 
    6001             :   SDLoc DL(N);
    6002             : 
    6003             :   // If Y is a constant, check that 'andn' works with immediates.
    6004           0 :   if (!TLI.hasAndNot(Y)) {
    6005             :     assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.");
    6006             :     // If not, we need to do a bit more work to make sure andn is still used.
    6007           0 :     SDValue NotX = DAG.getNOT(DL, X, VT);
    6008           0 :     SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
    6009           0 :     SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
    6010           0 :     SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
    6011           0 :     return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
    6012             :   }
    6013             : 
    6014           0 :   SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
    6015           0 :   SDValue NotM = DAG.getNOT(DL, M, VT);
    6016           0 :   SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
    6017             : 
    6018           0 :   return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
    6019             : }
    6020             : 
    6021      117833 : SDValue DAGCombiner::visitXOR(SDNode *N) {
    6022      117833 :   SDValue N0 = N->getOperand(0);
    6023      117833 :   SDValue N1 = N->getOperand(1);
    6024      235666 :   EVT VT = N0.getValueType();
    6025             : 
    6026             :   // fold vector ops
    6027      117833 :   if (VT.isVector()) {
    6028       24850 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6029           0 :       return FoldedVOp;
    6030             : 
    6031             :     // fold (xor x, 0) -> x, vector edition
    6032       24850 :     if (ISD::isBuildVectorAllZeros(N0.getNode()))
    6033           0 :       return N1;
    6034       24850 :     if (ISD::isBuildVectorAllZeros(N1.getNode()))
    6035         110 :       return N0;
    6036             :   }
    6037             : 
    6038             :   // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
    6039      117723 :   if (N0.isUndef() && N1.isUndef())
    6040           0 :     return DAG.getConstant(0, SDLoc(N), VT);
    6041             :   // fold (xor x, undef) -> undef
    6042      117723 :   if (N0.isUndef())
    6043          14 :     return N0;
    6044      235418 :   if (N1.isUndef())
    6045           0 :     return N1;
    6046             :   // fold (xor c1, c2) -> c1^c2
    6047             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6048             :   ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
    6049      117709 :   if (N0C && N1C)
    6050        1808 :     return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
    6051             :   // canonicalize constant to RHS
    6052      117331 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
    6053         526 :      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    6054        1046 :     return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
    6055             :   // fold (xor x, 0) -> x
    6056      116282 :   if (isNullConstant(N1))
    6057           0 :     return N0;
    6058             : 
    6059      116282 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6060           6 :     return NewSel;
    6061             : 
    6062             :   // reassociate xor
    6063      165686 :   if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1, N->getFlags()))
    6064         288 :     return RXOR;
    6065             : 
    6066             :   // fold !(x cc y) -> (x !cc y)
    6067      115988 :   SDValue LHS, RHS, CC;
    6068      115988 :   if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
    6069       22739 :     bool isInt = LHS.getValueType().isInteger();
    6070       45478 :     ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
    6071             :                                                isInt);
    6072             : 
    6073       22739 :     if (!LegalOperations ||
    6074         121 :         TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
    6075       22625 :       switch (N0.getOpcode()) {
    6076           0 :       default:
    6077           0 :         llvm_unreachable("Unhandled SetCC Equivalent!");
    6078       22625 :       case ISD::SETCC:
    6079       58896 :         return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
    6080           0 :       case ISD::SELECT_CC:
    6081           0 :         return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
    6082           0 :                                N0.getOperand(3), NotCC);
    6083             :       }
    6084             :     }
    6085             :   }
    6086             : 
    6087             :   // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
    6088      135505 :   if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
    6089       93370 :       N0.getNode()->hasOneUse() &&
    6090           7 :       isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
    6091           3 :     SDValue V = N0.getOperand(0);
    6092             :     SDLoc DL(N0);
    6093           3 :     V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
    6094           3 :                     DAG.getConstant(1, DL, V.getValueType()));
    6095           3 :     AddToWorklist(V.getNode());
    6096           6 :     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
    6097             :   }
    6098             : 
    6099             :   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
    6100      112996 :   if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
    6101       19310 :       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
    6102          82 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    6103          82 :     if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
    6104          53 :       unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
    6105          55 :       LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
    6106          55 :       RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
    6107          53 :       AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
    6108         108 :       return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
    6109             :     }
    6110             :   }
    6111             :   // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
    6112       93307 :   if (isAllOnesConstant(N1) && N0.hasOneUse() &&
    6113       46654 :       (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
    6114         612 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    6115             :     if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
    6116          64 :       unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
    6117          65 :       LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
    6118          64 :       RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
    6119          64 :       AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
    6120         129 :       return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
    6121             :     }
    6122             :   }
    6123             :   // fold (xor (and x, y), y) -> (and (not x), y)
    6124      186486 :   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
    6125        1890 :       N0->getOperand(1) == N1) {
    6126          73 :     SDValue X = N0->getOperand(0);
    6127          88 :     SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
    6128          73 :     AddToWorklist(NotX.getNode());
    6129         161 :     return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
    6130             :   }
    6131             : 
    6132             :   // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
    6133       93170 :   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
    6134       19117 :     SDValue A = N0.getOpcode() == ISD::ADD ? N0 : N1;
    6135       38134 :     SDValue S = N0.getOpcode() == ISD::SRA ? N0 : N1;
    6136       19117 :     if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
    6137         119 :       SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
    6138         119 :       SDValue S0 = S.getOperand(0);
    6139             :       if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0)) {
    6140             :         unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6141         113 :         if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
    6142         222 :           if (C->getAPIntValue() == (OpSizeInBits - 1))
    6143         225 :             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
    6144             :       }
    6145             :     }
    6146             :   }
    6147             : 
    6148             :   // fold (xor x, x) -> 0
    6149       93059 :   if (N0 == N1)
    6150          38 :     return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
    6151             : 
    6152             :   // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
    6153             :   // Here is a concrete example of this equivalence:
    6154             :   // i16   x ==  14
    6155             :   // i16 shl ==   1 << 14  == 16384 == 0b0100000000000000
    6156             :   // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
    6157             :   //
    6158             :   // =>
    6159             :   //
    6160             :   // i16     ~1      == 0b1111111111111110
    6161             :   // i16 rol(~1, 14) == 0b1011111111111111
    6162             :   //
    6163             :   // Some additional tips to help conceptualize this transform:
    6164             :   // - Try to see the operation as placing a single zero in a value of all ones.
    6165             :   // - There exists no value for x which would allow the result to contain zero.
    6166             :   // - Values of x larger than the bitwidth are undefined and do not require a
    6167             :   //   consistent result.
    6168             :   // - Pushing the zero left requires shifting one bits in from the right.
    6169             :   // A rotate left of ~1 is a nice way of achieving the desired result.
    6170      143541 :   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
    6171         632 :       && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
    6172             :     SDLoc DL(N);
    6173          79 :     return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
    6174          79 :                        N0.getOperand(1));
    6175             :   }
    6176             : 
    6177             :   // Simplify: xor (op x...), (op y...)  -> (op (xor x, y))
    6178       92961 :   if (N0.getOpcode() == N1.getOpcode())
    6179       14346 :     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
    6180         438 :       return Tmp;
    6181             : 
    6182             :   // Unfold  ((x ^ y) & m) ^ y  into  (x & m) | (y & ~m)  if profitable
    6183       92523 :   if (SDValue MM = unfoldMaskedMerge(N))
    6184         114 :     return MM;
    6185             : 
    6186             :   // Simplify the expression using non-local knowledge.
    6187       92409 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    6188         228 :     return SDValue(N, 0);
    6189             : 
    6190       92181 :   return SDValue();
    6191             : }
    6192             : 
    6193             : /// Handle transforms common to the three shifts, when the shift amount is a
    6194             : /// constant.
    6195           0 : SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
    6196           0 :   SDNode *LHS = N->getOperand(0).getNode();
    6197           0 :   if (!LHS->hasOneUse()) return SDValue();
    6198             : 
    6199             :   // We want to pull some binops through shifts, so that we have (and (shift))
    6200             :   // instead of (shift (and)), likewise for add, or, xor, etc.  This sort of
    6201             :   // thing happens with address calculations, so it's important to canonicalize
    6202             :   // it.
    6203             :   bool HighBitSet = false;  // Can we transform this if the high bit is set?
    6204             : 
    6205           0 :   switch (LHS->getOpcode()) {
    6206           0 :   default: return SDValue();
    6207             :   case ISD::OR:
    6208             :   case ISD::XOR:
    6209             :     HighBitSet = false; // We can only transform sra if the high bit is clear.
    6210             :     break;
    6211           0 :   case ISD::AND:
    6212             :     HighBitSet = true;  // We can only transform sra if the high bit is set.
    6213           0 :     break;
    6214           0 :   case ISD::ADD:
    6215           0 :     if (N->getOpcode() != ISD::SHL)
    6216           0 :       return SDValue(); // only shl(add) not sr[al](add).
    6217             :     HighBitSet = false; // We can only transform sra if the high bit is clear.
    6218             :     break;
    6219             :   }
    6220             : 
    6221             :   // We require the RHS of the binop to be a constant and not opaque as well.
    6222           0 :   ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
    6223           0 :   if (!BinOpCst) return SDValue();
    6224             : 
    6225             :   // FIXME: disable this unless the input to the binop is a shift by a constant
    6226             :   // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
    6227           0 :   SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
    6228           0 :   bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
    6229           0 :                  BinOpLHSVal->getOpcode() == ISD::SRA ||
    6230             :                  BinOpLHSVal->getOpcode() == ISD::SRL;
    6231           0 :   bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
    6232             :                         BinOpLHSVal->getOpcode() == ISD::SELECT;
    6233             : 
    6234           0 :   if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
    6235             :       !isCopyOrSelect)
    6236           0 :     return SDValue();
    6237             : 
    6238           0 :   if (isCopyOrSelect && N->hasOneUse())
    6239           0 :     return SDValue();
    6240             : 
    6241           0 :   EVT VT = N->getValueType(0);
    6242             : 
    6243             :   // If this is a signed shift right, and the high bit is modified by the
    6244             :   // logical operation, do not perform the transformation. The highBitSet
    6245             :   // boolean indicates the value of the high bit of the constant which would
    6246             :   // cause it to be modified for this operation.
    6247           0 :   if (N->getOpcode() == ISD::SRA) {
    6248           0 :     bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
    6249           0 :     if (BinOpRHSSignSet != HighBitSet)
    6250           0 :       return SDValue();
    6251             :   }
    6252             : 
    6253           0 :   if (!TLI.isDesirableToCommuteWithShift(N, Level))
    6254           0 :     return SDValue();
    6255             : 
    6256             :   // Fold the constants, shifting the binop RHS by the shift amount.
    6257           0 :   SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
    6258             :                                N->getValueType(0),
    6259           0 :                                LHS->getOperand(1), N->getOperand(1));
    6260             :   assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!");
    6261             : 
    6262             :   // Create the new shift.
    6263           0 :   SDValue NewShift = DAG.getNode(N->getOpcode(),
    6264           0 :                                  SDLoc(LHS->getOperand(0)),
    6265           0 :                                  VT, LHS->getOperand(0), N->getOperand(1));
    6266             : 
    6267             :   // Create the new binop.
    6268           0 :   return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
    6269             : }
    6270             : 
    6271         844 : SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
    6272             :   assert(N->getOpcode() == ISD::TRUNCATE);
    6273             :   assert(N->getOperand(0).getOpcode() == ISD::AND);
    6274             : 
    6275             :   // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
    6276         827 :   if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
    6277         587 :     SDValue N01 = N->getOperand(0).getOperand(1);
    6278         587 :     if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
    6279             :       SDLoc DL(N);
    6280         584 :       EVT TruncVT = N->getValueType(0);
    6281         584 :       SDValue N00 = N->getOperand(0).getOperand(0);
    6282        1168 :       SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
    6283        1168 :       SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
    6284         584 :       AddToWorklist(Trunc00.getNode());
    6285         584 :       AddToWorklist(Trunc01.getNode());
    6286        1168 :       return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
    6287             :     }
    6288             :   }
    6289             : 
    6290         260 :   return SDValue();
    6291             : }
    6292             : 
    6293        2760 : SDValue DAGCombiner::visitRotate(SDNode *N) {
    6294             :   SDLoc dl(N);
    6295        2760 :   SDValue N0 = N->getOperand(0);
    6296        2760 :   SDValue N1 = N->getOperand(1);
    6297        5520 :   EVT VT = N->getValueType(0);
    6298             :   unsigned Bitsize = VT.getScalarSizeInBits();
    6299             : 
    6300             :   // fold (rot x, 0) -> x
    6301        2760 :   if (isNullConstantOrNullSplatConstant(N1))
    6302           4 :     return N0;
    6303             : 
    6304             :   // fold (rot x, c) -> (rot x, c % BitSize)
    6305        2756 :   if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
    6306        3303 :     if (Cst->getAPIntValue().uge(Bitsize)) {
    6307          10 :       uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
    6308          10 :       return DAG.getNode(N->getOpcode(), dl, VT, N0,
    6309          10 :                          DAG.getConstant(RotAmt, dl, N1.getValueType()));
    6310             :     }
    6311             :   }
    6312             : 
    6313             :   // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
    6314        2746 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6315         257 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6316          53 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6317         159 :       return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
    6318             :   }
    6319             : 
    6320             :   unsigned NextOp = N0.getOpcode();
    6321             :   // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
    6322        2693 :   if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
    6323          17 :     SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
    6324          34 :     SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
    6325          17 :     if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
    6326          17 :       EVT ShiftVT = C1->getValueType(0);
    6327          17 :       bool SameSide = (N->getOpcode() == NextOp);
    6328          17 :       unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
    6329          17 :       if (SDValue CombinedShift =
    6330          17 :               DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
    6331          17 :         SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
    6332          17 :         SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
    6333             :             ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
    6334          17 :             BitsizeC.getNode());
    6335          34 :         return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
    6336          34 :                            CombinedShiftNorm);
    6337             :       }
    6338             :     }
    6339             :   }
    6340        2676 :   return SDValue();
    6341             : }
    6342             : 
    6343      123493 : SDValue DAGCombiner::visitSHL(SDNode *N) {
    6344      123493 :   SDValue N0 = N->getOperand(0);
    6345      123493 :   SDValue N1 = N->getOperand(1);
    6346      123493 :   EVT VT = N0.getValueType();
    6347             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6348             : 
    6349             :   // fold vector ops
    6350      123493 :   if (VT.isVector()) {
    6351        3968 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6352           6 :       return FoldedVOp;
    6353             : 
    6354             :     BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
    6355             :     // If setcc produces all-one true value then:
    6356             :     // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
    6357        2476 :     if (N1CV && N1CV->isConstant()) {
    6358        2350 :       if (N0.getOpcode() == ISD::AND) {
    6359          20 :         SDValue N00 = N0->getOperand(0);
    6360          20 :         SDValue N01 = N0->getOperand(1);
    6361             :         BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
    6362             : 
    6363          19 :         if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
    6364          12 :             TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
    6365             :                 TargetLowering::ZeroOrNegativeOneBooleanContent) {
    6366           4 :           if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
    6367           8 :                                                      N01CV, N1CV))
    6368           8 :             return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
    6369             :         }
    6370             :       }
    6371             :     }
    6372             :   }
    6373             : 
    6374      123483 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6375             : 
    6376             :   // fold (shl c1, c2) -> c1<<c2
    6377             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6378      123483 :   if (N0C && N1C && !N1C->isOpaque())
    6379        7530 :     return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
    6380             :   // fold (shl 0, x) -> 0
    6381      119718 :   if (isNullConstantOrNullSplatConstant(N0))
    6382          82 :     return N0;
    6383             :   // fold (shl x, c >= size(x)) -> undef
    6384             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6385             :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6386      205922 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6387             :   };
    6388      239272 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6389          49 :     return DAG.getUNDEF(VT);
    6390             :   // fold (shl x, 0) -> x
    6391      221801 :   if (N1C && N1C->isNullValue())
    6392         443 :     return N0;
    6393             :   // fold (shl undef, x) -> 0
    6394      119144 :   if (N0.isUndef())
    6395          22 :     return DAG.getConstant(0, SDLoc(N), VT);
    6396             : 
    6397      119133 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6398          12 :     return NewSel;
    6399             : 
    6400             :   // if (shl x, c) is known to be zero, return 0
    6401      119121 :   if (DAG.MaskedValueIsZero(SDValue(N, 0),
    6402      238242 :                             APInt::getAllOnesValue(OpSizeInBits)))
    6403        4238 :     return DAG.getConstant(0, SDLoc(N), VT);
    6404             :   // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
    6405      117002 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6406        5738 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6407         725 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6408        1022 :       return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
    6409             :   }
    6410             : 
    6411      116518 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6412        6762 :     return SDValue(N, 0);
    6413             : 
    6414             :   // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
    6415      109756 :   if (N0.getOpcode() == ISD::SHL) {
    6416             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    6417             :                                           ConstantSDNode *RHS) {
    6418             :       APInt c1 = LHS->getAPIntValue();
    6419             :       APInt c2 = RHS->getAPIntValue();
    6420             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6421             :       return (c1 + c2).uge(OpSizeInBits);
    6422             :     };
    6423         746 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    6424          12 :       return DAG.getConstant(0, SDLoc(N), VT);
    6425             : 
    6426             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    6427             :                                        ConstantSDNode *RHS) {
    6428             :       APInt c1 = LHS->getAPIntValue();
    6429             :       APInt c2 = RHS->getAPIntValue();
    6430             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6431             :       return (c1 + c2).ult(OpSizeInBits);
    6432             :     };
    6433         734 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    6434             :       SDLoc DL(N);
    6435          90 :       EVT ShiftVT = N1.getValueType();
    6436         180 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    6437         180 :       return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
    6438             :     }
    6439             :   }
    6440             : 
    6441             :   // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
    6442             :   // For this to be valid, the second form must not preserve any of the bits
    6443             :   // that are shifted out by the inner shift in the first form.  This means
    6444             :   // the outer shift size must be >= the number of bits added by the ext.
    6445             :   // As a corollary, we don't care what kind of ext it is.
    6446       92780 :   if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
    6447       78960 :               N0.getOpcode() == ISD::ANY_EXTEND ||
    6448      126183 :               N0.getOpcode() == ISD::SIGN_EXTEND) &&
    6449       16523 :       N0.getOperand(0).getOpcode() == ISD::SHL) {
    6450          47 :     SDValue N0Op0 = N0.getOperand(0);
    6451          47 :     if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6452          39 :       APInt c1 = N0Op0C1->getAPIntValue();
    6453          39 :       APInt c2 = N1C->getAPIntValue();
    6454          39 :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6455             : 
    6456          39 :       EVT InnerShiftVT = N0Op0.getValueType();
    6457          39 :       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
    6458          78 :       if (c2.uge(OpSizeInBits - InnerShiftSize)) {
    6459             :         SDLoc DL(N0);
    6460          15 :         APInt Sum = c1 + c2;
    6461          15 :         if (Sum.uge(OpSizeInBits))
    6462           4 :           return DAG.getConstant(0, DL, VT);
    6463             : 
    6464          11 :         return DAG.getNode(
    6465             :             ISD::SHL, DL, VT,
    6466          22 :             DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
    6467          11 :             DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
    6468             :       }
    6469             :     }
    6470             :   }
    6471             : 
    6472             :   // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
    6473             :   // Only fold this if the inner zext has no other uses to avoid increasing
    6474             :   // the total number of instructions.
    6475      119083 :   if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
    6476        8486 :       N0.getOperand(0).getOpcode() == ISD::SRL) {
    6477             :     SDValue N0Op0 = N0.getOperand(0);
    6478         103 :     if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6479         103 :       if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
    6480         103 :         uint64_t c1 = N0Op0C1->getZExtValue();
    6481         103 :         uint64_t c2 = N1C->getZExtValue();
    6482         103 :         if (c1 == c2) {
    6483          14 :           SDValue NewOp0 = N0.getOperand(0);
    6484          28 :           EVT CountVT = NewOp0.getOperand(1).getValueType();
    6485             :           SDLoc DL(N);
    6486          14 :           SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
    6487             :                                        NewOp0,
    6488          14 :                                        DAG.getConstant(c2, DL, CountVT));
    6489          14 :           AddToWorklist(NewSHL.getNode());
    6490          28 :           return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
    6491             :         }
    6492             :       }
    6493             :     }
    6494             :   }
    6495             : 
    6496             :   // fold (shl (sr[la] exact X,  C1), C2) -> (shl    X, (C2-C1)) if C1 <= C2
    6497             :   // fold (shl (sr[la] exact X,  C1), C2) -> (sr[la] X, (C2-C1)) if C1  > C2
    6498      109631 :   if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
    6499        5236 :       N0->getFlags().hasExact()) {
    6500        1864 :     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
    6501        1864 :       uint64_t C1 = N0C1->getZExtValue();
    6502        1864 :       uint64_t C2 = N1C->getZExtValue();
    6503             :       SDLoc DL(N);
    6504        1864 :       if (C1 <= C2)
    6505        1834 :         return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
    6506        1834 :                            DAG.getConstant(C2 - C1, DL, N1.getValueType()));
    6507          30 :       return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
    6508          30 :                          DAG.getConstant(C1 - C2, DL, N1.getValueType()));
    6509             :     }
    6510             :   }
    6511             : 
    6512             :   // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
    6513             :   //                               (and (srl x, (sub c1, c2), MASK)
    6514             :   // Only fold this if the inner shift has no other uses -- if it does, folding
    6515             :   // this will increase the total number of instructions.
    6516      110328 :   if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    6517        1820 :     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
    6518        1654 :       uint64_t c1 = N0C1->getZExtValue();
    6519        1654 :       if (c1 < OpSizeInBits) {
    6520        1654 :         uint64_t c2 = N1C->getZExtValue();
    6521        1654 :         APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
    6522        1654 :         SDValue Shift;
    6523        1654 :         if (c2 > c1) {
    6524         124 :           Mask <<= c2 - c1;
    6525             :           SDLoc DL(N);
    6526         124 :           Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
    6527         124 :                               DAG.getConstant(c2 - c1, DL, N1.getValueType()));
    6528             :         } else {
    6529        1530 :           Mask.lshrInPlace(c1 - c2);
    6530             :           SDLoc DL(N);
    6531        1530 :           Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
    6532        1530 :                               DAG.getConstant(c1 - c2, DL, N1.getValueType()));
    6533             :         }
    6534             :         SDLoc DL(N0);
    6535        1654 :         return DAG.getNode(ISD::AND, DL, VT, Shift,
    6536        1654 :                            DAG.getConstant(Mask, DL, VT));
    6537             :       }
    6538             :     }
    6539             :   }
    6540             : 
    6541             :   // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
    6542      106184 :   if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
    6543          71 :       isConstantOrConstantVector(N1, /* No Opaques */ true)) {
    6544             :     SDLoc DL(N);
    6545          71 :     SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
    6546         142 :     SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
    6547         142 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
    6548             :   }
    6549             : 
    6550             :   // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
    6551             :   // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
    6552             :   // Variant of version done on multiply, except mul by a power of 2 is turned
    6553             :   // into a shift.
    6554      102902 :   if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
    6555        3085 :       N0.getNode()->hasOneUse() &&
    6556        6148 :       isConstantOrConstantVector(N1, /* No Opaques */ true) &&
    6557      110241 :       isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true) &&
    6558        1136 :       TLI.isDesirableToCommuteWithShift(N, Level)) {
    6559        1231 :     SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
    6560        1062 :     SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
    6561        1062 :     AddToWorklist(Shl0.getNode());
    6562        1062 :     AddToWorklist(Shl1.getNode());
    6563        2293 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
    6564             :   }
    6565             : 
    6566             :   // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
    6567        1419 :   if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
    6568      107814 :       isConstantOrConstantVector(N1, /* No Opaques */ true) &&
    6569        1415 :       isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
    6570        1287 :     SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
    6571        1287 :     if (isConstantOrConstantVector(Shl))
    6572        2576 :       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
    6573             :   }
    6574             : 
    6575      103693 :   if (N1C && !N1C->isOpaque())
    6576       86829 :     if (SDValue NewSHL = visitShiftByConstant(N, N1C))
    6577         636 :       return NewSHL;
    6578             : 
    6579      103057 :   return SDValue();
    6580             : }
    6581             : 
    6582       30600 : SDValue DAGCombiner::visitSRA(SDNode *N) {
    6583       30600 :   SDValue N0 = N->getOperand(0);
    6584       30600 :   SDValue N1 = N->getOperand(1);
    6585       30600 :   EVT VT = N0.getValueType();
    6586       30600 :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6587             : 
    6588             :   // Arithmetic shifting an all-sign-bit value is a no-op.
    6589             :   // fold (sra 0, x) -> 0
    6590             :   // fold (sra -1, x) -> -1
    6591       30600 :   if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
    6592         455 :     return N0;
    6593             : 
    6594             :   // fold vector ops
    6595       30145 :   if (VT.isVector())
    6596        3183 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6597           3 :       return FoldedVOp;
    6598             : 
    6599       30142 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6600             : 
    6601             :   // fold (sra c1, c2) -> (sra c1, c2)
    6602             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6603       30142 :   if (N0C && N1C && !N1C->isOpaque())
    6604           0 :     return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
    6605             :   // fold (sra x, c >= size(x)) -> undef
    6606             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6607             :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6608       55274 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6609       30142 :   };
    6610       60284 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6611          23 :     return DAG.getUNDEF(VT);
    6612             :   // fold (sra x, 0) -> x
    6613       57255 :   if (N1C && N1C->isNullValue())
    6614          15 :     return N0;
    6615             : 
    6616       30104 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6617           3 :     return NewSel;
    6618             : 
    6619             :   // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
    6620             :   // sext_inreg.
    6621       30101 :   if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
    6622        3133 :     unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
    6623        3133 :     EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
    6624        3133 :     if (VT.isVector())
    6625         183 :       ExtVT = EVT::getVectorVT(*DAG.getContext(),
    6626         183 :                                ExtVT, VT.getVectorNumElements());
    6627        3133 :     if ((!LegalOperations ||
    6628        2433 :          TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
    6629         733 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    6630        2199 :                          N0.getOperand(0), DAG.getValueType(ExtVT));
    6631             :   }
    6632             : 
    6633             :   // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
    6634             :   // clamp (add c1, c2) to max shift.
    6635       29368 :   if (N0.getOpcode() == ISD::SRA) {
    6636             :     SDLoc DL(N);
    6637         332 :     EVT ShiftVT = N1.getValueType();
    6638         332 :     EVT ShiftSVT = ShiftVT.getScalarType();
    6639             :     SmallVector<SDValue, 16> ShiftValues;
    6640             : 
    6641             :     auto SumOfShifts = [&](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    6642             :       APInt c1 = LHS->getAPIntValue();
    6643             :       APInt c2 = RHS->getAPIntValue();
    6644             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6645             :       APInt Sum = c1 + c2;
    6646             :       unsigned ShiftSum =
    6647             :           Sum.uge(OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue();
    6648             :       ShiftValues.push_back(DAG.getConstant(ShiftSum, DL, ShiftSVT));
    6649             :       return true;
    6650             :     };
    6651         664 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) {
    6652         256 :       SDValue ShiftValue;
    6653         256 :       if (VT.isVector())
    6654          30 :         ShiftValue = DAG.getBuildVector(ShiftVT, DL, ShiftValues);
    6655             :       else
    6656         241 :         ShiftValue = ShiftValues[0];
    6657         512 :       return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue);
    6658             :     }
    6659             :   }
    6660             : 
    6661             :   // fold (sra (shl X, m), (sub result_size, n))
    6662             :   // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
    6663             :   // result_size - n != m.
    6664             :   // If truncate is free for the target sext(shl) is likely to result in better
    6665             :   // code.
    6666       29112 :   if (N0.getOpcode() == ISD::SHL && N1C) {
    6667             :     // Get the two constanst of the shifts, CN0 = m, CN = n.
    6668        4332 :     const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
    6669        4332 :     if (N01C) {
    6670        4332 :       LLVMContext &Ctx = *DAG.getContext();
    6671             :       // Determine what the truncate's result bitsize and type would be.
    6672        8664 :       EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue());
    6673             : 
    6674        4332 :       if (VT.isVector())
    6675          87 :         TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorNumElements());
    6676             : 
    6677             :       // Determine the residual right-shift amount.
    6678        8664 :       int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
    6679             : 
    6680             :       // If the shift is not a no-op (in which case this should be just a sign
    6681             :       // extend already), the truncated to type is legal, sign_extend is legal
    6682             :       // on that type, and the truncate to that type is both legal and free,
    6683             :       // perform the transform.
    6684             :       if ((ShiftAmt > 0) &&
    6685        1886 :           TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
    6686        4433 :           TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
    6687         101 :           TLI.isTruncateFree(VT, TruncVT)) {
    6688             :         SDLoc DL(N);
    6689          13 :         SDValue Amt = DAG.getConstant(ShiftAmt, DL,
    6690          26 :             getShiftAmountTy(N0.getOperand(0).getValueType()));
    6691          13 :         SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
    6692          13 :                                     N0.getOperand(0), Amt);
    6693          13 :         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
    6694          13 :                                     Shift);
    6695          13 :         return DAG.getNode(ISD::SIGN_EXTEND, DL,
    6696          26 :                            N->getValueType(0), Trunc);
    6697             :       }
    6698             :     }
    6699             :   }
    6700             : 
    6701             :   // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
    6702       29099 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6703         320 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6704          18 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6705          30 :       return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
    6706             :   }
    6707             : 
    6708             :   // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
    6709             :   //      if c1 is equal to the number of bits the trunc removes
    6710        4783 :   if (N0.getOpcode() == ISD::TRUNCATE &&
    6711        4783 :       (N0.getOperand(0).getOpcode() == ISD::SRL ||
    6712        2304 :        N0.getOperand(0).getOpcode() == ISD::SRA) &&
    6713        4074 :       N0.getOperand(0).hasOneUse() &&
    6714       32736 :       N0.getOperand(0).getOperand(1).hasOneUse() &&
    6715             :       N1C) {
    6716         106 :     SDValue N0Op0 = N0.getOperand(0);
    6717         106 :     if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
    6718         212 :       unsigned LargeShiftVal = LargeShift->getZExtValue();
    6719         106 :       EVT LargeVT = N0Op0.getValueType();
    6720             : 
    6721         106 :       if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) {
    6722             :         SDLoc DL(N);
    6723             :         SDValue Amt =
    6724         100 :           DAG.getConstant(LargeShiftVal + N1C->getZExtValue(), DL,
    6725         300 :                           getShiftAmountTy(N0Op0.getOperand(0).getValueType()));
    6726         100 :         SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT,
    6727         100 :                                   N0Op0.getOperand(0), Amt);
    6728         200 :         return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
    6729             :       }
    6730             :     }
    6731             :   }
    6732             : 
    6733             :   // Simplify, based on bits shifted out of the LHS.
    6734       28984 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6735        1502 :     return SDValue(N, 0);
    6736             : 
    6737             :   // If the sign bit is known to be zero, switch this to a SRL.
    6738       27482 :   if (DAG.SignBitIsZero(N0))
    6739         238 :     return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
    6740             : 
    6741       27363 :   if (N1C && !N1C->isOpaque())
    6742       24524 :     if (SDValue NewSRA = visitShiftByConstant(N, N1C))
    6743           0 :       return NewSRA;
    6744             : 
    6745       27363 :   return SDValue();
    6746             : }
    6747             : 
    6748      164810 : SDValue DAGCombiner::visitSRL(SDNode *N) {
    6749      164810 :   SDValue N0 = N->getOperand(0);
    6750      164810 :   SDValue N1 = N->getOperand(1);
    6751      164810 :   EVT VT = N0.getValueType();
    6752             :   unsigned OpSizeInBits = VT.getScalarSizeInBits();
    6753             : 
    6754             :   // fold vector ops
    6755      164810 :   if (VT.isVector())
    6756        4830 :     if (SDValue FoldedVOp = SimplifyVBinOp(N))
    6757           4 :       return FoldedVOp;
    6758             : 
    6759      164806 :   ConstantSDNode *N1C = isConstOrConstSplat(N1);
    6760             : 
    6761             :   // fold (srl c1, c2) -> c1 >>u c2
    6762             :   ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
    6763      164806 :   if (N0C && N1C && !N1C->isOpaque())
    6764        5270 :     return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
    6765             :   // fold (srl 0, x) -> 0
    6766      162171 :   if (isNullConstantOrNullSplatConstant(N0))
    6767          59 :     return N0;
    6768             :   // fold (srl x, c >= size(x)) -> undef
    6769             :   // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
    6770             :   auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
    6771      299900 :     return Val->getAPIntValue().uge(OpSizeInBits);
    6772             :   };
    6773      324224 :   if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
    6774          27 :     return DAG.getUNDEF(VT);
    6775             :   // fold (srl x, 0) -> x
    6776      311245 :   if (N1C && N1C->isNullValue())
    6777        2218 :     return N0;
    6778             : 
    6779      159867 :   if (SDValue NewSel = foldBinOpIntoSelect(N))
    6780           4 :     return NewSel;
    6781             : 
    6782             :   // if (srl x, c) is known to be zero, return 0
    6783      306802 :   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
    6784      453695 :                                    APInt::getAllOnesValue(OpSizeInBits)))
    6785          92 :     return DAG.getConstant(0, SDLoc(N), VT);
    6786             : 
    6787             :   // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
    6788      159817 :   if (N0.getOpcode() == ISD::SRL) {
    6789             :     auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
    6790             :                                           ConstantSDNode *RHS) {
    6791             :       APInt c1 = LHS->getAPIntValue();
    6792             :       APInt c2 = RHS->getAPIntValue();
    6793             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6794             :       return (c1 + c2).uge(OpSizeInBits);
    6795             :     };
    6796       10598 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
    6797          10 :       return DAG.getConstant(0, SDLoc(N), VT);
    6798             : 
    6799             :     auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
    6800             :                                        ConstantSDNode *RHS) {
    6801             :       APInt c1 = LHS->getAPIntValue();
    6802             :       APInt c2 = RHS->getAPIntValue();
    6803             :       zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
    6804             :       return (c1 + c2).ult(OpSizeInBits);
    6805             :     };
    6806       10588 :     if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
    6807             :       SDLoc DL(N);
    6808        4426 :       EVT ShiftVT = N1.getValueType();
    6809        8852 :       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
    6810        8852 :       return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
    6811             :     }
    6812             :   }
    6813             : 
    6814             :   // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
    6815      155386 :   if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
    6816       14494 :       N0.getOperand(0).getOpcode() == ISD::SRL) {
    6817        4673 :     if (auto N001C = isConstOrConstSplat(N0.getOperand(0).getOperand(1))) {
    6818        4673 :       uint64_t c1 = N001C->getZExtValue();
    6819        4673 :       uint64_t c2 = N1C->getZExtValue();
    6820        9346 :       EVT InnerShiftVT = N0.getOperand(0).getValueType();
    6821        9346 :       EVT ShiftCountVT = N0.getOperand(0).getOperand(1).getValueType();
    6822        4673 :       uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
    6823             :       // This is only valid if the OpSizeInBits + c1 = size of inner shift.
    6824        4673 :       if (c1 + OpSizeInBits == InnerShiftSize) {
    6825             :         SDLoc DL(N0);
    6826        3217 :         if (c1 + c2 >= InnerShiftSize)
    6827           0 :           return DAG.getConstant(0, DL, VT);
    6828        3217 :         return DAG.getNode(ISD::TRUNCATE, DL, VT,
    6829             :                            DAG.getNode(ISD::SRL, DL, InnerShiftVT,
    6830        3217 :                                        N0.getOperand(0).getOperand(0),
    6831             :                                        DAG.getConstant(c1 + c2, DL,
    6832        3217 :                                                        ShiftCountVT)));
    6833             :       }
    6834             :     }
    6835             :   }
    6836             : 
    6837             :   // fold (srl (shl x, c), c) -> (and x, cst2)
    6838      153501 :   if (N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
    6839        1332 :       isConstantOrConstantVector(N1, /* NoOpaques */ true)) {
    6840             :     SDLoc DL(N);
    6841             :     SDValue Mask =
    6842         410 :         DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
    6843         205 :     AddToWorklist(Mask.getNode());
    6844         410 :     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask);
    6845             :   }
    6846             : 
    6847             :   // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
    6848      151964 :   if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    6849             :     // Shifting in all undef bits?
    6850         770 :     EVT SmallVT = N0.getOperand(0).getValueType();
    6851             :     unsigned BitSize = SmallVT.getScalarSizeInBits();
    6852         770 :     if (N1C->getZExtValue() >= BitSize)
    6853         171 :       return DAG.getUNDEF(VT);
    6854             : 
    6855         385 :     if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
    6856         171 :       uint64_t ShiftAmt = N1C->getZExtValue();
    6857             :       SDLoc DL0(N0);
    6858         171 :       SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
    6859             :                                        N0.getOperand(0),
    6860             :                           DAG.getConstant(ShiftAmt, DL0,
    6861         171 :                                           getShiftAmountTy(SmallVT)));
    6862         171 :       AddToWorklist(SmallShift.getNode());
    6863         171 :       APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt);
    6864             :       SDLoc DL(N);
    6865         171 :       return DAG.getNode(ISD::AND, DL, VT,
    6866         171 :                          DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
    6867         171 :                          DAG.getConstant(Mask, DL, VT));
    6868             :     }
    6869             :   }
    6870             : 
    6871             :   // fold (srl (sra X, Y), 31) -> (srl X, 31).  This srl only looks at the sign
    6872             :   // bit, which is unmodified by sra.
    6873      290672 :   if (N1C && N1C->getZExtValue() + 1 == OpSizeInBits) {
    6874        4372 :     if (N0.getOpcode() == ISD::SRA)
    6875         826 :       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
    6876             :   }
    6877             : 
    6878             :   // fold (srl (ctlz x), "5") -> x  iff x has one bit set (the low bit).
    6879      151401 :   if (N1C && N0.getOpcode() == ISD::CTLZ &&
    6880         348 :       N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
    6881         171 :     KnownBits Known;
    6882         348 :     DAG.computeKnownBits(N0.getOperand(0), Known);
    6883             : 
    6884             :     // If any of the input bits are KnownOne, then the input couldn't be all
    6885             :     // zeros, thus the result of the srl will always be zero.
    6886         174 :     if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
    6887             : 
    6888             :     // If all of the bits input the to ctlz node are known to be zero, then
    6889             :     // the result of the ctlz is "32" and the result of the shift is one.
    6890         174 :     APInt UnknownBits = ~Known.Zero;
    6891         174 :     if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
    6892             : 
    6893             :     // Otherwise, check to see if there is exactly one bit input to the ctlz.
    6894         174 :     if (UnknownBits.isPowerOf2()) {
    6895             :       // Okay, we know that only that the single bit specified by UnknownBits
    6896             :       // could be set on input to the CTLZ node. If this bit is set, the SRL
    6897             :       // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
    6898             :       // to an SRL/XOR pair, which is likely to simplify more.
    6899           3 :       unsigned ShAmt = UnknownBits.countTrailingZeros();
    6900           3 :       SDValue Op = N0.getOperand(0);
    6901             : 
    6902           3 :       if (ShAmt) {
    6903             :         SDLoc DL(N0);
    6904           3 :         Op = DAG.getNode(ISD::SRL, DL, VT, Op,
    6905             :                   DAG.getConstant(ShAmt, DL,
    6906           3 :                                   getShiftAmountTy(Op.getValueType())));
    6907           3 :         AddToWorklist(Op.getNode());
    6908             :       }
    6909             : 
    6910             :       SDLoc DL(N);
    6911           3 :       return DAG.getNode(ISD::XOR, DL, VT,
    6912           3 :                          Op, DAG.getConstant(1, DL, VT));
    6913             :     }
    6914             :   }
    6915             : 
    6916             :   // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
    6917      151398 :   if (N1.getOpcode() == ISD::TRUNCATE &&
    6918        3190 :       N1.getOperand(0).getOpcode() == ISD::AND) {
    6919          48 :     if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
    6920          65 :       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
    6921             :   }
    6922             : 
    6923             :   // fold operands of srl based on knowledge that the low bits are not
    6924             :   // demanded.
    6925      151366 :   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
    6926        7862 :     return SDValue(N, 0);
    6927             : 
    6928      143504 :   if (N1C && !N1C->isOpaque())
    6929      130607 :     if (SDValue NewSRL = visitShiftByConstant(N, N1C))
    6930         600 :       return NewSRL;
    6931             : 
    6932             :   // Attempt to convert a srl of a load into a narrower zero-extending load.
    6933      142904 :   if (SDValue NarrowLoad = ReduceLoadWidth(N))
    6934         120 :     return NarrowLoad;
    6935             : 
    6936             :   // Here is a common situation. We want to optimize:
    6937             :   //
    6938             :   //   %a = ...
    6939             :   //   %b = and i32 %a, 2
    6940             :   //   %c = srl i32 %b, 1
    6941             :   //   brcond i32 %c ...
    6942             :   //
    6943             :   // into
    6944             :   //
    6945             :   //   %a = ...
    6946             :   //   %b = and %a, 2
    6947             :   //   %c = setcc eq %b, 0
    6948             :   //   brcond %c ...
    6949             :   //
    6950             :   // However when after the source operand of SRL is optimized into AND, the SRL
    6951             :   // itself may not be optimized further. Look for it and add the BRCOND into
    6952             :   // the worklist.
    6953             :   if (N->hasOneUse()) {
    6954             :     SDNode *Use = *N->use_begin();
    6955      271004 :     if (Use->getOpcode() == ISD::BRCOND)
    6956           4 :       AddToWorklist(Use);
    6957      135498 :     else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
    6958             :       // Also look pass the truncate.
    6959             :       Use = *Use->use_begin();
    6960       30099 :       if (Use->getOpcode() == ISD::BRCOND)
    6961           2 :         AddToWorklist(Use);
    6962             :     }
    6963             :   }
    6964             : 
    6965      142784 :   return SDValue();
    6966             : }
    6967             : 
    6968           0 : SDValue DAGCombiner::visitABS(SDNode *N) {
    6969           0 :   SDValue N0 = N->getOperand(0);
    6970           0 :   EVT VT = N->getValueType(0);
    6971             : 
    6972             :   // fold (abs c1) -> c2
    6973           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6974           0 :     return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0);
    6975             :   // fold (abs (abs x)) -> (abs x)
    6976           0 :   if (N0.getOpcode() == ISD::ABS)
    6977           0 :     return N0;
    6978             :   // fold (abs x) -> x iff not-negative
    6979           0 :   if (DAG.SignBitIsZero(N0))
    6980           0 :     return N0;
    6981           0 :   return SDValue();
    6982             : }
    6983             : 
    6984           0 : SDValue DAGCombiner::visitBSWAP(SDNode *N) {
    6985           0 :   SDValue N0 = N->getOperand(0);
    6986           0 :   EVT VT = N->getValueType(0);
    6987             : 
    6988             :   // fold (bswap c1) -> c2
    6989           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    6990           0 :     return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
    6991             :   // fold (bswap (bswap x)) -> x
    6992           0 :   if (N0.getOpcode() == ISD::BSWAP)
    6993           0 :     return N0->getOperand(0);
    6994           0 :   return SDValue();
    6995             : }
    6996             : 
    6997           0 : SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
    6998           0 :   SDValue N0 = N->getOperand(0);
    6999           0 :   EVT VT = N->getValueType(0);
    7000             : 
    7001             :   // fold (bitreverse c1) -> c2
    7002           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7003           0 :     return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0);
    7004             :   // fold (bitreverse (bitreverse x)) -> x
    7005           0 :   if (N0.getOpcode() == ISD::BITREVERSE)
    7006           0 :     return N0.getOperand(0);
    7007           0 :   return SDValue();
    7008             : }
    7009             : 
    7010        1221 : SDValue DAGCombiner::visitCTLZ(SDNode *N) {
    7011        1221 :   SDValue N0 = N->getOperand(0);
    7012        1221 :   EVT VT = N->getValueType(0);
    7013             : 
    7014             :   // fold (ctlz c1) -> c2
    7015        1221 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7016           4 :     return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
    7017             : 
    7018             :   // If the value is known never to be zero, switch to the undef version.
    7019        1219 :   if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) {
    7020         400 :     if (DAG.isKnownNeverZero(N0))
    7021          14 :       return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7022             :   }
    7023             : 
    7024        1212 :   return SDValue();
    7025             : }
    7026             : 
    7027           0 : SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
    7028           0 :   SDValue N0 = N->getOperand(0);
    7029           0 :   EVT VT = N->getValueType(0);
    7030             : 
    7031             :   // fold (ctlz_zero_undef c1) -> c2
    7032           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7033           0 :     return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7034           0 :   return SDValue();
    7035             : }
    7036             : 
    7037         500 : SDValue DAGCombiner::visitCTTZ(SDNode *N) {
    7038         500 :   SDValue N0 = N->getOperand(0);
    7039         500 :   EVT VT = N->getValueType(0);
    7040             : 
    7041             :   // fold (cttz c1) -> c2
    7042         500 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7043           0 :     return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
    7044             : 
    7045             :   // If the value is known never to be zero, switch to the undef version.
    7046         500 :   if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) {
    7047         322 :     if (DAG.isKnownNeverZero(N0))
    7048          34 :       return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7049             :   }
    7050             : 
    7051         483 :   return SDValue();
    7052             : }
    7053             : 
    7054           0 : SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
    7055           0 :   SDValue N0 = N->getOperand(0);
    7056           0 :   EVT VT = N->getValueType(0);
    7057             : 
    7058             :   // fold (cttz_zero_undef c1) -> c2
    7059           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7060           0 :     return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
    7061           0 :   return SDValue();
    7062             : }
    7063             : 
    7064           0 : SDValue DAGCombiner::visitCTPOP(SDNode *N) {
    7065           0 :   SDValue N0 = N->getOperand(0);
    7066           0 :   EVT VT = N->getValueType(0);
    7067             : 
    7068             :   // fold (ctpop c1) -> c2
    7069           0 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    7070           0 :     return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
    7071           0 :   return SDValue();
    7072             : }
    7073             : 
    7074             : // FIXME: This should be checking for no signed zeros on individual operands, as
    7075             : // well as no nans.
    7076       28878 : static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS) {
    7077       28878 :   const TargetOptions &Options = DAG.getTarget().Options;
    7078       28878 :   EVT VT = LHS.getValueType();
    7079             : 
    7080         808 :   return Options.NoSignedZerosFPMath && VT.isFloatingPoint() &&
    7081       29477 :          DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS);
    7082             : }
    7083             : 
    7084             : /// Generate Min/Max node
    7085           0 : static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
    7086             :                                    SDValue RHS, SDValue True, SDValue False,
    7087             :                                    ISD::CondCode CC, const TargetLowering &TLI,
    7088             :                                    SelectionDAG &DAG) {
    7089             :   if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
    7090           0 :     return SDValue();
    7091             : 
    7092           0 :   EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
    7093           0 :   switch (CC) {
    7094           0 :   case ISD::SETOLT:
    7095             :   case ISD::SETOLE:
    7096             :   case ISD::SETLT:
    7097             :   case ISD::SETLE:
    7098             :   case ISD::SETULT:
    7099             :   case ISD::SETULE: {
    7100             :     unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
    7101           0 :     if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
    7102           0 :       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    7103           0 :     return SDValue();
    7104             :   }
    7105           0 :   case ISD::SETOGT:
    7106             :   case ISD::SETOGE:
    7107             :   case ISD::SETGT:
    7108             :   case ISD::SETGE:
    7109             :   case ISD::SETUGT:
    7110             :   case ISD::SETUGE: {
    7111             :     unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
    7112           0 :     if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
    7113           0 :       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    7114           0 :     return SDValue();
    7115             :   }
    7116           0 :   default:
    7117           0 :     return SDValue();
    7118             :   }
    7119             : }
    7120             : 
    7121       31935 : SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
    7122       31935 :   SDValue Cond = N->getOperand(0);
    7123       31935 :   SDValue N1 = N->getOperand(1);
    7124       31935 :   SDValue N2 = N->getOperand(2);
    7125       63870 :   EVT VT = N->getValueType(0);
    7126       31935 :   EVT CondVT = Cond.getValueType();
    7127             :   SDLoc DL(N);
    7128             : 
    7129       31935 :   if (!VT.isInteger())
    7130        7164 :     return SDValue();
    7131             : 
    7132             :   auto *C1 = dyn_cast<ConstantSDNode>(N1);
    7133             :   auto *C2 = dyn_cast<ConstantSDNode>(N2);
    7134       24771 :   if (!C1 || !C2)
    7135       20499 :     return SDValue();
    7136             : 
    7137             :   // Only do this before legalization to avoid conflicting with target-specific
    7138             :   // transforms in the other direction (create a select from a zext/sext). There
    7139             :   // is also a target-independent combine here in DAGCombiner in the other
    7140             :   // direction for (select Cond, -1, 0) when the condition is not i1.
    7141        2865 :   if (CondVT == MVT::i1 && !LegalOperations) {
    7142        2849 :     if (C1->isNullValue() && C2->isOne()) {
    7143             :       // select Cond, 0, 1 --> zext (!Cond)
    7144          66 :       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    7145             :       if (VT != MVT::i1)
    7146          66 :         NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond);
    7147          33 :       return NotCond;
    7148             :     }
    7149        1438 :     if (C1->isNullValue() && C2->isAllOnesValue()) {
    7150             :       // select Cond, 0, -1 --> sext (!Cond)
    7151          52 :       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    7152             :       if (VT != MVT::i1)
    7153          52 :         NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond);
    7154          26 :       return NotCond;
    7155             :     }
    7156        1446 :     if (C1->isOne() && C2->isNullValue()) {
    7157             :       // select Cond, 1, 0 --> zext (Cond)
    7158             :       if (VT != MVT::i1)
    7159         156 :         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
    7160          78 :       return Cond;
    7161             :     }
    7162        1329 :     if (C1->isAllOnesValue() && C2->isNullValue()) {
    7163             :       // select Cond, -1, 0 --> sext (Cond)
    7164             :       if (VT != MVT::i1)
    7165         114 :         Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
    7166          57 :       return Cond;
    7167             :     }
    7168             : 
    7169             :     // For any constants that differ by 1, we can transform the select into an
    7170             :     // extend and add. Use a target hook because some targets may prefer to
    7171             :     // transform in the other direction.
    7172        1151 :     if (TLI.convertSelectOfConstantsToMath(VT)) {
    7173        1800 :       if (C1->getAPIntValue() - 1 == C2->getAPIntValue()) {
    7174             :         // select Cond, C1, C1-1 --> add (zext Cond), C1-1
    7175             :         if (VT != MVT::i1)
    7176          66 :           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
    7177          66 :         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
    7178             :       }
    7179        1701 :       if (C1->getAPIntValue() + 1 == C2->getAPIntValue()) {
    7180             :         // select Cond, C1, C1+1 --> add (sext Cond), C1+1
    7181             :         if (VT != MVT::i1)
    7182         170 :           Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
    7183         170 :         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
    7184             :       }
    7185             :     }
    7186             : 
    7187        1033 :     return SDValue();
    7188             :   }
    7189             : 
    7190             :   // fold (select Cond, 0, 1) -> (xor Cond, 1)
    7191             :   // We can't do this reliably if integer based booleans have different contents
    7192             :   // to floating point based booleans. This is because we can't tell whether we
    7193             :   // have an integer-based boolean or a floating-point-based boolean unless we
    7194             :   // can find the SETCC that produced it and inspect its operands. This is
    7195             :   // fairly easy if C is the SETCC node, but it can potentially be
    7196             :   // undiscoverable (or not reasonably discoverable). For example, it could be
    7197             :   // in another basic block or it could require searching a complicated
    7198             :   // expression.
    7199        2927 :   if (CondVT.isInteger() &&
    7200        2927 :       TLI.getBooleanContents(/*isVec*/false, /*isFloat*/true) ==
    7201        1273 :           TargetLowering::ZeroOrOneBooleanContent &&
    7202             :       TLI.getBooleanContents(/*isVec*/false, /*isFloat*/false) ==
    7203        1273 :           TargetLowering::ZeroOrOneBooleanContent &&
    7204        4942 :       C1->isNullValue() && C2->isOne()) {
    7205             :     SDValue NotCond =
    7206           0 :         DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
    7207           0 :     if (VT.bitsEq(CondVT))
    7208           0 :       return NotCond;
    7209           0 :     return DAG.getZExtOrTrunc(NotCond, DL, VT);
    7210             :   }
    7211             : 
    7212        2927 :   return SDValue();
    7213             : }
    7214             : 
    7215       32115 : SDValue DAGCombiner::visitSELECT(SDNode *N) {
    7216       32115 :   SDValue N0 = N->getOperand(0);
    7217       32115 :   SDValue N1 = N->getOperand(1);
    7218       32115 :   SDValue N2 = N->getOperand(2);
    7219       64230 :   EVT VT = N->getValueType(0);
    7220       32115 :   EVT VT0 = N0.getValueType();
    7221             :   SDLoc DL(N);
    7222             : 
    7223             :   // fold (select C, X, X) -> X
    7224       32115 :   if (N1 == N2)
    7225          70 :     return N1;
    7226             : 
    7227             :   if (const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
    7228             :     // fold (select true, X, Y) -> X
    7229             :     // fold (select false, X, Y) -> Y
    7230         230 :     return !N0C->isNullValue() ? N1 : N2;
    7231             :   }
    7232             : 
    7233             :   // fold (select X, X, Y) -> (or X, Y)
    7234             :   // fold (select X, 1, Y) -> (or C, Y)
    7235         102 :   if (VT == VT0 && VT == MVT::i1 && (N0 == N1 || isOneConstant(N1)))
    7236          54 :     return DAG.getNode(ISD::OR, DL, VT, N0, N2);
    7237             : 
    7238       31935 :   if (SDValue V = foldSelectOfConstants(N))
    7239         312 :     return V;
    7240             : 
    7241             :   // fold (select C, 0, X) -> (and (not C), X)
    7242          81 :   if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) {
    7243           7 :     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
    7244           7 :     AddToWorklist(NOTNode.getNode());
    7245          14 :     return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2);
    7246             :   }
    7247             :   // fold (select C, X, 1) -> (or (not C), X)
    7248          74 :   if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) {
    7249           2 :     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
    7250           2 :     AddToWorklist(NOTNode.getNode());
    7251           4 :     return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1);
    7252             :   }
    7253             :   // fold (select X, Y, X) -> (and X, Y)
    7254             :   // fold (select X, Y, 0) -> (and X, Y)
    7255          70 :   if (VT == VT0 && VT == MVT::i1 && (N0 == N2 || isNullConstant(N2)))
    7256          30 :     return DAG.getNode(ISD::AND, DL, VT, N0, N1);
    7257             : 
    7258             :   // If we can fold this based on the true/false value, do so.
    7259       31599 :   if (SimplifySelectOps(N, N1, N2))
    7260         108 :     return SDValue(N, 0); // Don't revisit N.
    7261             : 
    7262             :   if (VT0 == MVT::i1) {
    7263             :     // The code in this block deals with the following 2 equivalences:
    7264             :     //    select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
    7265             :     //    select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
    7266             :     // The target can specify its preferred form with the
    7267             :     // shouldNormalizeToSelectSequence() callback. However we always transform
    7268             :     // to the right anyway if we find the inner select exists in the DAG anyway
    7269             :     // and we always transform to the left side if we know that we can further
    7270             :     // optimize the combination of the conditions.
    7271             :     bool normalizeToSequence =
    7272       21913 :         TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
    7273             :     // select (and Cond0, Cond1), X, Y
    7274             :     //   -> select Cond0, (select Cond1, X, Y), Y
    7275       21913 :     if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
    7276         688 :       SDValue Cond0 = N0->getOperand(0);
    7277         688 :       SDValue Cond1 = N0->getOperand(1);
    7278             :       SDValue InnerSelect =
    7279        2064 :           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
    7280         688 :       if (normalizeToSequence || !InnerSelect.use_empty())
    7281          38 :         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
    7282          38 :                            InnerSelect, N2);
    7283             :     }
    7284             :     // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
    7285       21875 :     if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
    7286         177 :       SDValue Cond0 = N0->getOperand(0);
    7287         177 :       SDValue Cond1 = N0->getOperand(1);
    7288             :       SDValue InnerSelect =
    7289         531 :           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
    7290         177 :       if (normalizeToSequence || !InnerSelect.use_empty())
    7291          54 :         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
    7292          54 :                            InnerSelect);
    7293             :     }
    7294             : 
    7295             :     // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
    7296       21821 :     if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
    7297        1425 :       SDValue N1_0 = N1->getOperand(0);
    7298        1425 :       SDValue N1_1 = N1->getOperand(1);
    7299        1425 :       SDValue N1_2 = N1->getOperand(2);
    7300        1425 :       if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
    7301             :         // Create the actual and node if we can generate good code for it.
    7302         214 :         if (!normalizeToSequence) {
    7303         292 :           SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
    7304         438 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, N2);
    7305             :         }
    7306             :         // Otherwise see if we can optimize the "and" to a better pattern.
    7307          68 :         if (SDValue Combined = visitANDLike(N0, N1_0, N))
    7308           6 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
    7309          12 :                              N2);
    7310             :       }
    7311             :     }
    7312             :     // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
    7313       21669 :     if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
    7314         833 :       SDValue N2_0 = N2->getOperand(0);
    7315         833 :       SDValue N2_1 = N2->getOperand(1);
    7316         833 :       SDValue N2_2 = N2->getOperand(2);
    7317         833 :       if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
    7318             :         // Create the actual or node if we can generate good code for it.
    7319         117 :         if (!normalizeToSequence) {
    7320          34 :           SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
    7321          34 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, N2_2);
    7322             :         }
    7323             :         // Otherwise see if we can optimize to a better pattern.
    7324         100 :         if (SDValue Combined = visitORLike(N0, N2_0, N))
    7325           6 :           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
    7326           6 :                              N2_2);
    7327             :       }
    7328             :     }
    7329             :   }
    7330             : 
    7331             :   if (VT0 == MVT::i1) {
    7332             :     // select (not Cond), N1, N2 -> select Cond, N2, N1
    7333       21646 :     if (isBitwiseNot(N0))
    7334         564 :       return DAG.getNode(ISD::SELECT, DL, VT, N0->getOperand(0), N2, N1);
    7335             :   }
    7336             : 
    7337             :   // Fold selects based on a setcc into other things, such as min/max/abs.
    7338       31036 :   if (N0.getOpcode() == ISD::SETCC) {
    7339       25213 :     SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1);
    7340       25213 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7341             : 
    7342             :     // select (fcmp lt x, y), x, y -> fminnum x, y
    7343             :     // select (fcmp gt x, y), x, y -> fmaxnum x, y
    7344             :     //
    7345             :     // This is OK if we don't care what happens if either operand is a NaN.
    7346       25213 :     if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2))
    7347         136 :       if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2,
    7348         136 :                                                 CC, TLI, DAG))
    7349         103 :         return FMinMax;
    7350             : 
    7351             :     // Use 'unsigned add with overflow' to optimize an unsigned saturating add.
    7352             :     // This is conservatively limited to pre-legal-operations to give targets
    7353             :     // a chance to reverse the transform if they want to do that. Also, it is
    7354             :     // unlikely that the pattern would be formed late, so it's probably not
    7355             :     // worth going through the other checks.
    7356       25036 :     if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) &&
    7357         703 :         CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) &&
    7358       25147 :         N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(0)) {
    7359             :       auto *C = dyn_cast<ConstantSDNode>(N2.getOperand(1));
    7360             :       auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
    7361          68 :       if (C && NotC && C->getAPIntValue() == ~NotC->getAPIntValue()) {
    7362             :         // select (setcc Cond0, ~C, ugt), -1, (add Cond0, C) -->
    7363             :         // uaddo Cond0, C; select uaddo.1, -1, uaddo.0
    7364             :         //
    7365             :         // The IR equivalent of this transform would have this form:
    7366             :         //   %a = add %x, C
    7367             :         //   %c = icmp ugt %x, ~C
    7368             :         //   %r = select %c, -1, %a
    7369             :         //   =>
    7370             :         //   %u = call {iN,i1} llvm.uadd.with.overflow(%x, C)
    7371             :         //   %u0 = extractvalue %u, 0
    7372             :         //   %u1 = extractvalue %u, 1
    7373             :         //   %r = select %u1, -1, %u0
    7374          10 :         SDVTList VTs = DAG.getVTList(VT, VT0);
    7375          20 :         SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
    7376          10 :         return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0));
    7377             :       }
    7378             :     }
    7379             : 
    7380       25100 :     if (TLI.isOperationLegal(ISD::SELECT_CC, VT) ||
    7381       24739 :         (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)))
    7382        3598 :       return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2,
    7383        3598 :                          N0.getOperand(2));
    7384             : 
    7385       21502 :     return SimplifySelect(DL, N0, N1, N2);
    7386             :   }
    7387             : 
    7388        5823 :   return SDValue();
    7389             : }
    7390             : 
    7391             : static
    7392          19 : std::pair<SDValue, SDValue> SplitVSETCC(const SDNode *N, SelectionDAG &DAG) {
    7393             :   SDLoc DL(N);
    7394             :   EVT LoVT, HiVT;
    7395          38 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
    7396             : 
    7397             :   // Split the inputs.
    7398             :   SDValue Lo, Hi, LL, LH, RL, RH;
    7399          19 :   std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
    7400          19 :   std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
    7401             : 
    7402          57 :   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
    7403          57 :   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
    7404             : 
    7405          19 :   return std::make_pair(Lo, Hi);
    7406             : }
    7407             : 
    7408             : // This function assumes all the vselect's arguments are CONCAT_VECTOR
    7409             : // nodes and that the condition is a BV of ConstantSDNodes (or undefs).
    7410          43 : static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
    7411             :   SDLoc DL(N);
    7412          43 :   SDValue Cond = N->getOperand(0);
    7413          43 :   SDValue LHS = N->getOperand(1);
    7414          43 :   SDValue RHS = N->getOperand(2);
    7415          86 :   EVT VT = N->getValueType(0);
    7416          43 :   int NumElems = VT.getVectorNumElements();
    7417             :   assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
    7418             :          RHS.getOpcode() == ISD::CONCAT_VECTORS &&
    7419             :          Cond.getOpcode() == ISD::BUILD_VECTOR);
    7420             : 
    7421             :   // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
    7422             :   // binary ones here.
    7423          43 :   if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
    7424          11 :     return SDValue();
    7425             : 
    7426             :   // We're sure we have an even number of elements due to the
    7427             :   // concat_vectors we have as arguments to vselect.
    7428             :   // Skip BV elements until we find one that's not an UNDEF
    7429             :   // After we find an UNDEF element, keep looping until we get to half the
    7430             :   // length of the BV and see if all the non-undef nodes are the same.
    7431             :   ConstantSDNode *BottomHalf = nullptr;
    7432          98 :   for (int i = 0; i < NumElems / 2; ++i) {
    7433         170 :     if (Cond->getOperand(i)->isUndef())
    7434             :       continue;
    7435             : 
    7436          85 :     if (BottomHalf == nullptr)
    7437             :       BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    7438          53 :     else if (Cond->getOperand(i).getNode() != BottomHalf)
    7439          19 :       return SDValue();
    7440             :   }
    7441             : 
    7442             :   // Do the same for the second half of the BuildVector
    7443             :   ConstantSDNode *TopHalf = nullptr;
    7444          48 :   for (int i = NumElems / 2; i < NumElems; ++i) {
    7445          76 :     if (Cond->getOperand(i)->isUndef())
    7446             :       continue;
    7447             : 
    7448          38 :     if (TopHalf == nullptr)
    7449             :       TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    7450          25 :     else if (Cond->getOperand(i).getNode() != TopHalf)
    7451           3 :       return SDValue();
    7452             :   }
    7453             : 
    7454             :   assert(TopHalf && BottomHalf &&
    7455             :          "One half of the selector was all UNDEFs and the other was all the "
    7456             :          "same value. This should have been addressed before this function.");
    7457             :   return DAG.getNode(
    7458             :       ISD::CONCAT_VECTORS, DL, VT,
    7459          20 :       BottomHalf->isNullValue() ? RHS->getOperand(0) : LHS->getOperand(0),
    7460          30 :       TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
    7461             : }
    7462             : 
    7463         287 : SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
    7464         287 :   if (Level >= AfterLegalizeTypes)
    7465         166 :     return SDValue();
    7466             : 
    7467             :   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
    7468         121 :   SDValue Mask = MSC->getMask();
    7469         121 :   SDValue Data  = MSC->getValue();
    7470             :   SDLoc DL(N);
    7471             : 
    7472             :   // If the MSCATTER data type requires splitting and the mask is provided by a
    7473             :   // SETCC, then split both nodes and its operands before legalization. This
    7474             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7475             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7476         121 :   if (Mask.getOpcode() != ISD::SETCC)
    7477          96 :     return SDValue();
    7478             : 
    7479             :   // Check if any splitting is required.
    7480          50 :   if (TLI.getTypeAction(*DAG.getContext(), Data.getValueType()) !=
    7481             :       TargetLowering::TypeSplitVector)
    7482          20 :     return SDValue();
    7483             :   SDValue MaskLo, MaskHi;
    7484           5 :   std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7485             : 
    7486             :   EVT LoVT, HiVT;
    7487          10 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0));
    7488             : 
    7489           5 :   SDValue Chain = MSC->getChain();
    7490             : 
    7491           5 :   EVT MemoryVT = MSC->getMemoryVT();
    7492           5 :   unsigned Alignment = MSC->getOriginalAlignment();
    7493             : 
    7494           5 :   EVT LoMemVT, HiMemVT;
    7495           5 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7496             : 
    7497             :   SDValue DataLo, DataHi;
    7498           5 :   std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    7499             : 
    7500           5 :   SDValue Scale = MSC->getScale();
    7501           5 :   SDValue BasePtr = MSC->getBasePtr();
    7502             :   SDValue IndexLo, IndexHi;
    7503          10 :   std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->getIndex(), DL);
    7504             : 
    7505           5 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    7506          15 :     getMachineMemOperand(MSC->getPointerInfo(),
    7507             :                           MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
    7508           5 :                           Alignment, MSC->getAAInfo(), MSC->getRanges());
    7509             : 
    7510           5 :   SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo, Scale };
    7511           5 :   SDValue Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
    7512          10 :                                     DataLo.getValueType(), DL, OpsLo, MMO);
    7513             : 
    7514             :   // The order of the Scatter operation after split is well defined. The "Hi"
    7515             :   // part comes after the "Lo". So these two operations should be chained one
    7516             :   // after another.
    7517           5 :   SDValue OpsHi[] = { Lo, DataHi, MaskHi, BasePtr, IndexHi, Scale };
    7518           5 :   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
    7519          10 :                               DL, OpsHi, MMO);
    7520             : }
    7521             : 
    7522         761 : SDValue DAGCombiner::visitMSTORE(SDNode *N) {
    7523         761 :   if (Level >= AfterLegalizeTypes)
    7524         448 :     return SDValue();
    7525             : 
    7526             :   MaskedStoreSDNode *MST = dyn_cast<MaskedStoreSDNode>(N);
    7527         313 :   SDValue Mask = MST->getMask();
    7528         313 :   SDValue Data  = MST->getValue();
    7529         626 :   EVT VT = Data.getValueType();
    7530             :   SDLoc DL(N);
    7531             : 
    7532             :   // If the MSTORE data type requires splitting and the mask is provided by a
    7533             :   // SETCC, then split both nodes and its operands before legalization. This
    7534             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7535             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7536         313 :   if (Mask.getOpcode() == ISD::SETCC) {
    7537             :     // Check if any splitting is required.
    7538          36 :     if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7539             :         TargetLowering::TypeSplitVector)
    7540          34 :       return SDValue();
    7541             : 
    7542             :     SDValue MaskLo, MaskHi, Lo, Hi;
    7543           2 :     std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7544             : 
    7545           2 :     SDValue Chain = MST->getChain();
    7546           2 :     SDValue Ptr   = MST->getBasePtr();
    7547             : 
    7548           2 :     EVT MemoryVT = MST->getMemoryVT();
    7549           2 :     unsigned Alignment = MST->getOriginalAlignment();
    7550             : 
    7551             :     // if Alignment is equal to the vector size,
    7552             :     // take the half of it for the second part
    7553             :     unsigned SecondHalfAlignment =
    7554           2 :       (Alignment == VT.getSizeInBits() / 8) ? Alignment / 2 : Alignment;
    7555             : 
    7556           2 :     EVT LoMemVT, HiMemVT;
    7557           2 :     std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7558             : 
    7559             :     SDValue DataLo, DataHi;
    7560           2 :     std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
    7561             : 
    7562           2 :     MachineMemOperand *MMO = DAG.getMachineFunction().
    7563           6 :       getMachineMemOperand(MST->getPointerInfo(),
    7564             :                            MachineMemOperand::MOStore,  LoMemVT.getStoreSize(),
    7565           2 :                            Alignment, MST->getAAInfo(), MST->getRanges());
    7566             : 
    7567           2 :     Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
    7568           2 :                             MST->isTruncatingStore(),
    7569           4 :                             MST->isCompressingStore());
    7570             : 
    7571           2 :     Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    7572           2 :                                      MST->isCompressingStore());
    7573             :     unsigned HiOffset = LoMemVT.getStoreSize();
    7574             : 
    7575           6 :     MMO = DAG.getMachineFunction().getMachineMemOperand(
    7576             :         MST->getPointerInfo().getWithOffset(HiOffset),
    7577             :         MachineMemOperand::MOStore, HiMemVT.getStoreSize(), SecondHalfAlignment,
    7578           2 :         MST->getAAInfo(), MST->getRanges());
    7579             : 
    7580           2 :     Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
    7581           2 :                             MST->isTruncatingStore(),
    7582           4 :                             MST->isCompressingStore());
    7583             : 
    7584           2 :     AddToWorklist(Lo.getNode());
    7585           2 :     AddToWorklist(Hi.getNode());
    7586             : 
    7587           4 :     return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
    7588             :   }
    7589         277 :   return SDValue();
    7590             : }
    7591             : 
    7592         994 : SDValue DAGCombiner::visitMGATHER(SDNode *N) {
    7593         994 :   if (Level >= AfterLegalizeTypes)
    7594         508 :     return SDValue();
    7595             : 
    7596             :   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
    7597         486 :   SDValue Mask = MGT->getMask();
    7598             :   SDLoc DL(N);
    7599             : 
    7600             :   // If the MGATHER result requires splitting and the mask is provided by a
    7601             :   // SETCC, then split both nodes and its operands before legalization. This
    7602             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7603             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7604             : 
    7605         486 :   if (Mask.getOpcode() != ISD::SETCC)
    7606         456 :     return SDValue();
    7607             : 
    7608          30 :   EVT VT = N->getValueType(0);
    7609             : 
    7610             :   // Check if any splitting is required.
    7611          30 :   if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7612             :       TargetLowering::TypeSplitVector)
    7613          24 :     return SDValue();
    7614             : 
    7615             :   SDValue MaskLo, MaskHi, Lo, Hi;
    7616           6 :   std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7617             : 
    7618           6 :   SDValue PassThru = MGT->getPassThru();
    7619             :   SDValue PassThruLo, PassThruHi;
    7620           6 :   std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, DL);
    7621             : 
    7622             :   EVT LoVT, HiVT;
    7623           6 :   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
    7624             : 
    7625           6 :   SDValue Chain = MGT->getChain();
    7626           6 :   EVT MemoryVT = MGT->getMemoryVT();
    7627           6 :   unsigned Alignment = MGT->getOriginalAlignment();
    7628             : 
    7629           6 :   EVT LoMemVT, HiMemVT;
    7630           6 :   std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7631             : 
    7632           6 :   SDValue Scale = MGT->getScale();
    7633           6 :   SDValue BasePtr = MGT->getBasePtr();
    7634           6 :   SDValue Index = MGT->getIndex();
    7635             :   SDValue IndexLo, IndexHi;
    7636           6 :   std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
    7637             : 
    7638           6 :   MachineMemOperand *MMO = DAG.getMachineFunction().
    7639          18 :     getMachineMemOperand(MGT->getPointerInfo(),
    7640             :                           MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    7641           6 :                           Alignment, MGT->getAAInfo(), MGT->getRanges());
    7642             : 
    7643           6 :   SDValue OpsLo[] = { Chain, PassThruLo, MaskLo, BasePtr, IndexLo, Scale };
    7644           6 :   Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, DL, OpsLo,
    7645           6 :                            MMO);
    7646             : 
    7647           6 :   SDValue OpsHi[] = { Chain, PassThruHi, MaskHi, BasePtr, IndexHi, Scale };
    7648           6 :   Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, DL, OpsHi,
    7649           6 :                            MMO);
    7650             : 
    7651           6 :   AddToWorklist(Lo.getNode());
    7652           6 :   AddToWorklist(Hi.getNode());
    7653             : 
    7654             :   // Build a factor node to remember that this load is independent of the
    7655             :   // other one.
    7656           6 :   Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
    7657           6 :                       Hi.getValue(1));
    7658             : 
    7659             :   // Legalized the chain result - switch anything that used the old chain to
    7660             :   // use the new one.
    7661          12 :   DAG.ReplaceAllUsesOfValueWith(SDValue(MGT, 1), Chain);
    7662             : 
    7663          12 :   SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    7664             : 
    7665           6 :   SDValue RetOps[] = { GatherRes, Chain };
    7666          12 :   return DAG.getMergeValues(RetOps, DL);
    7667             : }
    7668             : 
    7669        1291 : SDValue DAGCombiner::visitMLOAD(SDNode *N) {
    7670        1291 :   if (Level >= AfterLegalizeTypes)
    7671         722 :     return SDValue();
    7672             : 
    7673             :   MaskedLoadSDNode *MLD = dyn_cast<MaskedLoadSDNode>(N);
    7674         569 :   SDValue Mask = MLD->getMask();
    7675             :   SDLoc DL(N);
    7676             : 
    7677             :   // If the MLOAD result requires splitting and the mask is provided by a
    7678             :   // SETCC, then split both nodes and its operands before legalization. This
    7679             :   // prevents the type legalizer from unrolling SETCC into scalar comparisons
    7680             :   // and enables future optimizations (e.g. min/max pattern matching on X86).
    7681         569 :   if (Mask.getOpcode() == ISD::SETCC) {
    7682          76 :     EVT VT = N->getValueType(0);
    7683             : 
    7684             :     // Check if any splitting is required.
    7685          76 :     if (TLI.getTypeAction(*DAG.getContext(), VT) !=
    7686             :         TargetLowering::TypeSplitVector)
    7687          70 :       return SDValue();
    7688             : 
    7689             :     SDValue MaskLo, MaskHi, Lo, Hi;
    7690           6 :     std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
    7691             : 
    7692           6 :     SDValue PassThru = MLD->getPassThru();
    7693             :     SDValue PassThruLo, PassThruHi;
    7694           6 :     std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, DL);
    7695             : 
    7696             :     EVT LoVT, HiVT;
    7697          12 :     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
    7698             : 
    7699           6 :     SDValue Chain = MLD->getChain();
    7700           6 :     SDValue Ptr   = MLD->getBasePtr();
    7701           6 :     EVT MemoryVT = MLD->getMemoryVT();
    7702           6 :     unsigned Alignment = MLD->getOriginalAlignment();
    7703             : 
    7704             :     // if Alignment is equal to the vector size,
    7705             :     // take the half of it for the second part
    7706             :     unsigned SecondHalfAlignment =
    7707          12 :       (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
    7708             :          Alignment/2 : Alignment;
    7709             : 
    7710           6 :     EVT LoMemVT, HiMemVT;
    7711           6 :     std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
    7712             : 
    7713           6 :     MachineMemOperand *MMO = DAG.getMachineFunction().
    7714          18 :     getMachineMemOperand(MLD->getPointerInfo(),
    7715             :                          MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
    7716           6 :                          Alignment, MLD->getAAInfo(), MLD->getRanges());
    7717             : 
    7718           6 :     Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, PassThruLo, LoMemVT,
    7719           6 :                            MMO, ISD::NON_EXTLOAD, MLD->isExpandingLoad());
    7720             : 
    7721           6 :     Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
    7722           6 :                                      MLD->isExpandingLoad());
    7723             :     unsigned HiOffset = LoMemVT.getStoreSize();
    7724             : 
    7725          18 :     MMO = DAG.getMachineFunction().getMachineMemOperand(
    7726             :         MLD->getPointerInfo().getWithOffset(HiOffset),
    7727             :         MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), SecondHalfAlignment,
    7728           6 :         MLD->getAAInfo(), MLD->getRanges());
    7729             : 
    7730           6 :     Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, PassThruHi, HiMemVT,
    7731           6 :                            MMO, ISD::NON_EXTLOAD, MLD->isExpandingLoad());
    7732             : 
    7733           6 :     AddToWorklist(Lo.getNode());
    7734           6 :     AddToWorklist(Hi.getNode());
    7735             : 
    7736             :     // Build a factor node to remember that this load is independent of the
    7737             :     // other one.
    7738           6 :     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
    7739           6 :                         Hi.getValue(1));
    7740             : 
    7741             :     // Legalized the chain result - switch anything that used the old chain to
    7742             :     // use the new one.
    7743          12 :     DAG.ReplaceAllUsesOfValueWith(SDValue(MLD, 1), Chain);
    7744             : 
    7745          12 :     SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    7746             : 
    7747           6 :     SDValue RetOps[] = { LoadRes, Chain };
    7748          12 :     return DAG.getMergeValues(RetOps, DL);
    7749             :   }
    7750         493 :   return SDValue();
    7751             : }
    7752             : 
    7753             : /// A vector select of 2 constant vectors can be simplified to math/logic to
    7754             : /// avoid a variable select instruction and possibly avoid constant loads.
    7755           0 : SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
    7756           0 :   SDValue Cond = N->getOperand(0);
    7757           0 :   SDValue N1 = N->getOperand(1);
    7758           0 :   SDValue N2 = N->getOperand(2);
    7759           0 :   EVT VT = N->getValueType(0);
    7760           0 :   if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
    7761           0 :       !TLI.convertSelectOfConstantsToMath(VT) ||
    7762           0 :       !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) ||
    7763           0 :       !ISD::isBuildVectorOfConstantSDNodes(N2.getNode()))
    7764           0 :     return SDValue();
    7765             : 
    7766             :   // Check if we can use the condition value to increment/decrement a single
    7767             :   // constant value. This simplifies a select to an add and removes a constant
    7768             :   // load/materialization from the general case.
    7769             :   bool AllAddOne = true;
    7770             :   bool AllSubOne = true;
    7771             :   unsigned Elts = VT.getVectorNumElements();
    7772           0 :   for (unsigned i = 0; i != Elts; ++i) {
    7773           0 :     SDValue N1Elt = N1.getOperand(i);
    7774           0 :     SDValue N2Elt = N2.getOperand(i);
    7775           0 :     if (N1Elt.isUndef() || N2Elt.isUndef())
    7776           0 :       continue;
    7777             : 
    7778           0 :     const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
    7779           0 :     const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
    7780           0 :     if (C1 != C2 + 1)
    7781             :       AllAddOne = false;
    7782           0 :     if (C1 != C2 - 1)
    7783             :       AllSubOne = false;
    7784             :   }
    7785             : 
    7786             :   // Further simplifications for the extra-special cases where the constants are
    7787             :   // all 0 or all -1 should be implemented as folds of these patterns.
    7788             :   SDLoc DL(N);
    7789           0 :   if (AllAddOne || AllSubOne) {
    7790             :     // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
    7791             :     // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
    7792           0 :     auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
    7793           0 :     SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
    7794           0 :     return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
    7795             :   }
    7796             : 
    7797             :   // The general case for select-of-constants:
    7798             :   // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
    7799             :   // ...but that only makes sense if a vselect is slower than 2 logic ops, so
    7800             :   // leave that to a machine-specific pass.
    7801           0 :   return SDValue();
    7802             : }
    7803             : 
    7804       39265 : SDValue DAGCombiner::visitVSELECT(SDNode *N) {
    7805       39265 :   SDValue N0 = N->getOperand(0);
    7806       39265 :   SDValue N1 = N->getOperand(1);
    7807       39265 :   SDValue N2 = N->getOperand(2);
    7808             :   SDLoc DL(N);
    7809             : 
    7810             :   // fold (vselect C, X, X) -> X
    7811             :   if (N1 == N2)
    7812           4 :     return N1;
    7813             : 
    7814             :   // Canonicalize integer abs.
    7815             :   // vselect (setg[te] X,  0),  X, -X ->
    7816             :   // vselect (setgt    X, -1),  X, -X ->
    7817             :   // vselect (setl[te] X,  0), -X,  X ->
    7818             :   // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
    7819       39261 :   if (N0.getOpcode() == ISD::SETCC) {
    7820       12694 :     SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    7821       12694 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7822             :     bool isAbs = false;
    7823       12694 :     bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
    7824             : 
    7825       20386 :     if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
    7826       12564 :          (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
    7827       13062 :         N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
    7828         356 :       isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode());
    7829        7722 :     else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
    7830       12425 :              N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
    7831          87 :       isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
    7832             : 
    7833         443 :     if (isAbs) {
    7834         443 :       EVT VT = LHS.getValueType();
    7835         443 :       if (TLI.isOperationLegalOrCustom(ISD::ABS, VT))
    7836         712 :         return DAG.getNode(ISD::ABS, DL, VT, LHS);
    7837             : 
    7838          87 :       SDValue Shift = DAG.getNode(
    7839             :           ISD::SRA, DL, VT, LHS,
    7840          87 :           DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
    7841         174 :       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
    7842          87 :       AddToWorklist(Shift.getNode());
    7843          87 :       AddToWorklist(Add.getNode());
    7844         174 :       return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
    7845             :     }
    7846             : 
    7847             :     // vselect x, y (fcmp lt x, y) -> fminnum x, y
    7848             :     // vselect x, y (fcmp gt x, y) -> fmaxnum x, y
    7849             :     //
    7850             :     // This is OK if we don't care about what happens if either operand is a
    7851             :     // NaN.
    7852             :     //
    7853       24502 :     EVT VT = N->getValueType(0);
    7854       12251 :     if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N0.getOperand(0), N0.getOperand(1))) {
    7855          35 :       ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    7856          35 :       if (SDValue FMinMax = combineMinNumMaxNum(
    7857          70 :             DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG))
    7858          10 :         return FMinMax;
    7859             :     }
    7860             : 
    7861             :     // If this select has a condition (setcc) with narrower operands than the
    7862             :     // select, try to widen the compare to match the select width.
    7863             :     // TODO: This should be extended to handle any constant.
    7864             :     // TODO: This could be extended to handle non-loading patterns, but that
    7865             :     //       requires thorough testing to avoid regressions.
    7866       12241 :     if (isNullConstantOrNullSplatConstant(RHS)) {
    7867             :       EVT NarrowVT = LHS.getValueType();
    7868        4772 :       EVT WideVT = N1.getValueType().changeVectorElementTypeToInteger();
    7869        4772 :       EVT SetCCVT = getSetCCResultType(LHS.getValueType());
    7870             :       unsigned SetCCWidth = SetCCVT.getScalarSizeInBits();
    7871             :       unsigned WideWidth = WideVT.getScalarSizeInBits();
    7872             :       bool IsSigned = isSignedIntSetCC(CC);
    7873        4772 :       auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
    7874          53 :       if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() &&
    7875          51 :           SetCCWidth != 1 && SetCCWidth < WideWidth &&
    7876        4790 :           TLI.isLoadExtLegalOrCustom(LoadExtOpcode, WideVT, NarrowVT) &&
    7877             :           TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) {
    7878             :         // Both compare operands can be widened for free. The LHS can use an
    7879             :         // extended load, and the RHS is a constant:
    7880             :         //   vselect (ext (setcc load(X), C)), N1, N2 -->
    7881             :         //   vselect (setcc extload(X), C'), N1, N2
    7882          10 :         auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    7883          20 :         SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
    7884          20 :         SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
    7885          10 :         EVT WideSetCCVT = getSetCCResultType(WideVT);
    7886          10 :         SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
    7887          20 :         return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
    7888             :       }
    7889             :     }
    7890             :   }
    7891             : 
    7892       38798 :   if (SimplifySelectOps(N, N1, N2))
    7893           6 :     return SDValue(N, 0);  // Don't revisit N.
    7894             : 
    7895             :   // Fold (vselect (build_vector all_ones), N1, N2) -> N1
    7896       38792 :   if (ISD::isBuildVectorAllOnes(N0.getNode()))
    7897          69 :     return N1;
    7898             :   // Fold (vselect (build_vector all_zeros), N1, N2) -> N2
    7899       38723 :   if (ISD::isBuildVectorAllZeros(N0.getNode()))
    7900         365 :     return N2;
    7901             : 
    7902             :   // The ConvertSelectToConcatVector function is assuming both the above
    7903             :   // checks for (vselect (build_vector all{ones,zeros) ...) have been made
    7904             :   // and addressed.
    7905         639 :   if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
    7906       38544 :       N2.getOpcode() == ISD::CONCAT_VECTORS &&
    7907         186 :       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
    7908          43 :     if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
    7909          10 :       return CV;
    7910             :   }
    7911             : 
    7912       38348 :   if (SDValue V = foldVSelectOfConstants(N))
    7913          40 :     return V;
    7914             : 
    7915       38308 :   return SDValue();
    7916             : }
    7917             : 
    7918       16226 : SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
    7919       16226 :   SDValue N0 = N->getOperand(0);
    7920       16226 :   SDValue N1 = N->getOperand(1);
    7921       16226 :   SDValue N2 = N->getOperand(2);
    7922       16226 :   SDValue N3 = N->getOperand(3);
    7923       16226 :   SDValue N4 = N->getOperand(4);
    7924       16226 :   ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
    7925             : 
    7926             :   // fold select_cc lhs, rhs, x, x, cc -> x
    7927             :   if (N2 == N3)
    7928          27 :     return N2;
    7929             : 
    7930             :   // Determine if the condition we're dealing with is constant
    7931       16199 :   if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
    7932       32406 :                                   CC, SDLoc(N), false)) {
    7933         325 :     AddToWorklist(SCC.getNode());
    7934             : 
    7935             :     if (ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
    7936          54 :       if (!SCCC->isNullValue())
    7937          17 :         return N2;    // cond always true -> true val
    7938             :       else
    7939          10 :         return N3;    // cond always false -> false val
    7940         298 :     } else if (SCC->isUndef()) {
    7941             :       // When the condition is UNDEF, just return the first operand. This is
    7942             :       // coherent the DAG creation, no setcc node is created in this case
    7943           0 :       return N2;
    7944         298 :     } else if (SCC.getOpcode() == ISD::SETCC) {
    7945             :       // Fold to a simpler select_cc
    7946         556 :       return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(),
    7947             :                          SCC.getOperand(0), SCC.getOperand(1), N2, N3,
    7948         278 :                          SCC.getOperand(2));
    7949             :     }
    7950             :   }
    7951             : 
    7952             :   // If we can fold this based on the true/false value, do so.
    7953       15894 :   if (SimplifySelectOps(N, N2, N3))
    7954           0 :     return SDValue(N, 0);  // Don't revisit N.
    7955             : 
    7956             :   // fold select_cc into other things, such as min/max/abs
    7957       31788 :   return SimplifySelectCC(SDLoc(N), N0, N1, N2, N3, CC);
    7958             : }
    7959             : 
    7960      298267 : SDValue DAGCombiner::visitSETCC(SDNode *N) {
    7961             :   // setcc is very commonly used as an argument to brcond. This pattern
    7962             :   // also lend itself to numerous combines and, as a result, it is desired
    7963             :   // we keep the argument to a brcond as a setcc as much as possible.
    7964             :   bool PreferSetCC =
    7965      292820 :       N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND;
    7966             : 
    7967             :   SDValue Combined = SimplifySetCC(
    7968             :       N->getValueType(0), N->getOperand(0), N->getOperand(1),
    7969      573879 :       cast<CondCodeSDNode>(N->getOperand(2))->get(), SDLoc(N), !PreferSetCC);
    7970             : 
    7971      298267 :   if (!Combined)
    7972      281078 :     return SDValue();
    7973             : 
    7974             :   // If we prefer to have a setcc, and we don't, we'll try our best to
    7975             :   // recreate one using rebuildSetCC.
    7976       17189 :   if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
    7977         685 :     SDValue NewSetCC = rebuildSetCC(Combined);
    7978             : 
    7979             :     // We don't have anything interesting to combine to.
    7980         685 :     if (NewSetCC.getNode() == N)
    7981         115 :       return SDValue();
    7982             : 
    7983         570 :     if (NewSetCC)
    7984           0 :       return NewSetCC;
    7985             :   }
    7986             : 
    7987       17074 :   return Combined;
    7988             : }
    7989             : 
    7990           0 : SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
    7991           0 :   SDValue LHS = N->getOperand(0);
    7992           0 :   SDValue RHS = N->getOperand(1);
    7993           0 :   SDValue Carry = N->getOperand(2);
    7994           0 :   SDValue Cond = N->getOperand(3);
    7995             : 
    7996             :   // If Carry is false, fold to a regular SETCC.
    7997           0 :   if (isNullConstant(Carry))
    7998           0 :     return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
    7999             : 
    8000           0 :   return SDValue();
    8001             : }
    8002             : 
    8003             : /// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
    8004             : /// a build_vector of constants.
    8005             : /// This function is called by the DAGCombiner when visiting sext/zext/aext
    8006             : /// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
    8007             : /// Vector extends are not folded if operations are legal; this is to
    8008             : /// avoid introducing illegal build_vector dag nodes.
    8009      290803 : static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
    8010             :                                          SelectionDAG &DAG, bool LegalTypes,
    8011             :                                          bool LegalOperations) {
    8012      290803 :   unsigned Opcode = N->getOpcode();
    8013      290803 :   SDValue N0 = N->getOperand(0);
    8014      581606 :   EVT VT = N->getValueType(0);
    8015             : 
    8016             :   assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
    8017             :          Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
    8018             :          Opcode == ISD::ZERO_EXTEND_VECTOR_INREG)
    8019             :          && "Expected EXTEND dag node in input!");
    8020             : 
    8021             :   // fold (sext c1) -> c1
    8022             :   // fold (zext c1) -> c1
    8023             :   // fold (aext c1) -> c1
    8024             :   if (isa<ConstantSDNode>(N0))
    8025         567 :     return DAG.getNode(Opcode, SDLoc(N), VT, N0).getNode();
    8026             : 
    8027             :   // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
    8028             :   // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
    8029             :   // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
    8030      290236 :   EVT SVT = VT.getScalarType();
    8031      311556 :   if (!(VT.isVector() &&
    8032        9658 :       (!LegalTypes || (!LegalOperations && TLI.isTypeLegal(SVT))) &&
    8033       21320 :       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())))
    8034      290158 :     return nullptr;
    8035             : 
    8036             :   // We can fold this node into a build_vector.
    8037          78 :   unsigned VTBits = SVT.getSizeInBits();
    8038         156 :   unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits();
    8039             :   SmallVector<SDValue, 8> Elts;
    8040             :   unsigned NumElts = VT.getVectorNumElements();
    8041             :   SDLoc DL(N);
    8042             : 
    8043         486 :   for (unsigned i=0; i != NumElts; ++i) {
    8044         408 :     SDValue Op = N0->getOperand(i);
    8045         408 :     if (Op->isUndef()) {
    8046          82 :       Elts.push_back(DAG.getUNDEF(SVT));
    8047          82 :       continue;
    8048             :     }
    8049             : 
    8050             :     SDLoc DL(Op);
    8051             :     // Get the constant value and if needed trunc it to the size of the type.
    8052             :     // Nodes like build_vector might have constants wider than the scalar type.
    8053         652 :     APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
    8054         326 :     if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
    8055         112 :       Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
    8056             :     else
    8057         214 :       Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
    8058             :   }
    8059             : 
    8060          78 :   return DAG.getBuildVector(VT, DL, Elts).getNode();
    8061             : }
    8062             : 
    8063             : // ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
    8064             : // "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
    8065             : // transformation. Returns true if extension are possible and the above
    8066             : // mentioned transformation is profitable.
    8067           0 : static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
    8068             :                                     unsigned ExtOpc,
    8069             :                                     SmallVectorImpl<SDNode *> &ExtendNodes,
    8070             :                                     const TargetLowering &TLI) {
    8071             :   bool HasCopyToRegUses = false;
    8072           0 :   bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType());
    8073           0 :   for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
    8074             :                             UE = N0.getNode()->use_end();
    8075           0 :        UI != UE; ++UI) {
    8076           0 :     SDNode *User = *UI;
    8077           0 :     if (User == N)
    8078           0 :       continue;
    8079           0 :     if (UI.getUse().getResNo() != N0.getResNo())
    8080           0 :       continue;
    8081             :     // FIXME: Only extend SETCC N, N and SETCC N, c for now.
    8082           0 :     if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
    8083           0 :       ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
    8084           0 :       if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
    8085             :         // Sign bits will be lost after a zext.
    8086           0 :         return false;
    8087             :       bool Add = false;
    8088           0 :       for (unsigned i = 0; i != 2; ++i) {
    8089           0 :         SDValue UseOp = User->getOperand(i);
    8090           0 :         if (UseOp == N0)
    8091           0 :           continue;
    8092             :         if (!isa<ConstantSDNode>(UseOp))
    8093           0 :           return false;
    8094             :         Add = true;
    8095             :       }
    8096           0 :       if (Add)
    8097           0 :         ExtendNodes.push_back(User);
    8098           0 :       continue;
    8099             :     }
    8100             :     // If truncates aren't free and there are users we can't
    8101             :     // extend, it isn't worthwhile.
    8102           0 :     if (!isTruncFree)
    8103           0 :       return false;
    8104             :     // Remember if this value is live-out.
    8105           0 :     if (User->getOpcode() == ISD::CopyToReg)
    8106             :       HasCopyToRegUses = true;
    8107             :   }
    8108             : 
    8109           0 :   if (HasCopyToRegUses) {
    8110             :     bool BothLiveOut = false;
    8111           0 :     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
    8112           0 :          UI != UE; ++UI) {
    8113             :       SDUse &Use = UI.getUse();
    8114           0 :       if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
    8115             :         BothLiveOut = true;
    8116             :         break;
    8117             :       }
    8118             :     }
    8119           0 :     if (BothLiveOut)
    8120             :       // Both unextended and extended values are live out. There had better be
    8121             :       // a good reason for the transformation.
    8122           0 :       return ExtendNodes.size();
    8123             :   }
    8124             :   return true;
    8125             : }
    8126             : 
    8127           0 : void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
    8128             :                                   SDValue OrigLoad, SDValue ExtLoad,
    8129             :                                   ISD::NodeType ExtType) {
    8130             :   // Extend SetCC uses if necessary.
    8131           0 :   SDLoc DL(ExtLoad);
    8132           0 :   for (SDNode *SetCC : SetCCs) {
    8133             :     SmallVector<SDValue, 4> Ops;
    8134             : 
    8135           0 :     for (unsigned j = 0; j != 2; ++j) {
    8136           0 :       SDValue SOp = SetCC->getOperand(j);
    8137           0 :       if (SOp == OrigLoad)
    8138           0 :         Ops.push_back(ExtLoad);
    8139             :       else
    8140           0 :         Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
    8141             :     }
    8142             : 
    8143           0 :     Ops.push_back(SetCC->getOperand(2));
    8144           0 :     CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
    8145             :   }
    8146           0 : }
    8147             : 
    8148             : // FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
    8149      131440 : SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
    8150      131440 :   SDValue N0 = N->getOperand(0);
    8151      262880 :   EVT DstVT = N->getValueType(0);
    8152             :   EVT SrcVT = N0.getValueType();
    8153             : 
    8154             :   assert((N->getOpcode() == ISD::SIGN_EXTEND ||
    8155             :           N->getOpcode() == ISD::ZERO_EXTEND) &&
    8156             :          "Unexpected node type (not an extend)!");
    8157             : 
    8158             :   // fold (sext (load x)) to multiple smaller sextloads; same for zext.
    8159             :   // For example, on a target with legal v4i32, but illegal v8i32, turn:
    8160             :   //   (v8i32 (sext (v8i16 (load x))))
    8161             :   // into:
    8162             :   //   (v8i32 (concat_vectors (v4i32 (sextload x)),
    8163             :   //                          (v4i32 (sextload (x + 16)))))
    8164             :   // Where uses of the original load, i.e.:
    8165             :   //   (v8i16 (load x))
    8166             :   // are replaced with:
    8167             :   //   (v8i16 (truncate
    8168             :   //     (v8i32 (concat_vectors (v4i32 (sextload x)),
    8169             :   //                            (v4i32 (sextload (x + 16)))))))
    8170             :   //
    8171             :   // This combine is only applicable to illegal, but splittable, vectors.
    8172             :   // All legal types, and illegal non-vector types, are handled elsewhere.
    8173             :   // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
    8174             :   //
    8175      131440 :   if (N0->getOpcode() != ISD::LOAD)
    8176      127356 :     return SDValue();
    8177             : 
    8178             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8179             : 
    8180        3827 :   if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
    8181        6313 :       !N0.hasOneUse() || LN0->isVolatile() || !DstVT.isVector() ||
    8182        2943 :       !DstVT.isPow2VectorType() || !TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
    8183        3668 :     return SDValue();
    8184             : 
    8185             :   SmallVector<SDNode *, 4> SetCCs;
    8186         832 :   if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI))
    8187           0 :     return SDValue();
    8188             : 
    8189             :   ISD::LoadExtType ExtType =
    8190         416 :       N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
    8191             : 
    8192             :   // Try to split the vector types to get down to legal types.
    8193         416 :   EVT SplitSrcVT = SrcVT;
    8194         416 :   EVT SplitDstVT = DstVT;
    8195        1702 :   while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
    8196             :          SplitSrcVT.getVectorNumElements() > 1) {
    8197         541 :     SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
    8198         541 :     SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
    8199             :   }
    8200             : 
    8201             :   if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
    8202         204 :     return SDValue();
    8203             : 
    8204             :   SDLoc DL(N);
    8205             :   const unsigned NumSplits =
    8206         212 :       DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
    8207             :   const unsigned Stride = SplitSrcVT.getStoreSize();
    8208             :   SmallVector<SDValue, 4> Loads;
    8209             :   SmallVector<SDValue, 4> Chains;
    8210             : 
    8211         212 :   SDValue BasePtr = LN0->getBasePtr();
    8212         645 :   for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
    8213         433 :     const unsigned Offset = Idx * Stride;
    8214         433 :     const unsigned Align = MinAlign(LN0->getAlignment(), Offset);
    8215             : 
    8216         433 :     SDValue SplitLoad = DAG.getExtLoad(
    8217         433 :         ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), BasePtr,
    8218             :         LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT, Align,
    8219        1732 :         LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    8220             : 
    8221         433 :     BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
    8222         433 :                           DAG.getConstant(Stride, DL, BasePtr.getValueType()));
    8223             : 
    8224         433 :     Loads.push_back(SplitLoad.getValue(0));
    8225         433 :     Chains.push_back(SplitLoad.getValue(1));
    8226             :   }
    8227             : 
    8228         424 :   SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
    8229         424 :   SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
    8230             : 
    8231             :   // Simplify TF.
    8232         212 :   AddToWorklist(NewChain.getNode());
    8233             : 
    8234         212 :   CombineTo(N, NewValue);
    8235             : 
    8236             :   // Replace uses of the original load (before extension)
    8237             :   // with a truncate of the concatenated sextloaded vectors.
    8238             :   SDValue Trunc =
    8239         218 :       DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
    8240         424 :   ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode());
    8241             :   CombineTo(N0.getNode(), Trunc, NewChain);
    8242         212 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8243             : }
    8244             : 
    8245             : // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
    8246             : //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
    8247      100653 : SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
    8248             :   assert(N->getOpcode() == ISD::ZERO_EXTEND);
    8249      100653 :   EVT VT = N->getValueType(0);
    8250             : 
    8251             :   // and/or/xor
    8252      100653 :   SDValue N0 = N->getOperand(0);
    8253       98358 :   if (!(N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8254       15638 :         N0.getOpcode() == ISD::XOR) ||
    8255      113996 :       N0.getOperand(1).getOpcode() != ISD::Constant ||
    8256       13358 :       (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT)))
    8257       87295 :     return SDValue();
    8258             : 
    8259             :   // shl/shr
    8260       13358 :   SDValue N1 = N0->getOperand(0);
    8261       13358 :   if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) ||
    8262       14388 :       N1.getOperand(1).getOpcode() != ISD::Constant ||
    8263         332 :       (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT)))
    8264       12313 :     return SDValue();
    8265             : 
    8266             :   // load
    8267        1045 :   if (!isa<LoadSDNode>(N1.getOperand(0)))
    8268         991 :     return SDValue();
    8269             :   LoadSDNode *Load = cast<LoadSDNode>(N1.getOperand(0));
    8270             :   EVT MemVT = Load->getMemoryVT();
    8271         104 :   if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) ||
    8272         103 :       Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
    8273           5 :     return SDValue();
    8274             : 
    8275             : 
    8276             :   // If the shift op is SHL, the logic op must be AND, otherwise the result
    8277             :   // will be wrong.
    8278          49 :   if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND)
    8279           2 :     return SDValue();
    8280             : 
    8281          82 :   if (!N0.hasOneUse() || !N1.hasOneUse())
    8282          12 :     return SDValue();
    8283             : 
    8284             :   SmallVector<SDNode*, 4> SetCCs;
    8285          70 :   if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0),
    8286             :                                ISD::ZERO_EXTEND, SetCCs, TLI))
    8287           9 :     return SDValue();
    8288             : 
    8289             :   // Actually do the transformation.
    8290          26 :   SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
    8291             :                                    Load->getChain(), Load->getBasePtr(),
    8292          34 :                                    Load->getMemoryVT(), Load->getMemOperand());
    8293             : 
    8294             :   SDLoc DL1(N1);
    8295          26 :   SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
    8296          26 :                               N1.getOperand(1));
    8297             : 
    8298          26 :   APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8299          52 :   Mask = Mask.zext(VT.getSizeInBits());
    8300             :   SDLoc DL0(N0);
    8301          26 :   SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
    8302          26 :                             DAG.getConstant(Mask, DL0, VT));
    8303             : 
    8304          26 :   ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
    8305          26 :   CombineTo(N, And);
    8306          26 :   if (SDValue(Load, 0).hasOneUse()) {
    8307          50 :     DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
    8308             :   } else {
    8309           1 :     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
    8310           2 :                                 Load->getValueType(0), ExtLoad);
    8311             :     CombineTo(Load, Trunc, ExtLoad.getValue(1));
    8312             :   }
    8313          26 :   return SDValue(N,0); // Return N so it doesn't get rechecked!
    8314             : }
    8315             : 
    8316             : /// If we're narrowing or widening the result of a vector select and the final
    8317             : /// size is the same size as a setcc (compare) feeding the select, then try to
    8318             : /// apply the cast operation to the select's operands because matching vector
    8319             : /// sizes for a select condition and other operands should be more efficient.
    8320      949740 : SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
    8321      949740 :   unsigned CastOpcode = Cast->getOpcode();
    8322             :   assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
    8323             :           CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
    8324             :           CastOpcode == ISD::FP_ROUND) &&
    8325             :          "Unexpected opcode for vector select narrowing/widening");
    8326             : 
    8327             :   // We only do this transform before legal ops because the pattern may be
    8328             :   // obfuscated by target-specific operations after legalization. Do not create
    8329             :   // an illegal select op, however, because that may be difficult to lower.
    8330      949740 :   EVT VT = Cast->getValueType(0);
    8331      949740 :   if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
    8332      477945 :     return SDValue();
    8333             : 
    8334      471795 :   SDValue VSel = Cast->getOperand(0);
    8335      471829 :   if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
    8336          34 :       VSel.getOperand(0).getOpcode() != ISD::SETCC)
    8337      471761 :     return SDValue();
    8338             : 
    8339             :   // Does the setcc have the same vector size as the casted select?
    8340          34 :   SDValue SetCC = VSel.getOperand(0);
    8341          68 :   EVT SetCCVT = getSetCCResultType(SetCC.getOperand(0).getValueType());
    8342          34 :   if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
    8343          18 :     return SDValue();
    8344             : 
    8345             :   // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
    8346          16 :   SDValue A = VSel.getOperand(1);
    8347          16 :   SDValue B = VSel.getOperand(2);
    8348          16 :   SDValue CastA, CastB;
    8349             :   SDLoc DL(Cast);
    8350          16 :   if (CastOpcode == ISD::FP_ROUND) {
    8351             :     // FP_ROUND (fptrunc) has an extra flag operand to pass along.
    8352          12 :     CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
    8353          12 :     CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
    8354             :   } else {
    8355          24 :     CastA = DAG.getNode(CastOpcode, DL, VT, A);
    8356          24 :     CastB = DAG.getNode(CastOpcode, DL, VT, B);
    8357             :   }
    8358          32 :   return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
    8359             : }
    8360             : 
    8361             : // fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    8362             : // fold ([s|z]ext (     extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    8363      131172 : static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
    8364             :                                      const TargetLowering &TLI, EVT VT,
    8365             :                                      bool LegalOperations, SDNode *N,
    8366             :                                      SDValue N0, ISD::LoadExtType ExtLoadType) {
    8367             :   SDNode *N0Node = N0.getNode();
    8368      131172 :   bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD) ? ISD::isSEXTLoad(N0Node)
    8369             :                                                    : ISD::isZEXTLoad(N0Node);
    8370             :   if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) ||
    8371      131379 :       !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse())
    8372      131083 :     return {};
    8373             : 
    8374             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8375          89 :   EVT MemVT = LN0->getMemoryVT();
    8376         170 :   if ((LegalOperations || LN0->isVolatile()) &&
    8377          81 :       !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT))
    8378          68 :     return {};
    8379             : 
    8380             :   SDValue ExtLoad =
    8381          21 :       DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
    8382          23 :                      LN0->getBasePtr(), MemVT, LN0->getMemOperand());
    8383          21 :   Combiner.CombineTo(N, ExtLoad);
    8384          42 :   DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8385          21 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8386             : }
    8387             : 
    8388             : // fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
    8389             : // Only generate vector extloads when 1) they're legal, and 2) they are
    8390             : // deemed desirable by the target.
    8391      151772 : static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
    8392             :                                   const TargetLowering &TLI, EVT VT,
    8393             :                                   bool LegalOperations, SDNode *N, SDValue N0,
    8394             :                                   ISD::LoadExtType ExtLoadType,
    8395             :                                   ISD::NodeType ExtOpc) {
    8396             :   if (!ISD::isNON_EXTLoad(N0.getNode()) ||
    8397       24159 :       !ISD::isUNINDEXEDLoad(N0.getNode()) ||
    8398       22628 :       ((LegalOperations || VT.isVector() ||
    8399       24121 :         cast<LoadSDNode>(N0)->isVolatile()) &&
    8400        5628 :        !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType())))
    8401      129514 :     return {};
    8402             : 
    8403             :   bool DoXform = true;
    8404             :   SmallVector<SDNode *, 4> SetCCs;
    8405       22258 :   if (!N0.hasOneUse())
    8406        2346 :     DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI);
    8407       22258 :   if (VT.isVector())
    8408        4832 :     DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
    8409       22258 :   if (!DoXform)
    8410        1926 :     return {};
    8411             : 
    8412             :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    8413       20332 :   SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
    8414             :                                    LN0->getBasePtr(), N0.getValueType(),
    8415       22201 :                                    LN0->getMemOperand());
    8416       20332 :   Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
    8417             :   // If the load value is used only by N, replace it via CombineTo N.
    8418             :   bool NoReplaceTrunc = SDValue(LN0, 0).hasOneUse();
    8419       20332 :   Combiner.CombineTo(N, ExtLoad);
    8420       20332 :   if (NoReplaceTrunc) {
    8421       19570 :     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    8422             :   } else {
    8423             :     SDValue Trunc =
    8424        1524 :         DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
    8425             :     Combiner.CombineTo(LN0, Trunc, ExtLoad.getValue(1));
    8426             :   }
    8427       20332 :   return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8428             : }
    8429             : 
    8430      131150 : static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
    8431             :                                        bool LegalOperations) {
    8432             :   assert((N->getOpcode() == ISD::SIGN_EXTEND ||
    8433             :           N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext");
    8434             : 
    8435      131150 :   SDValue SetCC = N->getOperand(0);
    8436      118172 :   if (LegalOperations || SetCC.getOpcode() != ISD::SETCC ||
    8437      131150 :       !SetCC.hasOneUse() || SetCC.getValueType() != MVT::i1)
    8438      125939 :     return SDValue();
    8439             : 
    8440        5211 :   SDValue X = SetCC.getOperand(0);
    8441        5211 :   SDValue Ones = SetCC.getOperand(1);
    8442        5211 :   ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get();
    8443       10422 :   EVT VT = N->getValueType(0);
    8444             :   EVT XVT = X.getValueType();
    8445             :   // setge X, C is canonicalized to setgt, so we do not need to match that
    8446             :   // pattern. The setlt sibling is folded in SimplifySelectCC() because it does
    8447             :   // not require the 'not' op.
    8448        5211 :   if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) {
    8449             :     // Invert and smear/shift the sign bit:
    8450             :     // sext i1 (setgt iN X, -1) --> sra (not X), (N - 1)
    8451             :     // zext i1 (setgt iN X, -1) --> srl (not X), (N - 1)
    8452             :     SDLoc DL(N);
    8453          32 :     SDValue NotX = DAG.getNOT(DL, X, VT);
    8454          32 :     SDValue ShiftAmount = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT);
    8455          32 :     auto ShiftOpcode = N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL;
    8456          32 :     return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount);
    8457             :   }
    8458        5179 :   return SDValue();
    8459             : }
    8460             : 
    8461       51003 : SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
    8462       51003 :   SDValue N0 = N->getOperand(0);
    8463      102006 :   EVT VT = N->getValueType(0);
    8464             :   SDLoc DL(N);
    8465             : 
    8466       51003 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8467       51003 :                                               LegalOperations))
    8468          73 :     return SDValue(Res, 0);
    8469             : 
    8470             :   // fold (sext (sext x)) -> (sext x)
    8471             :   // fold (sext (aext x)) -> (sext x)
    8472      101860 :   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    8473          12 :     return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
    8474             : 
    8475       50924 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8476             :     // fold (sext (truncate (load x))) -> (sext (smaller load x))
    8477             :     // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
    8478        6778 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8479          41 :       SDNode *oye = N0.getOperand(0).getNode();
    8480          41 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8481          41 :         CombineTo(N0.getNode(), NarrowLoad);
    8482             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8483          41 :         AddToWorklist(oye);
    8484             :       }
    8485          41 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8486             :     }
    8487             : 
    8488             :     // See if the value being truncated is already sign extended.  If so, just
    8489             :     // eliminate the trunc/sext pair.
    8490        6737 :     SDValue Op = N0.getOperand(0);
    8491        6737 :     unsigned OpBits   = Op.getScalarValueSizeInBits();
    8492        6737 :     unsigned MidBits  = N0.getScalarValueSizeInBits();
    8493             :     unsigned DestBits = VT.getScalarSizeInBits();
    8494        6737 :     unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
    8495             : 
    8496        6737 :     if (OpBits == DestBits) {
    8497             :       // Op is i32, Mid is i8, and Dest is i32.  If Op has more than 24 sign
    8498             :       // bits, it is already ready.
    8499        3880 :       if (NumSignBits > DestBits-MidBits)
    8500         900 :         return Op;
    8501        2857 :     } else if (OpBits < DestBits) {
    8502             :       // Op is i32, Mid is i8, and Dest is i64.  If Op has more than 24 sign
    8503             :       // bits, just sext from i32.
    8504        2542 :       if (NumSignBits > OpBits-MidBits)
    8505          50 :         return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
    8506             :     } else {
    8507             :       // Op is i64, Mid is i8, and Dest is i32.  If Op has more than 56 sign
    8508             :       // bits, just truncate to i32.
    8509         315 :       if (NumSignBits > OpBits-MidBits)
    8510          44 :         return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
    8511             :     }
    8512             : 
    8513             :     // fold (sext (truncate x)) -> (sextinreg x).
    8514        5790 :     if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
    8515             :                                                  N0.getValueType())) {
    8516        5790 :       if (OpBits < DestBits)
    8517        2539 :         Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
    8518        3273 :       else if (OpBits > DestBits)
    8519         297 :         Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
    8520        5790 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
    8521       11580 :                          DAG.getValueType(N0.getValueType()));
    8522             :     }
    8523             :   }
    8524             : 
    8525             :   // Try to simplify (sext (load x)).
    8526       44146 :   if (SDValue foldedExt =
    8527       44146 :           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
    8528       44146 :                              ISD::SEXTLOAD, ISD::SIGN_EXTEND))
    8529       13459 :     return foldedExt;
    8530             : 
    8531             :   // fold (sext (load x)) to multiple smaller sextloads.
    8532             :   // Only on illegal but splittable vectors.
    8533       30687 :   if (SDValue ExtLoad = CombineExtLoad(N))
    8534         142 :     return ExtLoad;
    8535             : 
    8536             :   // Try to simplify (sext (sextload x)).
    8537       30545 :   if (SDValue foldedExt = tryToFoldExtOfExtload(
    8538       30545 :           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
    8539           4 :     return foldedExt;
    8540             : 
    8541             :   // fold (sext (and/or/xor (load x), cst)) ->
    8542             :   //      (and/or/xor (sextload x), (sext cst))
    8543       30541 :   if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8544        1247 :        N0.getOpcode() == ISD::XOR) &&
    8545          31 :       isa<LoadSDNode>(N0.getOperand(0)) &&
    8546       30572 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8547           1 :       (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    8548             :     LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    8549             :     EVT MemVT = LN00->getMemoryVT();
    8550           1 :     if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) &&
    8551           2 :       LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
    8552             :       SmallVector<SDNode*, 4> SetCCs;
    8553           1 :       bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
    8554             :                                              ISD::SIGN_EXTEND, SetCCs, TLI);
    8555           1 :       if (DoXform) {
    8556           2 :         SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
    8557             :                                          LN00->getChain(), LN00->getBasePtr(),
    8558             :                                          LN00->getMemoryVT(),
    8559           2 :                                          LN00->getMemOperand());
    8560           2 :         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8561           1 :         Mask = Mask.sext(VT.getSizeInBits());
    8562           1 :         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
    8563           1 :                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
    8564           2 :         ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND);
    8565           1 :         bool NoReplaceTruncAnd = !N0.hasOneUse();
    8566             :         bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
    8567           1 :         CombineTo(N, And);
    8568             :         // If N0 has multiple uses, change other uses as well.
    8569           1 :         if (NoReplaceTruncAnd) {
    8570             :           SDValue TruncAnd =
    8571           0 :               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
    8572           0 :           CombineTo(N0.getNode(), TruncAnd);
    8573             :         }
    8574           1 :         if (NoReplaceTrunc) {
    8575           2 :           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
    8576             :         } else {
    8577           0 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
    8578           0 :                                       LN00->getValueType(0), ExtLoad);
    8579             :           CombineTo(LN00, Trunc, ExtLoad.getValue(1));
    8580             :         }
    8581           1 :         return SDValue(N,0); // Return N so it doesn't get rechecked!
    8582             :       }
    8583             :     }
    8584             :   }
    8585             : 
    8586       30540 :   if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
    8587          10 :     return V;
    8588             : 
    8589       61060 :   if (N0.getOpcode() == ISD::SETCC) {
    8590        5425 :     SDValue N00 = N0.getOperand(0);
    8591        5425 :     SDValue N01 = N0.getOperand(1);
    8592        5425 :     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    8593       10850 :     EVT N00VT = N0.getOperand(0).getValueType();
    8594             : 
    8595             :     // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
    8596             :     // Only do this before legalize for now.
    8597        5425 :     if (VT.isVector() && !LegalOperations &&
    8598        2921 :         TLI.getBooleanContents(N00VT) ==
    8599             :             TargetLowering::ZeroOrNegativeOneBooleanContent) {
    8600             :       // On some architectures (such as SSE/NEON/etc) the SETCC result type is
    8601             :       // of the same size as the compared operands. Only optimize sext(setcc())
    8602             :       // if this is the case.
    8603        2892 :       EVT SVT = getSetCCResultType(N00VT);
    8604             : 
    8605             :       // We know that the # elements of the results is the same as the
    8606             :       // # elements of the compare (and the # elements of the compare result
    8607             :       // for that matter).  Check to see that they are the same size.  If so,
    8608             :       // we know that the element size of the sext'd result matches the
    8609             :       // element size of the compare operands.
    8610        2892 :       if (VT.getSizeInBits() == SVT.getSizeInBits())
    8611        2653 :         return DAG.getSetCC(DL, VT, N00, N01, CC);
    8612             : 
    8613             :       // If the desired elements are smaller or larger than the source
    8614             :       // elements, we can use a matching integer vector type and then
    8615             :       // truncate/sign extend.
    8616         520 :       EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
    8617         523 :       if (SVT == MatchingVecType) {
    8618         281 :         SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
    8619         281 :         return DAG.getSExtOrTrunc(VsetCC, DL, VT);
    8620             :       }
    8621             :     }
    8622             : 
    8623             :     // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
    8624             :     // Here, T can be 1 or -1, depending on the type of the setcc and
    8625             :     // getBooleanContents().
    8626        2772 :     unsigned SetCCWidth = N0.getScalarValueSizeInBits();
    8627             : 
    8628             :     // To determine the "true" side of the select, we need to know the high bit
    8629             :     // of the value returned by the setcc if it evaluates to true.
    8630             :     // If the type of the setcc is i1, then the true case of the select is just
    8631             :     // sext(i1 1), that is, -1.
    8632             :     // If the type of the setcc is larger (say, i8) then the value of the high
    8633             :     // bit depends on getBooleanContents(), so ask TLI for a real "true" value
    8634             :     // of the appropriate width.
    8635             :     SDValue ExtTrueVal = (SetCCWidth == 1)
    8636        2771 :                              ? DAG.getAllOnesConstant(DL, VT)
    8637        2772 :                              : DAG.getBoolConstant(true, DL, VT, N00VT);
    8638        2772 :     SDValue Zero = DAG.getConstant(0, DL, VT);
    8639        2772 :     if (SDValue SCC =
    8640        2772 :             SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC, true))
    8641          22 :       return SCC;
    8642             : 
    8643        2750 :     if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) {
    8644         871 :       EVT SetCCVT = getSetCCResultType(N00VT);
    8645             :       // Don't do this transform for i1 because there's a select transform
    8646             :       // that would reverse it.
    8647             :       // TODO: We should not do this transform at all without a target hook
    8648             :       // because a sext is likely cheaper than a select?
    8649         871 :       if (SetCCVT.getScalarSizeInBits() != 1 &&
    8650         107 :           (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) {
    8651         107 :         SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
    8652         107 :         return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
    8653             :       }
    8654             :     }
    8655             :   }
    8656             : 
    8657             :   // fold (sext x) -> (zext x) if the sign bit is known zero.
    8658       54816 :   if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
    8659       27068 :       DAG.SignBitIsZero(N0))
    8660         884 :     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0);
    8661             : 
    8662       27306 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    8663           4 :     return NewVSel;
    8664             : 
    8665       27302 :   return SDValue();
    8666             : }
    8667             : 
    8668             : // isTruncateOf - If N is a truncate of some other value, return true, record
    8669             : // the value being truncated in Op and which of Op's bits are zero/one in Known.
    8670             : // This function computes KnownBits to avoid a duplicated call to
    8671             : // computeKnownBits in the caller.
    8672           0 : static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
    8673             :                          KnownBits &Known) {
    8674           0 :   if (N->getOpcode() == ISD::TRUNCATE) {
    8675           0 :     Op = N->getOperand(0);
    8676           0 :     DAG.computeKnownBits(Op, Known);
    8677           0 :     return true;
    8678             :   }
    8679             : 
    8680           0 :   if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
    8681           0 :       cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
    8682           0 :     return false;
    8683             : 
    8684           0 :   SDValue Op0 = N->getOperand(0);
    8685           0 :   SDValue Op1 = N->getOperand(1);
    8686             :   assert(Op0.getValueType() == Op1.getValueType());
    8687             : 
    8688           0 :   if (isNullConstant(Op0))
    8689           0 :     Op = Op1;
    8690           0 :   else if (isNullConstant(Op1))
    8691           0 :     Op = Op0;
    8692             :   else
    8693           0 :     return false;
    8694             : 
    8695           0 :   DAG.computeKnownBits(Op, Known);
    8696             : 
    8697           0 :   if (!(Known.Zero | 1).isAllOnesValue())
    8698           0 :     return false;
    8699             : 
    8700             :   return true;
    8701             : }
    8702             : 
    8703      142725 : SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
    8704      142725 :   SDValue N0 = N->getOperand(0);
    8705      142725 :   EVT VT = N->getValueType(0);
    8706             : 
    8707      142725 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8708      142725 :                                               LegalOperations))
    8709         452 :     return SDValue(Res, 0);
    8710             : 
    8711             :   // fold (zext (zext x)) -> (zext x)
    8712             :   // fold (zext (aext x)) -> (zext x)
    8713      284546 :   if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    8714         279 :     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
    8715         558 :                        N0.getOperand(0));
    8716             : 
    8717             :   // fold (zext (truncate x)) -> (zext x) or
    8718             :   //      (zext (truncate x)) -> (truncate x)
    8719             :   // This is valid when the truncated bits of x are already zero.
    8720             :   // FIXME: We should extend this to work for vectors too.
    8721      141994 :   SDValue Op;
    8722      141994 :   KnownBits Known;
    8723      141994 :   if (!VT.isVector() && isTruncateOf(DAG, N0, Op, Known)) {
    8724             :     APInt TruncatedBits =
    8725       33830 :       (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ?
    8726             :       APInt(Op.getValueSizeInBits(), 0) :
    8727             :       APInt::getBitsSet(Op.getValueSizeInBits(),
    8728             :                         N0.getValueSizeInBits(),
    8729       33828 :                         std::min(Op.getValueSizeInBits(),
    8730       67660 :                                  VT.getSizeInBits()));
    8731       33830 :     if (TruncatedBits.isSubsetOf(Known.Zero))
    8732       31937 :       return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
    8733             :   }
    8734             : 
    8735             :   // fold (zext (truncate x)) -> (and x, mask)
    8736      253298 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8737             :     // fold (zext (truncate (load x))) -> (zext (smaller load x))
    8738             :     // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
    8739       18670 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8740          13 :       SDNode *oye = N0.getOperand(0).getNode();
    8741          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8742          13 :         CombineTo(N0.getNode(), NarrowLoad);
    8743             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8744          13 :         AddToWorklist(oye);
    8745             :       }
    8746          13 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    8747             :     }
    8748             : 
    8749       37314 :     EVT SrcVT = N0.getOperand(0).getValueType();
    8750       18657 :     EVT MinVT = N0.getValueType();
    8751             : 
    8752             :     // Try to mask before the extension to avoid having to generate a larger mask,
    8753             :     // possibly over several sub-vectors.
    8754       30722 :     if (SrcVT.bitsLT(VT) && VT.isVector()) {
    8755          51 :       if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
    8756             :                                TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) {
    8757          51 :         SDValue Op = N0.getOperand(0);
    8758         102 :         Op = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
    8759          51 :         AddToWorklist(Op.getNode());
    8760          51 :         SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
    8761             :         // Transfer the debug info; the new node is equivalent to N0.
    8762          51 :         DAG.transferDbgValues(N0, ZExtOrTrunc);
    8763          51 :         return ZExtOrTrunc;
    8764             :       }
    8765             :     }
    8766             : 
    8767       18606 :     if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
    8768       18720 :       SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
    8769       18606 :       AddToWorklist(Op.getNode());
    8770       37212 :       SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
    8771             :       // We may safely transfer the debug info describing the truncate node over
    8772             :       // to the equivalent and operation.
    8773       18606 :       DAG.transferDbgValues(N0, And);
    8774       18606 :       return And;
    8775             :     }
    8776             :   }
    8777             : 
    8778             :   // Fold (zext (and (trunc x), cst)) -> (and x, cst),
    8779             :   // if either of the casts is not free.
    8780      107979 :   if (N0.getOpcode() == ISD::AND &&
    8781        2667 :       N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
    8782      108834 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8783         712 :       (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
    8784         411 :                            N0.getValueType()) ||
    8785         110 :        !TLI.isZExtFree(N0.getValueType(), VT))) {
    8786         706 :     SDValue X = N0.getOperand(0).getOperand(0);
    8787         361 :     X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
    8788         706 :     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8789         706 :     Mask = Mask.zext(VT.getSizeInBits());
    8790             :     SDLoc DL(N);
    8791         353 :     return DAG.getNode(ISD::AND, DL, VT,
    8792         353 :                        X, DAG.getConstant(Mask, DL, VT));
    8793             :   }
    8794             : 
    8795             :   // Try to simplify (zext (load x)).
    8796      107626 :   if (SDValue foldedExt =
    8797      107626 :           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
    8798      107626 :                              ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
    8799        6873 :     return foldedExt;
    8800             : 
    8801             :   // fold (zext (load x)) to multiple smaller zextloads.
    8802             :   // Only on illegal but splittable vectors.
    8803      100753 :   if (SDValue ExtLoad = CombineExtLoad(N))
    8804          70 :     return ExtLoad;
    8805             : 
    8806             :   // fold (zext (and/or/xor (load x), cst)) ->
    8807             :   //      (and/or/xor (zextload x), (zext cst))
    8808             :   // Unless (and (load x) cst) will match as a zextload already and has
    8809             :   // additional users.
    8810      100683 :   if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
    8811       15668 :        N0.getOpcode() == ISD::XOR) &&
    8812         200 :       isa<LoadSDNode>(N0.getOperand(0)) &&
    8813      100883 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8814          83 :       (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    8815             :     LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    8816             :     EVT MemVT = LN00->getMemoryVT();
    8817          30 :     if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) &&
    8818          70 :         LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
    8819             :       bool DoXform = true;
    8820             :       SmallVector<SDNode*, 4> SetCCs;
    8821          30 :       if (!N0.hasOneUse()) {
    8822          18 :         if (N0.getOpcode() == ISD::AND) {
    8823             :           auto *AndC = cast<ConstantSDNode>(N0.getOperand(1));
    8824           0 :           EVT LoadResultTy = AndC->getValueType(0);
    8825           0 :           EVT ExtVT;
    8826           0 :           if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
    8827             :             DoXform = false;
    8828             :         }
    8829             :       }
    8830             :       if (DoXform)
    8831          60 :         DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
    8832             :                                           ISD::ZERO_EXTEND, SetCCs, TLI);
    8833          30 :       if (DoXform) {
    8834          60 :         SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
    8835             :                                          LN00->getChain(), LN00->getBasePtr(),
    8836             :                                          LN00->getMemoryVT(),
    8837          30 :                                          LN00->getMemOperand());
    8838          60 :         APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8839          60 :         Mask = Mask.zext(VT.getSizeInBits());
    8840             :         SDLoc DL(N);
    8841          30 :         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
    8842          30 :                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
    8843          60 :         ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
    8844          30 :         bool NoReplaceTruncAnd = !N0.hasOneUse();
    8845             :         bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
    8846          30 :         CombineTo(N, And);
    8847             :         // If N0 has multiple uses, change other uses as well.
    8848          30 :         if (NoReplaceTruncAnd) {
    8849             :           SDValue TruncAnd =
    8850          27 :               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
    8851           9 :           CombineTo(N0.getNode(), TruncAnd);
    8852             :         }
    8853          30 :         if (NoReplaceTrunc) {
    8854          58 :           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
    8855             :         } else {
    8856           1 :           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
    8857           2 :                                       LN00->getValueType(0), ExtLoad);
    8858             :           CombineTo(LN00, Trunc, ExtLoad.getValue(1));
    8859             :         }
    8860          30 :         return SDValue(N,0); // Return N so it doesn't get rechecked!
    8861             :       }
    8862             :     }
    8863             :   }
    8864             : 
    8865             :   // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
    8866             :   //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
    8867      100653 :   if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
    8868          26 :     return ZExtLoad;
    8869             : 
    8870             :   // Try to simplify (zext (zextload x)).
    8871      100627 :   if (SDValue foldedExt = tryToFoldExtOfExtload(
    8872      100627 :           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
    8873          17 :     return foldedExt;
    8874             : 
    8875      100610 :   if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
    8876          22 :     return V;
    8877             : 
    8878      201176 :   if (N0.getOpcode() == ISD::SETCC) {
    8879             :     // Only do this before legalize for now.
    8880       54757 :     if (!LegalOperations && VT.isVector() &&
    8881       28342 :         N0.getValueType().getVectorElementType() == MVT::i1) {
    8882         678 :       EVT N00VT = N0.getOperand(0).getValueType();
    8883        1017 :       if (getSetCCResultType(N00VT) == N0.getValueType())
    8884         142 :         return SDValue();
    8885             : 
    8886             :       // We know that the # elements of the results is the same as the #
    8887             :       // elements of the compare (and the # elements of the compare result for
    8888             :       // that matter). Check to see that they are the same size. If so, we know
    8889             :       // that the element size of the sext'd result matches the element size of
    8890             :       // the compare operands.
    8891             :       SDLoc DL(N);
    8892         197 :       SDValue VecOnes = DAG.getConstant(1, DL, VT);
    8893         197 :       if (VT.getSizeInBits() == N00VT.getSizeInBits()) {
    8894             :         // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
    8895         178 :         SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
    8896         356 :                                      N0.getOperand(1), N0.getOperand(2));
    8897         356 :         return DAG.getNode(ISD::AND, DL, VT, VSetCC, VecOnes);
    8898             :       }
    8899             : 
    8900             :       // If the desired elements are smaller or larger than the source
    8901             :       // elements we can use a matching integer vector type and then
    8902             :       // truncate/sign extend.
    8903          19 :       EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
    8904             :       SDValue VsetCC =
    8905          19 :           DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
    8906          38 :                       N0.getOperand(1), N0.getOperand(2));
    8907          19 :       return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT),
    8908          19 :                          VecOnes);
    8909             :     }
    8910             : 
    8911             :     // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    8912             :     SDLoc DL(N);
    8913       27662 :     if (SDValue SCC = SimplifySelectCC(
    8914       27662 :             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
    8915       27662 :             DAG.getConstant(0, DL, VT),
    8916       55324 :             cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
    8917         137 :       return SCC;
    8918             :   }
    8919             : 
    8920             :   // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
    8921      100112 :   if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
    8922        7229 :       isa<ConstantSDNode>(N0.getOperand(1)) &&
    8923      107352 :       N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
    8924          11 :       N0.hasOneUse()) {
    8925          10 :     SDValue ShAmt = N0.getOperand(1);
    8926          20 :     unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
    8927          10 :     if (N0.getOpcode() == ISD::SHL) {
    8928           9 :       SDValue InnerZExt = N0.getOperand(0);
    8929             :       // If the original shl may be shifting out bits, do not perform this
    8930             :       // transformation.
    8931           9 :       unsigned KnownZeroBits = InnerZExt.getValueSizeInBits() -
    8932          18 :         InnerZExt.getOperand(0).getValueSizeInBits();
    8933           9 :       if (ShAmtVal > KnownZeroBits)
    8934           0 :         return SDValue();
    8935             :     }
    8936             : 
    8937             :     SDLoc DL(N);
    8938             : 
    8939             :     // Ensure that the shift amount is wide enough for the shifted value.
    8940          10 :     if (VT.getSizeInBits() >= 256)
    8941           0 :       ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
    8942             : 
    8943          10 :     return DAG.getNode(N0.getOpcode(), DL, VT,
    8944          10 :                        DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
    8945          10 :                        ShAmt);
    8946             :   }
    8947             : 
    8948      100102 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    8949           2 :     return NewVSel;
    8950             : 
    8951      100100 :   return SDValue();
    8952             : }
    8953             : 
    8954       88162 : SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
    8955       88162 :   SDValue N0 = N->getOperand(0);
    8956       88162 :   EVT VT = N->getValueType(0);
    8957             : 
    8958       88162 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    8959       88162 :                                               LegalOperations))
    8960          78 :     return SDValue(Res, 0);
    8961             : 
    8962             :   // fold (aext (aext x)) -> (aext x)
    8963             :   // fold (aext (zext x)) -> (zext x)
    8964             :   // fold (aext (sext x)) -> (sext x)
    8965       88084 :   if (N0.getOpcode() == ISD::ANY_EXTEND  ||
    8966      176151 :       N0.getOpcode() == ISD::ZERO_EXTEND ||
    8967             :       N0.getOpcode() == ISD::SIGN_EXTEND)
    8968          44 :     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
    8969             : 
    8970             :   // fold (aext (truncate (load x))) -> (aext (smaller load x))
    8971             :   // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
    8972       88062 :   if (N0.getOpcode() == ISD::TRUNCATE) {
    8973        7527 :     if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
    8974          13 :       SDNode *oye = N0.getOperand(0).getNode();
    8975          13 :       if (NarrowLoad.getNode() != N0.getNode()) {
    8976          13 :         CombineTo(N0.getNode(), NarrowLoad);
    8977             :         // CombineTo deleted the truncate, if needed, but not what's under it.
    8978          13 :         AddToWorklist(oye);
    8979             :       }
    8980          13 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    8981             :     }
    8982             :   }
    8983             : 
    8984             :   // fold (aext (truncate x))
    8985       88049 :   if (N0.getOpcode() == ISD::TRUNCATE)
    8986       15117 :     return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
    8987             : 
    8988             :   // Fold (aext (and (trunc x), cst)) -> (and x, cst)
    8989             :   // if the trunc is not free.
    8990        4001 :   if (N0.getOpcode() == ISD::AND &&
    8991        4001 :       N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
    8992       86633 :       N0.getOperand(1).getOpcode() == ISD::Constant &&
    8993        5988 :       !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
    8994        2994 :                           N0.getValueType())) {
    8995             :     SDLoc DL(N);
    8996           8 :     SDValue X = N0.getOperand(0).getOperand(0);
    8997           8 :     X = DAG.getAnyExtOrTrunc(X, DL, VT);
    8998           8 :     APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
    8999           8 :     Mask = Mask.zext(VT.getSizeInBits());
    9000           8 :     return DAG.getNode(ISD::AND, DL, VT,
    9001           8 :                        X, DAG.getConstant(Mask, DL, VT));
    9002             :   }
    9003             : 
    9004             :   // fold (aext (load x)) -> (aext (truncate (extload x)))
    9005             :   // None of the supported targets knows how to perform load and any_ext
    9006             :   // on vectors in one instruction.  We only perform this transformation on
    9007             :   // scalars.
    9008       15059 :   if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
    9009       14875 :       ISD::isUNINDEXEDLoad(N0.getNode()) &&
    9010       15011 :       TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) {
    9011             :     bool DoXform = true;
    9012             :     SmallVector<SDNode*, 4> SetCCs;
    9013       14670 :     if (!N0.hasOneUse())
    9014       10994 :       DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs,
    9015             :                                         TLI);
    9016       10994 :     if (DoXform) {
    9017             :       LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9018       28980 :       SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
    9019             :                                        LN0->getChain(),
    9020             :                                        LN0->getBasePtr(), N0.getValueType(),
    9021       14568 :                                        LN0->getMemOperand());
    9022       14490 :       ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
    9023             :       // If the load value is used only by N, replace it via CombineTo N.
    9024             :       bool NoReplaceTrunc = N0.hasOneUse();
    9025       14490 :       CombineTo(N, ExtLoad);
    9026       14490 :       if (NoReplaceTrunc) {
    9027        7352 :         DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    9028             :       } else {
    9029       10814 :         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
    9030       21628 :                                     N0.getValueType(), ExtLoad);
    9031             :         CombineTo(LN0, Trunc, ExtLoad.getValue(1));
    9032             :       }
    9033       14490 :       return SDValue(N, 0); // Return N so it doesn't get rechecked!
    9034             :     }
    9035             :   }
    9036             : 
    9037             :   // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
    9038             :   // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
    9039             :   // fold (aext ( extload x)) -> (aext (truncate (extload  x)))
    9040             :   if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) &&
    9041       66472 :       ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
    9042             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9043             :     ISD::LoadExtType ExtType = LN0->getExtensionType();
    9044         263 :     EVT MemVT = LN0->getMemoryVT();
    9045         308 :     if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) {
    9046         239 :       SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
    9047             :                                        VT, LN0->getChain(), LN0->getBasePtr(),
    9048         239 :                                        MemVT, LN0->getMemOperand());
    9049         239 :       CombineTo(N, ExtLoad);
    9050         478 :       DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
    9051         239 :       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    9052             :     }
    9053             :   }
    9054             : 
    9055       65798 :   if (N0.getOpcode() == ISD::SETCC) {
    9056             :     // For vectors:
    9057             :     // aext(setcc) -> vsetcc
    9058             :     // aext(setcc) -> truncate(vsetcc)
    9059             :     // aext(setcc) -> aext(vsetcc)
    9060             :     // Only do this before legalize for now.
    9061        3230 :     if (VT.isVector() && !LegalOperations) {
    9062         948 :       EVT N00VT = N0.getOperand(0).getValueType();
    9063         948 :       if (getSetCCResultType(N00VT) == N0.getValueType())
    9064          89 :         return SDValue();
    9065             : 
    9066             :       // We know that the # elements of the results is the same as the
    9067             :       // # elements of the compare (and the # elements of the compare result
    9068             :       // for that matter).  Check to see that they are the same size.  If so,
    9069             :       // we know that the element size of the sext'd result matches the
    9070             :       // element size of the compare operands.
    9071         385 :       if (VT.getSizeInBits() == N00VT.getSizeInBits())
    9072         558 :         return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0),
    9073             :                              N0.getOperand(1),
    9074         279 :                              cast<CondCodeSDNode>(N0.getOperand(2))->get());
    9075             :       // If the desired elements are smaller or larger than the source
    9076             :       // elements we can use a matching integer vector type and then
    9077             :       // truncate/any extend
    9078             :       else {
    9079         106 :         EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
    9080             :         SDValue VsetCC =
    9081         106 :           DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0),
    9082             :                         N0.getOperand(1),
    9083         106 :                         cast<CondCodeSDNode>(N0.getOperand(2))->get());
    9084         212 :         return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT);
    9085             :       }
    9086             :     }
    9087             : 
    9088             :     // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    9089             :     SDLoc DL(N);
    9090        2756 :     if (SDValue SCC = SimplifySelectCC(
    9091        2756 :             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
    9092        2756 :             DAG.getConstant(0, DL, VT),
    9093        2756 :             cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
    9094         138 :       return SCC;
    9095             :   }
    9096             : 
    9097       65186 :   return SDValue();
    9098             : }
    9099             : 
    9100           0 : SDValue DAGCombiner::visitAssertExt(SDNode *N) {
    9101           0 :   unsigned Opcode = N->getOpcode();
    9102           0 :   SDValue N0 = N->getOperand(0);
    9103           0 :   SDValue N1 = N->getOperand(1);
    9104           0 :   EVT AssertVT = cast<VTSDNode>(N1)->getVT();
    9105             : 
    9106             :   // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt)
    9107           0 :   if (N0.getOpcode() == Opcode &&
    9108             :       AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT())
    9109           0 :     return N0;
    9110             : 
    9111           0 :   if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
    9112           0 :       N0.getOperand(0).getOpcode() == Opcode) {
    9113             :     // We have an assert, truncate, assert sandwich. Make one stronger assert
    9114             :     // by asserting on the smallest asserted type to the larger source type.
    9115             :     // This eliminates the later assert:
    9116             :     // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN
    9117             :     // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN
    9118           0 :     SDValue BigA = N0.getOperand(0);
    9119           0 :     EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
    9120             :     assert(BigA_AssertVT.bitsLE(N0.getValueType()) &&
    9121             :            "Asserting zero/sign-extended bits to a type larger than the "
    9122             :            "truncated destination does not provide information");
    9123             : 
    9124             :     SDLoc DL(N);
    9125           0 :     EVT MinAssertVT = AssertVT.bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
    9126           0 :     SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
    9127           0 :     SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
    9128           0 :                                     BigA.getOperand(0), MinAssertVTVal);
    9129           0 :     return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
    9130             :   }
    9131             : 
    9132           0 :   return SDValue();
    9133             : }
    9134             : 
    9135             : /// If the result of a wider load is shifted to right of N  bits and then
    9136             : /// truncated to a narrower type and where N is a multiple of number of bits of
    9137             : /// the narrower type, transform it to a narrower load from address + N / num of
    9138             : /// bits of new type. Also narrow the load if the result is masked with an AND
    9139             : /// to effectively produce a smaller type. If the result is to be extended, also
    9140             : /// fold the extension to form a extending load.
    9141     1115566 : SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
    9142     1115566 :   unsigned Opc = N->getOpcode();
    9143             : 
    9144             :   ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
    9145     1115566 :   SDValue N0 = N->getOperand(0);
    9146     1115566 :   EVT VT = N->getValueType(0);
    9147     1115566 :   EVT ExtVT = VT;
    9148             : 
    9149             :   // This transformation isn't valid for vector loads.
    9150     1115566 :   if (VT.isVector())
    9151       16240 :     return SDValue();
    9152             : 
    9153             :   unsigned ShAmt = 0;
    9154             :   bool HasShiftedOffset = false;
    9155             :   // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
    9156             :   // extended to VT.
    9157     1099326 :   if (Opc == ISD::SIGN_EXTEND_INREG) {
    9158             :     ExtType = ISD::SEXTLOAD;
    9159       41766 :     ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
    9160     1057560 :   } else if (Opc == ISD::SRL) {
    9161             :     // Another special-case: SRL is basically zero-extending a narrower value,
    9162             :     // or it maybe shifting a higher subword, half or byte into the lowest
    9163             :     // bits.
    9164             :     ExtType = ISD::ZEXTLOAD;
    9165      138786 :     N0 = SDValue(N, 0);
    9166             : 
    9167             :     auto *LN0 = dyn_cast<LoadSDNode>(N0.getOperand(0));
    9168             :     auto *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    9169      138786 :     if (!N01 || !LN0)
    9170      116385 :       return SDValue();
    9171             : 
    9172       22401 :     uint64_t ShiftAmt = N01->getZExtValue();
    9173       22401 :     uint64_t MemoryWidth = LN0->getMemoryVT().getSizeInBits();
    9174       22401 :     if (LN0->getExtensionType() != ISD::SEXTLOAD && MemoryWidth > ShiftAmt)
    9175       22232 :       ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShiftAmt);
    9176             :     else
    9177         169 :       ExtVT = EVT::getIntegerVT(*DAG.getContext(),
    9178         169 :                                 VT.getSizeInBits() - ShiftAmt);
    9179      918774 :   } else if (Opc == ISD::AND) {
    9180             :     // An AND with a constant mask is the same as a truncate + zero-extend.
    9181             :     auto AndC = dyn_cast<ConstantSDNode>(N->getOperand(1));
    9182             :     if (!AndC)
    9183           0 :       return SDValue();
    9184             : 
    9185       80218 :     const APInt &Mask = AndC->getAPIntValue();
    9186             :     unsigned ActiveBits = 0;
    9187       80218 :     if (Mask.isMask()) {
    9188             :       ActiveBits = Mask.countTrailingOnes();
    9189        8824 :     } else if (Mask.isShiftedMask()) {
    9190        7785 :       ShAmt = Mask.countTrailingZeros();
    9191        7785 :       APInt ShiftedMask = Mask.lshr(ShAmt);
    9192             :       ActiveBits = ShiftedMask.countTrailingOnes();
    9193             :       HasShiftedOffset = true;
    9194             :     } else
    9195        1039 :       return SDValue();
    9196             : 
    9197             :     ExtType = ISD::ZEXTLOAD;
    9198       79179 :     ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
    9199             :   }
    9200             : 
    9201     1963804 :   if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
    9202       63686 :     SDValue SRL = N0;
    9203             :     if (auto *ConstShift = dyn_cast<ConstantSDNode>(SRL.getOperand(1))) {
    9204       62198 :       ShAmt = ConstShift->getZExtValue();
    9205       62198 :       unsigned EVTBits = ExtVT.getSizeInBits();
    9206             :       // Is the shift amount a multiple of size of VT?
    9207       62198 :       if ((ShAmt & (EVTBits-1)) == 0) {
    9208       53583 :         N0 = N0.getOperand(0);
    9209             :         // Is the load width a multiple of size of VT?
    9210       53583 :         if ((N0.getValueSizeInBits() & (EVTBits-1)) != 0)
    9211          21 :           return SDValue();
    9212             :       }
    9213             : 
    9214             :       // At this point, we must have a load or else we can't do the transform.
    9215       62177 :       if (!isa<LoadSDNode>(N0)) return SDValue();
    9216             : 
    9217             :       auto *LN0 = cast<LoadSDNode>(N0);
    9218             : 
    9219             :       // Because a SRL must be assumed to *need* to zero-extend the high bits
    9220             :       // (as opposed to anyext the high bits), we can't combine the zextload
    9221             :       // lowering of SRL and an sextload.
    9222       22980 :       if (LN0->getExtensionType() == ISD::SEXTLOAD)
    9223         188 :         return SDValue();
    9224             : 
    9225             :       // If the shift amount is larger than the input type then we're not
    9226             :       // accessing any of the loaded bytes.  If the load was a zextload/extload
    9227             :       // then the result of the shift+trunc is zero/undef (handled elsewhere).
    9228       22792 :       if (ShAmt >= LN0->getMemoryVT().getSizeInBits())
    9229           4 :         return SDValue();
    9230             : 
    9231             :       // If the SRL is only used by a masking AND, we may be able to adjust
    9232             :       // the ExtVT to make the AND redundant.
    9233       22788 :       SDNode *Mask = *(SRL->use_begin());
    9234       22788 :       if (Mask->getOpcode() == ISD::AND &&
    9235        7819 :           isa<ConstantSDNode>(Mask->getOperand(1))) {
    9236             :         const APInt &ShiftMask =
    9237        7812 :           cast<ConstantSDNode>(Mask->getOperand(1))->getAPIntValue();
    9238        7812 :         if (ShiftMask.isMask()) {
    9239        6141 :           EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(),
    9240        6141 :                                            ShiftMask.countTrailingOnes());
    9241             :           // If the mask is smaller, recompute the type.
    9242        9738 :           if ((ExtVT.getSizeInBits() > MaskedVT.getSizeInBits()) &&
    9243       10254 :               TLI.isLoadExtLegal(ExtType, N0.getValueType(), MaskedVT))
    9244         773 :             ExtVT = MaskedVT;
    9245             :         }
    9246             :       }
    9247             :     }
    9248             :   }
    9249             : 
    9250             :   // If the load is shifted left (and the result isn't shifted back right),
    9251             :   // we can fold the truncate through the shift.
    9252             :   unsigned ShLeftAmt = 0;
    9253     1823842 :   if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
    9254      947018 :       ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) {
    9255         393 :     if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    9256           9 :       ShLeftAmt = N01->getZExtValue();
    9257           9 :       N0 = N0.getOperand(0);
    9258             :     }
    9259             :   }
    9260             : 
    9261             :   // If we haven't found a load, we can't narrow it.
    9262      942492 :   if (!isa<LoadSDNode>(N0))
    9263      792923 :     return SDValue();
    9264             : 
    9265      149569 :   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9266      149569 :   if (!isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
    9267      146449 :     return SDValue();
    9268             : 
    9269             :   auto AdjustBigEndianShift = [&](unsigned ShAmt) {
    9270             :     unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
    9271             :     unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
    9272             :     return LVTStoreBits - EVTStoreBits - ShAmt;
    9273             :   };
    9274             : 
    9275             :   // For big endian targets, we need to adjust the offset to the pointer to
    9276             :   // load the correct bytes.
    9277        3120 :   if (DAG.getDataLayout().isBigEndian())
    9278         167 :     ShAmt = AdjustBigEndianShift(ShAmt);
    9279             : 
    9280        6240 :   EVT PtrType = N0.getOperand(1).getValueType();
    9281        3120 :   uint64_t PtrOff = ShAmt / 8;
    9282        3120 :   unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
    9283        3120 :   SDLoc DL(LN0);
    9284             :   // The original load itself didn't wrap, so an offset within it doesn't.
    9285             :   SDNodeFlags Flags;
    9286             :   Flags.setNoUnsignedWrap(true);
    9287        3120 :   SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
    9288        3120 :                                PtrType, LN0->getBasePtr(),
    9289             :                                DAG.getConstant(PtrOff, DL, PtrType),
    9290        3120 :                                Flags);
    9291        3120 :   AddToWorklist(NewPtr.getNode());
    9292             : 
    9293             :   SDValue Load;
    9294        3120 :   if (ExtType == ISD::NON_EXTLOAD)
    9295        5022 :     Load = DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr,
    9296             :                        LN0->getPointerInfo().getWithOffset(PtrOff), NewAlign,
    9297       12555 :                        LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
    9298             :   else
    9299        1218 :     Load = DAG.getExtLoad(ExtType, SDLoc(N0), VT, LN0->getChain(), NewPtr,
    9300             :                           LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
    9301         609 :                           NewAlign, LN0->getMemOperand()->getFlags(),
    9302        3045 :                           LN0->getAAInfo());
    9303             : 
    9304             :   // Replace the old load's chain with the new load's chain.
    9305             :   WorklistRemover DeadNodes(*this);
    9306        3120 :   DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
    9307             : 
    9308             :   // Shift the result left, if we've swallowed a left shift.
    9309        3120 :   SDValue Result = Load;
    9310        3120 :   if (ShLeftAmt != 0) {
    9311           9 :     EVT ShImmTy = getShiftAmountTy(Result.getValueType());
    9312           9 :     if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt))
    9313           0 :       ShImmTy = VT;
    9314             :     // If the shift amount is as large as the result size (but, presumably,
    9315             :     // no larger than the source) then the useful bits of the result are
    9316             :     // zero; we can't simply return the shortened shift, because the result
    9317             :     // of that operation is undefined.
    9318             :     SDLoc DL(N0);
    9319           9 :     if (ShLeftAmt >= VT.getSizeInBits())
    9320           7 :       Result = DAG.getConstant(0, DL, VT);
    9321             :     else
    9322           2 :       Result = DAG.getNode(ISD::SHL, DL, VT,
    9323           2 :                           Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
    9324             :   }
    9325             : 
    9326        3120 :   if (HasShiftedOffset) {
    9327             :     // Recalculate the shift amount after it has been altered to calculate
    9328             :     // the offset.
    9329          80 :     if (DAG.getDataLayout().isBigEndian())
    9330          16 :       ShAmt = AdjustBigEndianShift(ShAmt);
    9331             : 
    9332             :     // We're using a shifted mask, so the load now has an offset. This means we
    9333             :     // now need to shift right the mask to match the new load and then shift
    9334             :     // right the result of the AND.
    9335         160 :     const APInt &Mask = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue();
    9336          80 :     APInt ShiftedMask = Mask.lshr(ShAmt);
    9337          80 :     DAG.UpdateNodeOperands(N, Result, DAG.getConstant(ShiftedMask, DL, VT));
    9338          80 :     SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
    9339          80 :     SDValue Shifted = DAG.getNode(ISD::SHL, DL, VT, SDValue(N, 0),
    9340          80 :                                   ShiftC);
    9341         160 :     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Shifted);
    9342         160 :     DAG.UpdateNodeOperands(Shifted.getNode(), SDValue(N, 0), ShiftC);
    9343             :   }
    9344             :   // Return the new loaded value.
    9345        3120 :   return Result;
    9346             : }
    9347             : 
    9348       46974 : SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
    9349       46974 :   SDValue N0 = N->getOperand(0);
    9350       46974 :   SDValue N1 = N->getOperand(1);
    9351       46974 :   EVT VT = N->getValueType(0);
    9352       46974 :   EVT EVT = cast<VTSDNode>(N1)->getVT();
    9353             :   unsigned VTBits = VT.getScalarSizeInBits();
    9354             :   unsigned EVTBits = EVT.getScalarSizeInBits();
    9355             : 
    9356       46974 :   if (N0.isUndef())
    9357           1 :     return DAG.getUNDEF(VT);
    9358             : 
    9359             :   // fold (sext_in_reg c1) -> c1
    9360       46973 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
    9361          24 :     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
    9362             : 
    9363             :   // If the input is already sign extended, just drop the extension.
    9364       46961 :   if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
    9365        1473 :     return N0;
    9366             : 
    9367             :   // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
    9368       45488 :   if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
    9369           4 :       EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
    9370           4 :     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    9371           8 :                        N0.getOperand(0), N1);
    9372             : 
    9373             :   // fold (sext_in_reg (sext x)) -> (sext x)
    9374             :   // fold (sext_in_reg (aext x)) -> (sext x)
    9375             :   // if x is small enough.
    9376       45484 :   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
    9377        9486 :     SDValue N00 = N0.getOperand(0);
    9378        9486 :     if (N00.getScalarValueSizeInBits() <= EVTBits &&
    9379           4 :         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
    9380           4 :       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
    9381             :   }
    9382             : 
    9383             :   // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
    9384       45468 :   if ((N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
    9385       45468 :        N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
    9386       45496 :        N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
    9387          14 :       N0.getOperand(0).getScalarValueSizeInBits() == EVTBits) {
    9388           0 :     if (!LegalOperations ||
    9389           0 :         TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))
    9390           0 :       return DAG.getSignExtendVectorInReg(N0.getOperand(0), SDLoc(N), VT);
    9391             :   }
    9392             : 
    9393             :   // fold (sext_in_reg (zext x)) -> (sext x)
    9394             :   // iff we are extending the source sign bit.
    9395       45482 :   if (N0.getOpcode() == ISD::ZERO_EXTEND) {
    9396           6 :     SDValue N00 = N0.getOperand(0);
    9397           6 :     if (N00.getScalarValueSizeInBits() == EVTBits &&
    9398           4 :         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
    9399           8 :       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
    9400             :   }
    9401             : 
    9402             :   // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
    9403       90956 :   if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, EVTBits - 1)))
    9404           6 :     return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT.getScalarType());
    9405             : 
    9406             :   // fold operands of sext_in_reg based on knowledge that the top bits are not
    9407             :   // demanded.
    9408       45476 :   if (SimplifyDemandedBits(SDValue(N, 0)))
    9409        2118 :     return SDValue(N, 0);
    9410             : 
    9411             :   // fold (sext_in_reg (load x)) -> (smaller sextload x)
    9412             :   // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
    9413       43358 :   if (SDValue NarrowLoad = ReduceLoadWidth(N))
    9414         206 :     return NarrowLoad;
    9415             : 
    9416             :   // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
    9417             :   // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
    9418             :   // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
    9419       43152 :   if (N0.getOpcode() == ISD::SRL) {
    9420             :     if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
    9421       20196 :       if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
    9422             :         // We can turn this into an SRA iff the input to the SRL is already sign
    9423             :         // extended enough.
    9424       20194 :         unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
    9425       20194 :         if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
    9426        2358 :           return DAG.getNode(ISD::SRA, SDLoc(N), VT,
    9427        4716 :                              N0.getOperand(0), N0.getOperand(1));
    9428             :       }
    9429             :   }
    9430             : 
    9431             :   // fold (sext_inreg (extload x)) -> (sextload x)
    9432             :   // If sextload is not supported by target, we can only do the combine when
    9433             :   // load has one use. Doing otherwise can block folding the extload with other
    9434             :   // extends that the target does support.
    9435             :   if (ISD::isEXTLoad(N0.getNode()) &&
    9436          40 :       ISD::isUNINDEXEDLoad(N0.getNode()) &&
    9437        1700 :       EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
    9438        2412 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile() &&
    9439        1043 :         N0.hasOneUse()) ||
    9440        1061 :        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
    9441             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9442         727 :     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
    9443             :                                      LN0->getChain(),
    9444             :                                      LN0->getBasePtr(), EVT,
    9445         728 :                                      LN0->getMemOperand());
    9446         727 :     CombineTo(N, ExtLoad);
    9447             :     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    9448         727 :     AddToWorklist(ExtLoad.getNode());
    9449         727 :     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    9450             :   }
    9451             :   // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
    9452          73 :   if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
    9453           0 :       N0.hasOneUse() &&
    9454           0 :       EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
    9455           0 :       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
    9456           0 :        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
    9457             :     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9458           0 :     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
    9459             :                                      LN0->getChain(),
    9460             :                                      LN0->getBasePtr(), EVT,
    9461           0 :                                      LN0->getMemOperand());
    9462           0 :     CombineTo(N, ExtLoad);
    9463             :     CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
    9464           0 :     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    9465             :   }
    9466             : 
    9467             :   // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
    9468       40067 :   if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
    9469          58 :     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
    9470          58 :                                            N0.getOperand(1), false))
    9471           8 :       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
    9472          16 :                          BSwap, N1);
    9473             :   }
    9474             : 
    9475       40059 :   return SDValue();
    9476             : }
    9477             : 
    9478        3049 : SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
    9479        3049 :   SDValue N0 = N->getOperand(0);
    9480        6098 :   EVT VT = N->getValueType(0);
    9481             : 
    9482        3049 :   if (N0.isUndef())
    9483           0 :     return DAG.getUNDEF(VT);
    9484             : 
    9485        3049 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    9486        3049 :                                               LegalOperations))
    9487          20 :     return SDValue(Res, 0);
    9488             : 
    9489        3029 :   return SDValue();
    9490             : }
    9491             : 
    9492        5864 : SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
    9493        5864 :   SDValue N0 = N->getOperand(0);
    9494       11728 :   EVT VT = N->getValueType(0);
    9495             : 
    9496        5864 :   if (N0.isUndef())
    9497           0 :     return DAG.getUNDEF(VT);
    9498             : 
    9499        5864 :   if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
    9500        5864 :                                               LegalOperations))
    9501          22 :     return SDValue(Res, 0);
    9502             : 
    9503        5842 :   return SDValue();
    9504             : }
    9505             : 
    9506      858614 : SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
    9507      858614 :   SDValue N0 = N->getOperand(0);
    9508      858614 :   EVT VT = N->getValueType(0);
    9509      858614 :   bool isLE = DAG.getDataLayout().isLittleEndian();
    9510             : 
    9511             :   // noop truncate
    9512     1717674 :   if (N0.getValueType() == N->getValueType(0))
    9513           0 :     return N0;
    9514             : 
    9515             :   // fold (truncate (truncate x)) -> (truncate x)
    9516      858614 :   if (N0.getOpcode() == ISD::TRUNCATE)
    9517        4587 :     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
    9518             : 
    9519             :   // fold (truncate c1) -> c1
    9520      856331 :   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
    9521        3355 :     SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
    9522        3355 :     if (C.getNode() != N)
    9523        3332 :       return C;
    9524             :   }
    9525             : 
    9526             :   // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
    9527      852999 :   if (N0.getOpcode() == ISD::ZERO_EXTEND ||
    9528     1703424 :       N0.getOpcode() == ISD::SIGN_EXTEND ||
    9529             :       N0.getOpcode() == ISD::ANY_EXTEND) {
    9530             :     // if the source is smaller than the dest, we still need an extend.
    9531       12544 :     if (N0.getOperand(0).getValueType().bitsLT(VT))
    9532        2422 :       return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
    9533             :     // if the source is larger than the dest, than we just need the truncate.
    9534        5061 :     if (N0.getOperand(0).getValueType().bitsGT(VT))
    9535        2276 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
    9536             :     // if the source and dest are the same type, we can drop both the extend
    9537             :     // and the truncate.
    9538        3967 :     return N0.getOperand(0);
    9539             :   }
    9540             : 
    9541             :   // If this is anyext(trunc), don't fold it, allow ourselves to be folded.
    9542      831141 :   if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND))
    9543        1931 :     return SDValue();
    9544             : 
    9545             :   // Fold extract-and-trunc into a narrow extract. For example:
    9546             :   //   i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
    9547             :   //   i32 y = TRUNCATE(i64 x)
    9548             :   //        -- becomes --
    9549             :   //   v16i8 b = BITCAST (v2i64 val)
    9550             :   //   i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
    9551             :   //
    9552             :   // Note: We only run this optimization after type legalization (which often
    9553             :   // creates this pattern) and before operation legalization after which
    9554             :   // we need to be more careful about the vector instructions that we generate.
    9555       17435 :   if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
    9556      852112 :       LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) {
    9557        7316 :     EVT VecTy = N0.getOperand(0).getValueType();
    9558        7316 :     EVT ExTy = N0.getValueType();
    9559       14632 :     EVT TrTy = N->getValueType(0);
    9560             : 
    9561             :     unsigned NumElem = VecTy.getVectorNumElements();
    9562        7316 :     unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits();
    9563             : 
    9564        7316 :     EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
    9565             :     assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
    9566             : 
    9567        7316 :     SDValue EltNo = N0->getOperand(1);
    9568        7316 :     if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
    9569        7102 :       int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
    9570        7102 :       EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
    9571        7102 :       int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
    9572             : 
    9573             :       SDLoc DL(N);
    9574        7102 :       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
    9575        7102 :                          DAG.getBitcast(NVT, N0.getOperand(0)),
    9576        7102 :                          DAG.getConstant(Index, DL, IndexTy));
    9577             :     }
    9578             :   }
    9579             : 
    9580             :   // trunc (select c, a, b) -> select c, (trunc a), (trunc b)
    9581     1675388 :   if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) {
    9582         124 :     EVT SrcVT = N0.getValueType();
    9583         248 :     if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) &&
    9584         124 :         TLI.isTruncateFree(SrcVT, VT)) {
    9585             :       SDLoc SL(N0);
    9586          62 :       SDValue Cond = N0.getOperand(0);
    9587         124 :       SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
    9588         186 :       SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
    9589         126 :       return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1);
    9590             :     }
    9591             :   }
    9592             : 
    9593             :   // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits()
    9594      837632 :   if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
    9595      840182 :       (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) &&
    9596        1275 :       TLI.isTypeDesirableForOp(ISD::SHL, VT)) {
    9597        1888 :     SDValue Amt = N0.getOperand(1);
    9598         442 :     KnownBits Known;
    9599         944 :     DAG.computeKnownBits(Amt, Known);
    9600             :     unsigned Size = VT.getScalarSizeInBits();
    9601        1888 :     if (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size)) {
    9602             :       SDLoc SL(N);
    9603         502 :       EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
    9604             : 
    9605        1506 :       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
    9606         502 :       if (AmtVT != Amt.getValueType()) {
    9607           7 :         Amt = DAG.getZExtOrTrunc(Amt, SL, AmtVT);
    9608           7 :         AddToWorklist(Amt.getNode());
    9609             :       }
    9610        1004 :       return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt);
    9611             :     }
    9612             :   }
    9613             : 
    9614             :   // Fold a series of buildvector, bitcast, and truncate if possible.
    9615             :   // For example fold
    9616             :   //   (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to
    9617             :   //   (2xi32 (buildvector x, y)).
    9618       13549 :   if (Level == AfterLegalizeVectorOps && VT.isVector() &&
    9619        1181 :       N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
    9620      837161 :       N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
    9621           1 :       N0.getOperand(0).hasOneUse()) {
    9622           2 :     SDValue BuildVect = N0.getOperand(0);
    9623           1 :     EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType();
    9624           1 :     EVT TruncVecEltTy = VT.getVectorElementType();
    9625             : 
    9626             :     // Check that the element types match.
    9627           1 :     if (BuildVectEltTy == TruncVecEltTy) {
    9628             :       // Now we only need to compute the offset of the truncated elements.
    9629             :       unsigned BuildVecNumElts =  BuildVect.getNumOperands();
    9630             :       unsigned TruncVecNumElts = VT.getVectorNumElements();
    9631           1 :       unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
    9632             : 
    9633             :       assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
    9634             :              "Invalid number of elements");
    9635             : 
    9636             :       SmallVector<SDValue, 8> Opnds;
    9637           3 :       for (unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset)
    9638           2 :         Opnds.push_back(BuildVect.getOperand(i));
    9639             : 
    9640           2 :       return DAG.getBuildVector(VT, SDLoc(N), Opnds);
    9641             :     }
    9642             :   }
    9643             : 
    9644             :   // See if we can simplify the input to this truncate through knowledge that
    9645             :   // only the low bits are being used.
    9646             :   // For example "trunc (or (shl x, 8), y)" // -> trunc y
    9647             :   // Currently we only perform this optimization on scalars because vectors
    9648             :   // may have different active low bits.
    9649      837129 :   if (!VT.isVector()) {
    9650             :     APInt Mask =
    9651      826969 :         APInt::getLowBitsSet(N0.getValueSizeInBits(), VT.getSizeInBits());
    9652      826969 :     if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
    9653        3195 :       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter);
    9654             :   }
    9655             : 
    9656             :   // fold (truncate (load x)) -> (smaller load x)
    9657             :   // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
    9658      835539 :   if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
    9659      816111 :     if (SDValue Reduced = ReduceLoadWidth(N))
    9660        2444 :       return Reduced;
    9661             : 
    9662             :     // Handle the case where the load remains an extending load even
    9663             :     // after truncation.
    9664      813667 :     if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
    9665             :       LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    9666       12528 :       if (!LN0->isVolatile() &&
    9667       24941 :           LN0->getMemoryVT().getStoreSizeInBits() < VT.getSizeInBits()) {
    9668         548 :         SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
    9669             :                                          VT, LN0->getChain(), LN0->getBasePtr(),
    9670             :                                          LN0->getMemoryVT(),
    9671         274 :                                          LN0->getMemOperand());
    9672         274 :         DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
    9673         274 :         return NewLoad;
    9674             :       }
    9675             :     }
    9676             :   }
    9677             : 
    9678             :   // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
    9679             :   // where ... are all 'undef'.
    9680     1665642 :   if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
    9681             :     SmallVector<EVT, 8> VTs;
    9682             :     SDValue V;
    9683             :     unsigned Idx = 0;
    9684             :     unsigned NumDefs = 0;
    9685             : 
    9686         398 :     for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
    9687         776 :       SDValue X = N0.getOperand(i);
    9688         388 :       if (!X.isUndef()) {
    9689         362 :         V = X;
    9690             :         Idx = i;
    9691         362 :         NumDefs++;
    9692             :       }
    9693             :       // Stop if more than one members are non-undef.
    9694         388 :       if (NumDefs > 1)
    9695             :         break;
    9696         212 :       VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
    9697             :                                      VT.getVectorElementType(),
    9698         636 :                                      X.getValueType().getVectorNumElements()));
    9699             :     }
    9700             : 
    9701         186 :     if (NumDefs == 0)
    9702           0 :       return DAG.getUNDEF(VT);
    9703             : 
    9704         186 :     if (NumDefs == 1) {
    9705             :       assert(V.getNode() && "The single defined operand is empty!");
    9706             :       SmallVector<SDValue, 8> Opnds;
    9707          46 :       for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
    9708          36 :         if (i != Idx) {
    9709          52 :           Opnds.push_back(DAG.getUNDEF(VTs[i]));
    9710          26 :           continue;
    9711             :         }
    9712          20 :         SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
    9713          10 :         AddToWorklist(NV.getNode());
    9714          10 :         Opnds.push_back(NV);
    9715             :       }
    9716          20 :       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds);
    9717             :     }
    9718             :   }
    9719             : 
    9720             :   // Fold truncate of a bitcast of a vector to an extract of the low vector
    9721             :   // element.
    9722             :   //
    9723             :   // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx
    9724     1682853 :   if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) {
    9725       17014 :     SDValue VecSrc = N0.getOperand(0);
    9726       17014 :     EVT SrcVT = VecSrc.getValueType();
    9727       28694 :     if (SrcVT.isVector() && SrcVT.getScalarType() == VT &&
    9728       11680 :         (!LegalOperations ||
    9729       10767 :          TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, SrcVT))) {
    9730             :       SDLoc SL(N);
    9731             : 
    9732       10483 :       EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
    9733       10486 :       unsigned Idx = isLE ? 0 : SrcVT.getVectorNumElements() - 1;
    9734       10483 :       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT,
    9735       10483 :                          VecSrc, DAG.getConstant(Idx, SL, IdxVT));
    9736             :     }
    9737             :   }
    9738             : 
    9739             :   // Simplify the operands using demanded-bits information.
    9740     1634507 :   if (!VT.isVector() &&
    9741      812179 :       SimplifyDemandedBits(SDValue(N, 0)))
    9742       12314 :     return SDValue(N, 0);
    9743             : 
    9744             :   // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
    9745             :   // (trunc addcarry(X, Y, Carry)) -> (addcarry trunc(X), trunc(Y), Carry)
    9746             :   // When the adde's carry is not used.
    9747      810037 :   if ((N0.getOpcode() == ISD::ADDE || N0.getOpcode() == ISD::ADDCARRY) &&
    9748      810058 :       N0.hasOneUse() && !N0.getNode()->hasAnyUseOfValue(1) &&
    9749          21 :       (!LegalOperations || TLI.isOperationLegal(N0.getOpcode(), VT))) {
    9750             :     SDLoc SL(N);
    9751          60 :     auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
    9752          60 :     auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
    9753          40 :     auto VTs = DAG.getVTList(VT, N0->getValueType(1));
    9754          40 :     return DAG.getNode(N0.getOpcode(), SL, VTs, X, Y, N0.getOperand(2));
    9755             :   }
    9756             : 
    9757             :   // fold (truncate (extract_subvector(ext x))) ->
    9758             :   //      (extract_subvector x)
    9759             :   // TODO: This can be generalized to cover cases where the truncate and extract
    9760             :   // do not fully cancel each other out.
    9761      809994 :   if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
    9762         153 :     SDValue N00 = N0.getOperand(0);
    9763          25 :     if (N00.getOpcode() == ISD::SIGN_EXTEND ||
    9764         170 :         N00.getOpcode() == ISD::ZERO_EXTEND ||
    9765             :         N00.getOpcode() == ISD::ANY_EXTEND) {
    9766         272 :       if (N00.getOperand(0)->getValueType(0).getVectorElementType() ==
    9767             :           VT.getVectorElementType())
    9768         136 :         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
    9769         272 :                            N00.getOperand(0), N0.getOperand(1));
    9770             :     }
    9771             :   }
    9772             : 
    9773      809858 :   if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    9774           4 :     return NewVSel;
    9775             : 
    9776      809854 :   return SDValue();
    9777             : }
    9778             : 
    9779             : static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
    9780       58415 :   SDValue Elt = N->getOperand(i);
    9781      116830 :   if (Elt.getOpcode() != ISD::MERGE_VALUES)
    9782             :     return Elt.getNode();
    9783         513 :   return Elt.getOperand(Elt.getResNo()).getNode();
    9784             : }
    9785             : 
    9786             : /// build_pair (load, load) -> load
    9787             : /// if load locations are consecutive.
    9788       58415 : SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
    9789             :   assert(N->getOpcode() == ISD::BUILD_PAIR);
    9790             : 
    9791             :   LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
    9792             :   LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
    9793             : 
    9794             :   // A BUILD_PAIR is always having the least significant part in elt 0 and the
    9795             :   // most significant part in elt 1. So when combining into one large load, we
    9796             :   // need to consider the endianness.
    9797       58415 :   if (DAG.getDataLayout().isBigEndian())
    9798             :     std::swap(LD1, LD2);
    9799             : 
    9800       61036 :   if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
    9801             :       LD1->getAddressSpace() != LD2->getAddressSpace())
    9802       55794 :     return SDValue();
    9803        5242 :   EVT LD1VT = LD1->getValueType(0);
    9804             :   unsigned LD1Bytes = LD1VT.getStoreSize();
    9805        2621 :   if (ISD::isNON_EXTLoad(LD2) && LD2->hasOneUse() &&
    9806        2621 :       DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) {
    9807        2483 :     unsigned Align = LD1->getAlignment();
    9808        4966 :     unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
    9809        2483 :         VT.getTypeForEVT(*DAG.getContext()));
    9810             : 
    9811        2483 :     if (NewAlign <= Align &&
    9812        2256 :         (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
    9813        2256 :       return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
    9814        4512 :                          LD1->getPointerInfo(), Align);
    9815             :   }
    9816             : 
    9817         365 :   return SDValue();
    9818             : }
    9819             : 
    9820             : static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
    9821             :   // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi
    9822             :   // and Lo parts; on big-endian machines it doesn't.
    9823          10 :   return DAG.getDataLayout().isBigEndian() ? 1 : 0;
    9824             : }
    9825             : 
    9826      679701 : static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
    9827             :                                     const TargetLowering &TLI) {
    9828             :   // If this is not a bitcast to an FP type or if the target doesn't have
    9829             :   // IEEE754-compliant FP logic, we're done.
    9830      679701 :   EVT VT = N->getValueType(0);
    9831      679701 :   if (!VT.isFloatingPoint() || !TLI.hasBitPreservingFPLogic(VT))
    9832      625359 :     return SDValue();
    9833             : 
    9834             :   // TODO: Handle cases where the integer constant is a different scalar
    9835             :   // bitwidth to the FP.
    9836       54342 :   SDValue N0 = N->getOperand(0);
    9837       54342 :   EVT SourceVT = N0.getValueType();
    9838       54342 :   if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits())
    9839       19441 :     return SDValue();
    9840             : 
    9841             :   unsigned FPOpcode;
    9842             :   APInt SignMask;
    9843       34901 :   switch (N0.getOpcode()) {
    9844        2099 :   case ISD::AND:
    9845             :     FPOpcode = ISD::FABS;
    9846        2099 :     SignMask = ~APInt::getSignMask(SourceVT.getScalarSizeInBits());
    9847        2099 :     break;
    9848         689 :   case ISD::XOR:
    9849             :     FPOpcode = ISD::FNEG;
    9850         689 :     SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
    9851         689 :     break;
    9852         971 :   case ISD::OR:
    9853             :     FPOpcode = ISD::FABS;
    9854         971 :     SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
    9855         971 :     break;
    9856       31142 :   default:
    9857       31142 :     return SDValue();
    9858             :   }
    9859             : 
    9860             :   // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X
    9861             :   // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X
    9862             :   // Fold (bitcast int (or (bitcast fp X to int), 0x8000...) to fp) ->
    9863             :   //   fneg (fabs X)
    9864        3759 :   SDValue LogicOp0 = N0.getOperand(0);
    9865        3759 :   ConstantSDNode *LogicOp1 = isConstOrConstSplat(N0.getOperand(1), true);
    9866         650 :   if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask &&
    9867        3759 :       LogicOp0.getOpcode() == ISD::BITCAST &&
    9868          98 :       LogicOp0.getOperand(0).getValueType() == VT) {
    9869          98 :     SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0));
    9870             :     NumFPLogicOpsConv++;
    9871          49 :     if (N0.getOpcode() == ISD::OR)
    9872          20 :       return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp);
    9873          39 :     return FPOp;
    9874             :   }
    9875             : 
    9876        3710 :   return SDValue();
    9877             : }
    9878             : 
    9879      821241 : SDValue DAGCombiner::visitBITCAST(SDNode *N) {
    9880      821241 :   SDValue N0 = N->getOperand(0);
    9881     1642482 :   EVT VT = N->getValueType(0);
    9882             : 
    9883      821241 :   if (N0.isUndef())
    9884         126 :     return DAG.getUNDEF(VT);
    9885             : 
    9886             :   // If the input is a BUILD_VECTOR with all constant elements, fold this now.
    9887             :   // Only do this before legalize types, since we might create an illegal
    9888             :   // scalar type. Even if we knew we wouldn't create an illegal scalar type
    9889             :   // we can only do this before legalize ops, since the target maybe
    9890             :   // depending on the bitcast.
    9891             :   // First check to see if this is all constant.
    9892      151854 :   if (!LegalTypes &&
    9893       90104 :       N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
    9894      910331 :       VT.isVector() && cast<BuildVectorSDNode>(N0)->isConstant())
    9895             :     return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(),
    9896       88248 :                                              VT.getVectorElementType());
    9897             : 
    9898             :   // If the input is a constant, let getNode fold it.
    9899             :   if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {