LLVM  10.0.0svn
Macros | Enumerations | Functions | Variables
X86ISelLowering.cpp File Reference
#include "X86ISelLowering.h"
#include "Utils/X86ShuffleDecode.h"
#include "X86CallingConv.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/EHPersonalities.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>
Include dependency graph for X86ISelLowering.cpp:

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "x86-isel"
 

Enumerations

enum  StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn }
 CallIsStructReturn - Determines whether a call uses struct return semantics. More...
 
enum  ShrinkMode { MULS8, MULU8, MULS16, MULU16 }
 Different mul shrinking modes. More...
 

Functions

 STATISTIC (NumTailCalls, "Number of tail calls")
 
static void errorUnsupported (SelectionDAG &DAG, const SDLoc &dl, const char *Msg)
 Call this when the user attempts to do something unsupported, like returning a double without SSE2 enabled on x86_64. More...
 
static void getMaxByValAlign (Type *Ty, unsigned &MaxAlign)
 Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More...
 
static bool hasStackGuardSlotTLS (const Triple &TargetTriple)
 
static ConstantSegmentOffset (IRBuilder<> &IRB, unsigned Offset, unsigned AddressSpace)
 
static SDValue lowerMasksToReg (const SDValue &ValArg, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG)
 Lowers masks values (v*i1) to the local register values. More...
 
static void Passv64i1ArgInRegs (const SDLoc &Dl, SelectionDAG &DAG, SDValue Chain, SDValue &Arg, SmallVector< std::pair< unsigned, SDValue >, 8 > &RegsToPass, CCValAssign &VA, CCValAssign &NextVA, const X86Subtarget &Subtarget)
 Breaks v64i1 value into two registers and adds the new node to the DAG. More...
 
static SDValue getv64i1Argument (CCValAssign &VA, CCValAssign &NextVA, SDValue &Root, SelectionDAG &DAG, const SDLoc &Dl, const X86Subtarget &Subtarget, SDValue *InFlag=nullptr)
 Reads two 32 bit registers and creates a 64 bit mask value. More...
 
static SDValue lowerRegToMasks (const SDValue &ValArg, const EVT &ValVT, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG)
 The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1) More...
 
static StructReturnType callIsStructReturn (ArrayRef< ISD::OutputArg > Outs, bool IsMCU)
 
static StructReturnType argsAreStructReturn (ArrayRef< ISD::InputArg > Ins, bool IsMCU)
 Determines whether a function uses struct return semantics. More...
 
static SDValue CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl)
 Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More...
 
static bool canGuaranteeTCO (CallingConv::ID CC)
 Return true if the calling convention is one that we can guarantee TCO for. More...
 
static bool mayTailCallThisCC (CallingConv::ID CC)
 Return true if we might ever do TCO for calls with this calling convention. More...
 
static bool shouldGuaranteeTCO (CallingConv::ID CC, bool GuaranteedTailCallOpt)
 Return true if the function is being made into a tailcall target by changing its ABI. More...
 
static ArrayRef< MCPhysReg > get64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget &Subtarget)
 
static ArrayRef< MCPhysReg > get64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget &Subtarget)
 
static bool isSortedByValueNo (ArrayRef< CCValAssign > ArgLocs)
 
static SDValue EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl)
 Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More...
 
static SDValue getMOVL (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More...
 
static bool MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII, const CCValAssign &VA)
 Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More...
 
static bool MayFoldLoad (SDValue Op)
 
static bool MayFoldIntoStore (SDValue Op)
 
static bool MayFoldIntoZeroExtend (SDValue Op)
 
static bool isTargetShuffle (unsigned Opcode)
 
static bool isTargetShuffleVariableMask (unsigned Opcode)
 
static bool isX86CCUnsigned (unsigned X86CC)
 Return true if the condition is an unsigned comparison operation. More...
 
static X86::CondCode TranslateIntegerX86CC (ISD::CondCode SetCCOpcode)
 
static X86::CondCode TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
 Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More...
 
static bool hasFPCMov (unsigned X86CC)
 Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. More...
 
static bool isUndefOrEqual (int Val, int CmpVal)
 Val is the undef sentinel value or equal to the specified value. More...
 
static bool isUndefOrZero (int Val)
 Val is either the undef or zero sentinel value. More...
 
static bool isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value. More...
 
static bool isUndefLowerHalf (ArrayRef< int > Mask)
 Return true if the mask creates a vector whose lower half is undefined. More...
 
static bool isUndefUpperHalf (ArrayRef< int > Mask)
 Return true if the mask creates a vector whose upper half is undefined. More...
 
static bool isInRange (int Val, int Low, int Hi)
 Return true if Val falls within the specified range (L, H]. More...
 
static bool isAnyInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if the value of any element in Mask falls within the specified range (L, H]. More...
 
static bool isUndefOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrZeroOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef, zero or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H]. More...
 
static bool isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
 Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef. More...
 
static bool isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero. More...
 
static bool isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero. More...
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
 Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More...
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, SmallVectorImpl< int > &WidenedMask)
 
static bool canWidenShuffleElements (ArrayRef< int > Mask)
 
static SDValue getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
 
static SDValue getConstVector (ArrayRef< APInt > Bits, APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all zero elements. More...
 
static SDValue extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 128-bits from a vector > 128 bits. More...
 
static SDValue extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 256-bits from a 512-bit vector. More...
 
static SDValue insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to put 128-bits into a vector > 128 bits. More...
 
static SDValue widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. More...
 
static SDValue widenSubVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, unsigned WideSizeInBits)
 Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. More...
 
static bool collectConcatOps (SDNode *N, SmallVectorImpl< SDValue > &Ops)
 
template<typename F >
SDValue SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true)
 
static SDValue insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert i1-subvector to i1-vector. More...
 
static SDValue concatSubVectors (SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all bits set. More...
 
static unsigned getOpcode_EXTEND_VECTOR_INREG (unsigned Opcode)
 
static SDValue getExtendInVec (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG)
 
static SDValue IsNOT (SDValue V, SelectionDAG &DAG)
 
static SDValue getUnpackl (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackl operation. More...
 
static SDValue getUnpackh (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackh operation. More...
 
static SDValue getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return a vector_shuffle of the specified vector of zero or undef vector. More...
 
static const ConstantgetTargetConstantFromNode (LoadSDNode *Load)
 
static const ConstantgetTargetConstantFromNode (SDValue Op)
 
static bool getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=true)
 
static bool isConstantSplat (SDValue Op, APInt &SplatVal)
 
static bool getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts)
 
static void createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary)
 Create a shuffle mask that matches the PACKSS/PACKUS truncation. More...
 
static void getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
 
static void getHorizDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
 
static bool getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
 Calculates the shuffle mask corresponding to the target-specific opcode. More...
 
static bool setTargetShuffleZeroElements (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops)
 Check a target shuffle mask's inputs to see if we can set any values to SM_SentinelZero - this is for elements that are known to be zero (not just zeroable) from their inputs. More...
 
static bool resolveTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, SelectionDAG &DAG, unsigned Depth)
 
static bool getFauxShuffleMask (SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG, unsigned Depth)
 
static void resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask)
 Removes unused/repeated shuffle source inputs and adjusts the shuffle mask. More...
 
static bool resolveTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, SelectionDAG &DAG, unsigned Depth)
 Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values. More...
 
static SDValue getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth)
 Returns the scalar element that will make up the ith element of the result of the vector shuffle. More...
 
static SDValue LowerBuildVectorAsInsert (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v16i8. More...
 
static SDValue LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v8i16. More...
 
static SDValue LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v4i32 or v4f32. More...
 
static SDValue getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
 Return a vector logical shift node. More...
 
static SDValue LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
 
static bool findEltLoadSrc (SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset)
 
static SDValue EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool isAfterLegalize)
 Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More...
 
static SDValue combineToConsecutiveLoads (EVT VT, SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool isAfterLegalize)
 
static ConstantgetConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C)
 
static bool isFoldableUseOfShuffle (SDNode *N)
 
static SDValue isSplatZeroExtended (const BuildVectorSDNode *Op, unsigned &NumElt, MVT &EltType)
 
static SDValue lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a. More...
 
static int getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
 For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More...
 
static SDValue buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG)
 
static SDValue ConvertI1VectorToInteger (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBUILD_VECTORvXi1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool isHorizontalBinOpPart (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
 This is a helper function of LowerToHorizontalOp(). More...
 
static SDValue ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
 Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More...
 
static bool isAddSubOrSubAdd (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd)
 Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation. More...
 
static bool isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses)
 Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1). More...
 
static SDValue lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node. More...
 
static bool isHopBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1)
 
static SDValue getHopForBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1)
 
static SDValue LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More...
 
static SDValue lowerBuildVectorToBitOp (BuildVectorSDNode *Op, SelectionDAG &DAG)
 If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors. More...
 
static SDValue materializeVectorConstant (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Create a vector constant without a load. More...
 
static SDValue createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices. More...
 
static SDValue LowerBUILD_VECTORAsVariablePermute (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isNoopShuffleMask (ArrayRef< int > Mask)
 Tiny helper function to identify a no-op mask. More...
 
static bool is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
 Test whether there are elements crossing 128-bit lanes in this shuffle mask. More...
 
static bool isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each sub-lane. More...
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 128-bit lane. More...
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask)
 
static bool is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 256-bit lane. More...
 
static bool isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a target shuffle mask is equivalent within each sub-lane. More...
 
static bool isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask)
 Checks whether a shuffle mask is equivalent to an explicit list of arguments. More...
 
static bool isTargetShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue())
 Checks whether a target shuffle mask is equivalent to an explicit pattern. More...
 
static SmallVector< int, 64 > createTargetShuffleMask (ArrayRef< int > Mask, const APInt &Zeroable)
 
static bool createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond)
 
static bool isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT)
 
static bool is128BitUnpackShuffleMask (ArrayRef< int > Mask)
 
static bool hasIdenticalHalvesShuffleMask (ArrayRef< int > Mask)
 Return true if a shuffle mask chooses elements identically in its top and bottom halves. More...
 
static unsigned getV4X86ShuffleImm (ArrayRef< int > Mask)
 Get a 4-lane 8-bit shuffle immediate for a mask. More...
 
static SDValue getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
 
static APInt computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2)
 Compute whether each element of a shuffle is zeroable. More...
 
static bool isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
 
static SDValue lowerShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle with a single PSHUFB of V1 or V2. More...
 
static SDValue getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics. More...
 
static SDValue lowerShuffleToEXPAND (const SDLoc &DL, MVT VT, const APInt &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchVectorShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static bool matchVectorShuffleAsVPMOV (ArrayRef< int > Mask, bool SwappedOps, int Delta)
 
static SDValue lowerShuffleWithVPMOV (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchVectorShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleWithPACK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a bitmask instruction for a shuffle. More...
 
static SDValue lowerShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle using bit math. More...
 
static SDValue getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics. More...
 
static bool matchVectorShuffleAsBlend (SDValue V1, SDValue V2, MutableArrayRef< int > TargetMask, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask)
 
static uint64_t scaleVectorShuffleBlendMask (uint64_t BlendMask, int Size, int Scale)
 
static SDValue lowerShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle. More...
 
static SDValue lowerShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false)
 Try to lower as a blend of elements from two inputs followed by a single-input permutation. More...
 
static SDValue lowerShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower as an unpack of elements from two inputs followed by a single-input permutation. More...
 
static SDValue lowerShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place. More...
 
static SDValue lowerShuffleAsDecomposedShuffleBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic routine to decompose a shuffle and blend into independent blends and permutes. More...
 
static int matchShuffleAsRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to lower a vector shuffle as a rotation. More...
 
static int matchShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to lower a vector shuffle as a byte rotation. More...
 
static SDValue lowerShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a dword/qword rotation. More...
 
static SDValue lowerVectorShuffleAsByteShiftMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a byte shift sequence. More...
 
static int matchShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget)
 Try to lower a vector shuffle as a bit shift (shifts in zeros). More...
 
static SDValue lowerShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool matchShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable)
 
static bool matchShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx)
 
static SDValue lowerShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More...
 
static SDValue lowerShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle as a zero or any extension. More...
 
static SDValue lowerShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a zero extension on any microarch. More...
 
static SDValue getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
 Try to get a scalar value for a specific element of a vector. More...
 
static bool isShuffleFoldableLoad (SDValue V)
 Helper to test for a load that can be folded with x86 shuffles. More...
 
static SDValue lowerShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower insertion of a single element into a zero vector. More...
 
static SDValue lowerShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements. More...
 
static bool isSingleSHUFPSMask (ArrayRef< int > Mask)
 Test whether this can be lowered with a single SHUFPS instruction. More...
 
static SDValue lowerShuffleOfExtractsAsVperm (const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG)
 If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering. More...
 
static SDValue lowerShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single element. More...
 
static bool matchShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More...
 
static SDValue lowerV2F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV2I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit integer shuffles. More...
 
static SDValue lowerShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Lower a vector shuffle using the SHUFPS instruction. More...
 
static SDValue lowerV4F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV4I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane i32 vector shuffles. More...
 
static SDValue lowerV8I16GeneralSingleInputShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More...
 
static SDValue lowerShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
 Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used. More...
 
static SDValue lowerV8I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of 8-lane i16 shuffles. More...
 
static int canLowerByDroppingEvenElements (ArrayRef< int > Mask, bool IsSingleInput)
 Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More...
 
static SDValue lowerShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerV16I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of v16i8 shuffles. More...
 
static SDValue lower128BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Dispatching routine to lower various 128-bit x86 vector shuffles. More...
 
static SDValue splitAndLowerShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Generic routine to split vector shuffle into half-sized shuffles. More...
 
static SDValue lowerShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Either split a vector in halves or decompose the shuffles and the blend. More...
 
static SDValue lowerShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation. More...
 
static SDValue lowerShuffleAsLanePermuteAndBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes. More...
 
static SDValue lowerV2X128Shuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering 2-lane 128-bit shuffles. More...
 
static SDValue lowerShuffleAsLanePermuteAndRepeatedMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More...
 
static bool getHalfShuffleMask (ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2)
 If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true. More...
 
static SDValue getShuffleHalfVectors (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG)
 Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width. More...
 
static SDValue lowerShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF. More...
 
static bool isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
 Test whether the specified input (0 or 1) is in-place blended by the given mask. More...
 
static SDValue lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes. More...
 
static bool matchShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, unsigned &ShuffleImm, ArrayRef< int > Mask)
 
static SDValue lowerShuffleWithSHUFPD (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerV4F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV4I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit integer shuffles. More...
 
static SDValue lowerV8F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit integer shuffles. More...
 
static SDValue lowerV16I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 16-bit integer shuffles. More...
 
static SDValue lowerV32I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 8-bit integer shuffles. More...
 
static SDValue lower256BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 256-bit x86 vector shuffles. More...
 
static SDValue lowerV4X128Shuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a 128-bit shuffles. More...
 
static SDValue lowerV8F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV16F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit integer shuffles. More...
 
static SDValue lowerV16I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit integer shuffles. More...
 
static SDValue lowerV32I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 16-bit integer shuffles. More...
 
static SDValue lowerV64I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 64-lane 8-bit integer shuffles. More...
 
static SDValue lower512BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 512-bit x86 vector shuffles. More...
 
static SDValue lower1BitShuffleAsKSHIFTR (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static int match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
 
static SDValue lower1BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask)
 Helper function that returns true if the shuffle mask should be commuted to improve canonicalization. More...
 
static SDValue lowerVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Top-level lowering for x86 vector shuffles. More...
 
static SDValue lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a VSELECT instruction to a vector shuffle. More...
 
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
 
static SDValue ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Extract one bit from mask vector, like v16i1 or v8i1. More...
 
static SDValue InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert one bit to mask vector, like v16i1 or v8i1. More...
 
static SDValue LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)
 
static SDValue LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit)
 
static SDValue LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
 
static SDValue LowerShiftParts (SDValue Op, SelectionDAG &DAG)
 Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More...
 
static SDValue LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerI64IntToFP_AVX512DQ (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool useVectorCast (unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget)
 
static SDValue vectorizeExtractedCast (SDValue Cast, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction. More...
 
static SDValue LowerUINT_TO_FP_i64 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 64-bit unsigned integer to double expansion. More...
 
static SDValue LowerUINT_TO_FP_i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 32-bit unsigned integer to float expansion. More...
 
static SDValue lowerUINT_TO_FP_v2i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 
static SDValue lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerUINT_TO_FP_vec (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper to recursively truncate vector elements in half with PACKSS/PACKUS. More...
 
static SDValue LowerTruncateVecI1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG)
 
static bool shouldUseHorizontalOp (bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Horizontal vector math instructions may be slower than normal math with shuffles. More...
 
static SDValue lowerAddSubToHorizontalOp (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation. More...
 
static SDValue lowerFaddFsub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation. More...
 
static SDValue LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
 The only differences between FABS and FNEG are the mask and the logic op. More...
 
static SDValue LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG)
 Helper for creating a X86ISD::SETCC node. More...
 
static bool matchBitOpReduction (SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps)
 Helper for matching OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns. More...
 
static SDValue LowerVectorAllZeroTest (SDValue Op, ISD::CondCode CC, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &X86CC)
 
static bool hasNonFlagsUse (SDValue Op)
 return true if Op has a use that doesn't just read flags. More...
 
static SDValue EmitTest (SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Emit nodes that will be selected as "test Op0,Op0", or something equivalent. More...
 
static SDValue LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, SDValue &X86CC)
 Result of 'and' is compared against zero. More...
 
static unsigned translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1)
 Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs. More...
 
static SDValue Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG)
 Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back. More...
 
static SDValue LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG)
 
static SDValue incDecVectorConstant (SDValue V, SelectionDAG &DAG, bool IsInc)
 Given a buildvector constant, return a new vector constant with each element incremented or decremented. More...
 
static SDValue LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 As another special case, use PSUBUS[BW] when it's profitable. More...
 
static SDValue LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue EmitKORTEST (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC)
 
static std::pair< SDValue, SDValuegetX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerXALUO (SDValue Op, SelectionDAG &DAG)
 
static bool isX86LogicalCmp (SDValue Op)
 Return true if opcode is a X86 logical comparison. More...
 
static bool isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue splitVectorStore (StoreSDNode *Store, SelectionDAG &DAG)
 Change a vector store into a pair of half-size vector stores. More...
 
static SDValue scalarizeVectorStore (StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
 Scalarize a vector store, bitcasting to TargetVT to determine the scalar type. More...
 
static SDValue LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
 Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR. More...
 
static bool isXor1OfSetCC (SDValue Op)
 Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use. More...
 
static SDValue LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static unsigned getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable)
 
static SDValue getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount is a constant. More...
 
static SDValue getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount may or may not be a constant. More...
 
static SDValue getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Creates an SDNode for a predicated scalar operation. More...
 
static int getSEHRegistrationNodeSize (const Function *Fn)
 
static SDValue recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
 When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More...
 
static SDValue getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getGatherNode (SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue expandIntrinsicWChainHelper (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX. More...
 
static void getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp). More...
 
static SDValue LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG)
 
static SDValue MarkEHGuard (SDValue Op, SelectionDAG &DAG)
 
static SDValue EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Truncating Store with signed or unsigned saturation. More...
 
static SDValue EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Masked Truncating Store with signed or unsigned saturation. More...
 
static SDValue LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorIntUnary (SDValue Op, SelectionDAG &DAG)
 
static SDValue Lower256IntUnary (SDValue Op, SelectionDAG &DAG)
 
static SDValue Lower512IntUnary (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector CTLZ using native supported vector CTLZ instruction. More...
 
static SDValue LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue split256IntArith (SDValue Op, SelectionDAG &DAG)
 Break a 256-bit integer operation into two new 128-bit ones and then concatenate the result back. More...
 
static SDValue split512IntArith (SDValue Op, SelectionDAG &DAG)
 Break a 512-bit integer operation into two new 256-bit ones and then concatenate the result back. More...
 
static SDValue lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMINMAX (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool SupportedVectorShiftWithImm (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool SupportedVectorVarShift (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static SDValue LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue emitLockedStackOp (SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, SDLoc DL)
 Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix. More...
 
static SDValue LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue getPMOVMSKB (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Compute the horizontal sum of bytes in V for the elements of VT. More...
 
static SDValue LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower atomic_load_ops into LOCK-prefixed operations. More...
 
static SDValue LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerADDSUBCARRY (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
 Widen a vector input to a vector of NVT. More...
 
static SDValue LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static MachineBasicBlockemitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
 Utility function to emit xbegin specifying the start of an RTM region. More...
 
static bool checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
 
static bool isCMOVPseudo (MachineInstr &MI)
 
static MachineInstrBuilder createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
 
static unsigned getOpcodeForRetpoline (unsigned RPOpc)
 
static const chargetRetpolineSymbol (const X86Subtarget &Subtarget, unsigned Reg)
 
static bool matchUnaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
 
static bool matchUnaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static bool matchBinaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary)
 
static bool matchBinaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static SDValue combineX86ShuffleChainWithExtract (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Combine an arbitrary chain of shuffles into a single instruction if possible. More...
 
static SDValue combineX86ShufflesConstants (ArrayRef< SDValue > Ops, ArrayRef< int > Mask, SDValue Root, bool HasVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode *> SrcNodes, unsigned Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Fully generic combining of x86 shuffle instructions. More...
 
static SDValue combineX86ShufflesRecursively (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper entry wrapper to combineX86ShufflesRecursively. More...
 
static SmallVector< int, 4 > getPSHUFShuffleMask (SDValue N)
 Get the PSHUF-style mask from PSHUF node. More...
 
static SDValue combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG)
 Search for a combinable shuffle across a chain ending in pshufd. More...
 
static SDValue combineTargetShuffle (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Try to combine x86 target specific shuffles. More...
 
static bool isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even)
 Checks if the shuffle mask takes subsequent elements alternately from two vectors. More...
 
static bool isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd)
 Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation. More...
 
static SDValue combineShuffleToFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Combine shuffle of two fma nodes into FMAddSub or FMSubAdd. More...
 
static SDValue combineShuffleToAddSubOrFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to combine a shuffle into a target-specific add-sub or mul-add-sub node. More...
 
static SDValue combineShuffleOfConcatUndef (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue foldShuffleOfHorizOp (SDNode *N)
 Eliminate a redundant shuffle of a horizontal math op. More...
 
static SDValue narrowShuffle (ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
 If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size. More...
 
static SDValue combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load. More...
 
static bool checkBitcastSrcVectorSize (SDValue Src, unsigned Size)
 
static SDValue combineBitcastvxi1 (SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG)
 
static SDValue combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue createMMXBuildVector (BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool detectZextAbsDiff (const SDValue &Abs, SDValue &Op0, SDValue &Op1)
 
static SDValue createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue combineHorizontalMinMaxResult (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineHorizontalPredicateResult (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue scalarizeExtEltFP (SDNode *ExtElt, SelectionDAG &DAG)
 Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op. More...
 
static SDValue combineReductionToHorizontal (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops. More...
 
static SDValue combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More...
 
static SDValue combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation. More...
 
static SDValue narrowVectorSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) –> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1) More...
 
static SDValue combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element. More...
 
static SDValue combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on SELECT and VSELECT nodes. More...
 
static SDValue combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction. More...
 
static SDValue checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
 
static bool checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
 Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More...
 
static SDValue combineCarryThroughADD (SDValue EFLAGS, SelectionDAG &DAG)
 
static SDValue combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values. More...
 
static SDValue combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More...
 
static bool canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
 
static SDValue reduceVMULWidth (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code. More...
 
static SDValue combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL)
 
static SDValue combineMulToPMADDWD (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMulToPMULDQ (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g. More...
 
static SDValue combineShiftLeft (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends. More...
 
static SDValue combineANDXORWithAllOnesIntoANDNP (SDNode *N, SelectionDAG &DAG)
 Try to fold: (and (xor X, -1), Y) -> (andnp X, Y). More...
 
static SDValue PromoteMaskArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers. More...
 
static SDValue combineAndMaskToShift (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If this is a zero/all-bits result that is bitwise-anded with a low bits mask. More...
 
static SDValue getIndexFromUnindexedLoad (LoadSDNode *Ld)
 
static bool hasBZHI (const X86Subtarget &Subtarget, MVT VT)
 
static SDValue combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineParity (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue canonicalizeBitSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask)
 
static SDValue combineLogicBlendIntoConditionalNegate (EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerX86CmpEqZeroToCtlzSrl (SDValue Op, EVT ExtTy, SelectionDAG &DAG)
 
static SDValue combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG)
 Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1) More...
 
static SDValue foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1. More...
 
static bool isSATValidOnAVX512Subtarget (EVT SrcVT, EVT DstVT, const X86Subtarget &Subtarget)
 Check if truncation with saturation form type SrcVT to DstVT is valid for the given Subtarget. More...
 
static SDValue detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
 Detect patterns of truncation with unsigned saturation: More...
 
static SDValue detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false)
 Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type). More...
 
static SDValue combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue detectAVGPattern (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction. More...
 
static SDValue combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static int getOneTrueElt (SDValue V)
 If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element. More...
 
static bool getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, unsigned &Alignment)
 Given a masked memory load/store operation, return true if it has one mask bit set. More...
 
static SDValue reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert. More...
 
static SDValue combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG)
 If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store. More...
 
static SDValue combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isHorizontalBinOp (SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative)
 Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More...
 
static SDValue combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating-point adds/subs. More...
 
static SDValue combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen. More...
 
static SDValue combineVectorTruncationWithPACKUS (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Truncate using ISD::AND mask and X86ISD::PACKUS. More...
 
static SDValue combineVectorTruncationWithPACKSS (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS. More...
 
static SDValue combineVectorTruncation (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations. More...
 
static SDValue combineVectorSignBitsTruncation (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 This function transforms vector truncation of 'extended sign-bits' or 'extended zero-bits' values. More...
 
static SDValue combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 
static SDValue combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue isFNEG (SelectionDAG &DAG, SDNode *N)
 Returns the negated value if the node N flips sign of FP value. More...
 
static SDValue combineFneg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating point negations. More...
 
static SDValue lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue foldXor1SetCC (SDNode *N, SelectionDAG &DAG)
 Fold a xor(setcc cond, val), 1 –> setcc (inverted(cond), val) More...
 
static SDValue combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineBEXTR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isNullFPScalarOrVectorConst (SDValue V)
 
static SDValue getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value. More...
 
static SDValue combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FAND nodes. More...
 
static SDValue combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FANDN nodes. More...
 
static SDValue combineFOr (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More...
 
static SDValue combineFMinFMax (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More...
 
static SDValue combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineCVTP2I_CVTTP2I (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::ANDNP nodes. More...
 
static SDValue combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSextInRegCmov (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode. More...
 
static SDValue combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG)
 
static SDValue combineToExtendBoolVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineToExtendVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements. More...
 
static SDValue combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static unsigned negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc)
 
static SDValue combineFMA (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFMADDSUB (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorSizedSetCCEquality (SDNode *SetCC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks. More...
 
static SDValue combineSetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize branch condition evaluation. More...
 
static SDValue combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineToFPTruncExtElt (SDNode *N, SelectionDAG &DAG)
 If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast. More...
 
static SDValue combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSIntToFP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool needCarryOrOverflowFlag (SDValue Flags)
 
static bool onlyZeroFlagUsed (SDValue Flags)
 
static SDValue combineCMP (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineX86AddSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSBB (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineAddOrSubToADCOrSBB (SDNode *N, SelectionDAG &DAG)
 If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB. More...
 
static SDValue combineLoopMAddPattern (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineLoopSADPattern (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineIncDecVector (SDNode *N, SelectionDAG &DAG)
 Convert vector increment or decrement to sub/add with an all-ones constant: add X, <1, 1...> –> sub X, <-1, -1...> sub X, <1, 1...> –> add X, <-1, -1...> The all-ones vector constant can be materialized using a pcmpeq instruction that is commonly recognized as an idiom (has no register dependency), so that's better/smaller than loading a splat 1 constant. More...
 
static SDValue matchPMADDWD (SelectionDAG &DAG, SDValue Op0, SDValue Op1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
 
static SDValue matchPMADDWD_2 (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
 
static SDValue combineAdd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSubToSubus (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineConcatVectorOps (const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g. More...
 
static SDValue combineConcatVectors (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineInsertSubvector (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue narrowExtractedVectorSelect (SDNode *Ext, SelectionDAG &DAG)
 If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width. More...
 
static SDValue combineExtractSubvector (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineScalarToVector (SDNode *N, SelectionDAG &DAG)
 
static SDValue combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineExtInVec (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool matchAsm (StringRef S, ArrayRef< const char *> Pieces)
 
static bool clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)
 
static X86::CondCode parseConstraintCode (llvm::StringRef Constraint)
 
static bool isGRClass (const TargetRegisterClass &RC)
 Check if RC is a general purpose register class. More...
 
static bool isFRClass (const TargetRegisterClass &RC)
 Check if RC is a vector register class. More...
 
static bool isVKClass (const TargetRegisterClass &RC)
 Check if RC is a mask register class. More...
 

Variables

cl::opt< boolExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(true), cl::desc("Enable an experimental vector type legalization through widening " "rather than promotion."), cl::Hidden)
 
static cl::opt< int > ExperimentalPrefLoopAlignment ("x86-experimental-pref-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments " "(the last x86-experimental-pref-loop-alignment bits" " of the loop header PC will be 0)."), cl::Hidden)
 
static cl::opt< boolMulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden)
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "x86-isel"

Definition at line 64 of file X86ISelLowering.cpp.

Enumeration Type Documentation

◆ ShrinkMode

enum ShrinkMode

Different mul shrinking modes.

Enumerator
MULS8 
MULU8 
MULS16 
MULU16 

Definition at line 37601 of file X86ISelLowering.cpp.

◆ StructReturnType

CallIsStructReturn - Determines whether a call uses struct return semantics.

Enumerator
NotStructReturn 
RegStructReturn 
StackStructReturn 

Definition at line 2886 of file X86ISelLowering.cpp.

Function Documentation

◆ argsAreStructReturn()

static StructReturnType argsAreStructReturn ( ArrayRef< ISD::InputArg Ins,
bool  IsMCU 
)
static

Determines whether a function uses struct return semantics.

Definition at line 2906 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.

Referenced by isSortedByValueNo().

◆ buildFromShuffleMostly()

static SDValue buildFromShuffleMostly ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ callIsStructReturn()

static StructReturnType callIsStructReturn ( ArrayRef< ISD::OutputArg Outs,
bool  IsMCU 
)
static

◆ canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC)
static

Return true if the calling convention is one that we can guarantee TCO for.

Definition at line 2933 of file X86ISelLowering.cpp.

References llvm::CallingConv::Fast, llvm::CallingConv::GHC, llvm::CallingConv::HHVM, llvm::CallingConv::HiPE, and llvm::CallingConv::X86_RegCall.

Referenced by getMOVL(), isSortedByValueNo(), MatchingStackOffset(), mayTailCallThisCC(), and shouldGuaranteeTCO().

◆ canLowerByDroppingEvenElements()

static int canLowerByDroppingEvenElements ( ArrayRef< int >  Mask,
bool  IsSingleInput 
)
static

Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns
N above, or the number of times even elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 13994 of file X86ISelLowering.cpp.

References llvm::array_lengthof(), assert(), llvm::isPowerOf2_32(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8Shuffle().

◆ canonicalizeBitSelect()

static SDValue canonicalizeBitSelect ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ canonicalizeShuffleMaskWithCommute()

static bool canonicalizeShuffleMaskWithCommute ( ArrayRef< int >  Mask)
static

Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.

Definition at line 16783 of file X86ISelLowering.cpp.

References assert(), Size, and llvm::ArrayRef< T >::size().

Referenced by combineX86ShufflesRecursively(), and lowerVectorShuffle().

◆ canReduceVMulWidth()

static bool canReduceVMulWidth ( SDNode N,
SelectionDAG DAG,
ShrinkMode Mode 
)
static

◆ canWidenShuffleElements() [1/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
SmallVectorImpl< int > &  WidenedMask 
)
static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 5250 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), Size, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canWidenShuffleElements(), combineExtractWithShuffle(), combineX86ShufflesRecursively(), LowerShift(), lowerShuffleAsBlendAndPermute(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), and lowerVectorShuffle().

◆ canWidenShuffleElements() [2/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
const APInt Zeroable,
SmallVectorImpl< int > &  WidenedMask 
)
static

◆ canWidenShuffleElements() [3/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask)
static

Definition at line 5313 of file X86ISelLowering.cpp.

References canWidenShuffleElements().

◆ checkAndUpdateEFLAGSKill()

static bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator  SelectItr,
MachineBasicBlock BB,
const TargetRegisterInfo TRI 
)
static

◆ checkBitcastSrcVectorSize()

static bool checkBitcastSrcVectorSize ( SDValue  Src,
unsigned  Size 
)
static

◆ checkBoolTestAndOrSetCCCombine()

static bool checkBoolTestAndOrSetCCCombine ( SDValue  Cond,
X86::CondCode CC0,
X86::CondCode CC1,
SDValue Flags,
bool isAnd 
)
static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 37276 of file X86ISelLowering.cpp.

◆ checkBoolTestSetCCCombine()

static SDValue checkBoolTestSetCCCombine ( SDValue  Cmp,
X86::CondCode CC 
)
static

◆ clobbersFlagRegisters()

static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > &  AsmPieces)
static

◆ collectConcatOps()

static bool collectConcatOps ( SDNode N,
SmallVectorImpl< SDValue > &  Ops 
)
static

◆ combineADC()

static SDValue combineADC ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineAdd()

static SDValue combineAdd ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineAddOrSubToADCOrSBB()

static SDValue combineAddOrSubToADCOrSBB ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineAnd()

static SDValue combineAnd ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 38962 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, combineAndLoadToBZHI(), combineAndMaskToShift(), combineANDXORWithAllOnesIntoANDNP(), combineBitcastvxi1(), combineCompareEqual(), combineParity(), combineX86ShufflesRecursively(), convertIntLogicToFPLogic(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::APInt::isAllOnesValue(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNullValue(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), matchBitOpReduction(), llvm::ISD::SETEQ, llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineAndLoadToBZHI()

static SDValue combineAndLoadToBZHI ( SDNode Node,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Definition at line 38817 of file X86ISelLowering.cpp.

Referenced by combineAnd().

◆ combineAndMaskToShift()

static SDValue combineAndMaskToShift ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineAndnp()

static SDValue combineAndnp ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineANDXORWithAllOnesIntoANDNP()

static SDValue combineANDXORWithAllOnesIntoANDNP ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineBasicSADPattern()

static SDValue combineBasicSADPattern ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBEXTR()

static SDValue combineBEXTR ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineBitcast()

static SDValue combineBitcast ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 35346 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, C, combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countLeadingZeros(), createMMXBuildVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_TO_SINT, llvm::X86ISD::FXOR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasDQI(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::APFloatBase::IEEEdouble(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::EVT::isScalarInteger(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETLT, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v2f32, llvm::MVT::v2i1, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i1, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8i1, llvm::MVT::v8i8, llvm::MVT::x86mmx, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineBitcastvxi1()

static SDValue combineBitcastvxi1 ( SelectionDAG DAG,
EVT  VT,
SDValue  Src,
const SDLoc DL,
const X86Subtarget Subtarget 
)
static

◆ combineBrCond()

static SDValue combineBrCond ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Optimize branch condition evaluation.

Definition at line 42940 of file X86ISelLowering.cpp.

References combineSetCCEFLAGS(), llvm::SDNode::getConstantOperandVal(), and llvm::SDNode::getOperand().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineBT()

static SDValue combineBT ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineCarryThroughADD()

static SDValue combineCarryThroughADD ( SDValue  EFLAGS,
SelectionDAG DAG 
)
static

◆ combineCastedMaskArithmetic()

static SDValue combineCastedMaskArithmetic ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCMov()

static SDValue combineCMov ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].

Definition at line 37381 of file X86ISelLowering.cpp.

References llvm::SDNode::getConstantOperandVal(), and llvm::SDNode::getOperand().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineCMP()

static SDValue combineCMP ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineCompareEqual()

static SDValue combineCompareEqual ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineConcatVectorOps()

static SDValue combineConcatVectorOps ( const SDLoc DL,
MVT  VT,
ArrayRef< SDValue Ops,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.

ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.

Definition at line 44126 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::TargetLoweringBase::allowsMemoryAccess(), assert(), llvm::ISD::CONCAT_VECTORS, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::CallingConv::Fast, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), LLVM_FALLTHROUGH, MayFoldLoad(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ArrayRef< T >::size(), llvm::X86ISD::SUBV_BROADCAST, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, llvm::MVT::v8i32, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VPERMILPI.

Referenced by combineConcatVectors(), and combineInsertSubvector().

◆ combineConcatVectors()

static SDValue combineConcatVectors ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCVTP2I_CVTTP2I()

static SDValue combineCVTP2I_CVTTP2I ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineExtInVec()

static SDValue combineExtInVec ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineExtractSubvector()

static SDValue combineExtractSubvector ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 44417 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, Concat, llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), getOpcode_EXTEND_VECTOR_INREG(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::EVT::isVector(), narrowExtractedVectorSelect(), llvm::SDNode::ops(), llvm::peekThroughBitcasts(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::SINT_TO_FP, split256IntArith(), llvm::ISD::UINT_TO_FP, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VFPEXT, llvm::ISD::VSELECT, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineExtractVectorElt()

static SDValue combineExtractVectorElt ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 36190 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combineBasicSADPattern(), combineBitcastvxi1(), combineExtractWithShuffle(), combineHorizontalMinMaxResult(), combineHorizontalPredicateResult(), combineReductionToHorizontal(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::X86ISD::MMX_MOVD2W, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T >::push_back(), scalarizeExtEltFP(), llvm::ISD::SETEQ, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase::size(), llvm::ISD::TRUNCATE, llvm::SDNode::uses(), llvm::MVT::v1i64, llvm::MVT::v2i32, llvm::MVT::x86mmx, and XFormVExtractWithShuffleIntoLoad().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineExtractWithShuffle()

static SDValue combineExtractWithShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineExtSetcc()

static SDValue combineExtSetcc ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFaddFsub()

static SDValue combineFaddFsub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFAnd()

static SDValue combineFAnd ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 41677 of file X86ISelLowering.cpp.

References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), llvm::SDNode::getOperand(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFAndFNotToFAndn()

static SDValue combineFAndFNotToFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFAndn()

static SDValue combineFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 41694 of file X86ISelLowering.cpp.

References getNullFPConstForNullVal(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFMA()

static SDValue combineFMA ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFMADDSUB()

static SDValue combineFMADDSUB ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFMinFMax()

static SDValue combineFMinFMax ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineFMinNumFMaxNum()

static SDValue combineFMinNumFMaxNum ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFneg()

static SDValue combineFneg ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFOr()

static SDValue combineFOr ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.

Definition at line 41708 of file X86ISelLowering.cpp.

References assert(), combineFneg(), llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineGatherScatter()

static SDValue combineGatherScatter ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineHorizontalMinMaxResult()

static SDValue combineHorizontalMinMaxResult ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineHorizontalPredicateResult()

static SDValue combineHorizontalPredicateResult ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineIncDecVector()

static SDValue combineIncDecVector ( SDNode N,
SelectionDAG DAG 
)
static

Convert vector increment or decrement to sub/add with an all-ones constant: add X, <1, 1...> –> sub X, <-1, -1...> sub X, <1, 1...> –> add X, <-1, -1...> The all-ones vector constant can be materialized using a pcmpeq instruction that is commonly recognized as an idiom (has no register dependency), so that's better/smaller than loading a splat 1 constant.

Definition at line 43688 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), isConstantSplat(), llvm::APInt::isOneValue(), and llvm::ISD::SUB.

Referenced by combineAdd(), and combineSub().

◆ combineInsertSubvector()

static SDValue combineInsertSubvector ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineLoad()

static SDValue combineLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineLogicBlendIntoConditionalNegate()

static SDValue combineLogicBlendIntoConditionalNegate ( EVT  VT,
SDValue  Mask,
SDValue  X,
SDValue  Y,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineLogicBlendIntoPBLENDV()

static SDValue combineLogicBlendIntoPBLENDV ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineLoopMAddPattern()

static SDValue combineLoopMAddPattern ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineLoopSADPattern()

static SDValue combineLoopSADPattern ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMaskedLoad()

static SDValue combineMaskedLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMaskedLoadConstantMask()

static SDValue combineMaskedLoadConstantMask ( MaskedLoadSDNode ML,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineMaskedStore()

static SDValue combineMaskedStore ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMOVMSK()

static SDValue combineMOVMSK ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMul()

static SDValue combineMul ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMulSpecial()

static SDValue combineMulSpecial ( uint64_t  MulAmt,
SDNode N,
SelectionDAG DAG,
EVT  VT,
const SDLoc DL 
)
static

◆ combineMulToPMADDWD()

static SDValue combineMulToPMADDWD ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMulToPMULDQ()

static SDValue combineMulToPMULDQ ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineOr()

static SDValue combineOr ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineOrCmpEqZeroToCtlzSrl()

static SDValue combineOrCmpEqZeroToCtlzSrl ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineParity()

static SDValue combineParity ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combinePMULDQ()

static SDValue combinePMULDQ ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combinePMULH()

static SDValue combinePMULH ( SDValue  Src,
EVT  VT,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineReductionToHorizontal()

static SDValue combineReductionToHorizontal ( SDNode ExtElt,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineRedundantDWordShuffle()

static SDValue combineRedundantDWordShuffle ( SDValue  N,
MutableArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

◆ combineSBB()

static SDValue combineSBB ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineScalarToVector()

static SDValue combineScalarToVector ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineSelect()

static SDValue combineSelect ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 36608 of file X86ISelLowering.cpp.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineSelectOfTwoConstants()

static SDValue combineSelectOfTwoConstants ( SDNode N,
SelectionDAG DAG 
)
static

Definition at line 36462 of file X86ISelLowering.cpp.

◆ combineSetCC()

static SDValue combineSetCC ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSetCCAtomicArith()

static SDValue combineSetCCAtomicArith ( SDValue  Cmp,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSetCCEFLAGS()

static SDValue combineSetCCEFLAGS ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.

Definition at line 37368 of file X86ISelLowering.cpp.

References checkBoolTestSetCCCombine(), combineCarryThroughADD(), combineSetCCAtomicArith(), and llvm::X86::COND_B.

Referenced by combineBrCond(), and combineX86SetCC().

◆ combineSext()

static SDValue combineSext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineSextInRegCmov()

static SDValue combineSextInRegCmov ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftLeft()

static SDValue combineShiftLeft ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftRightArithmetic()

static SDValue combineShiftRightArithmetic ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftRightLogical()

static SDValue combineShiftRightLogical ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineShuffle()

static SDValue combineShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 33990 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::BITCAST, combineShuffleOfConcatUndef(), combineShuffleToAddSubOrFMAddSub(), combineTargetShuffle(), combineToConsecutiveLoads(), combineX86ShufflesRecursively(), llvm::X86ISD::CVTP2SI, llvm::X86ISD::CVTP2UI, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTTP2SI, llvm::X86ISD::CVTTP2UI, llvm::X86ISD::CVTUI2P, ExperimentalVectorWideningLegalization, llvm::ISD::FADD, llvm::ISD::FMUL, foldShuffleOfHorizOp(), llvm::ISD::FSUB, llvm::MemSDNode::getAlignment(), llvm::APInt::getAllOnesValue(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::ShuffleVectorSDNode::getMask(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::tgtok::In, llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::TargetLoweringBase::isOperationLegal(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ARM_MB::LD, llvm::X86ISD::MCVTP2SI, llvm::X86ISD::MCVTP2UI, llvm::X86ISD::MCVTSI2P, llvm::X86ISD::MCVTTP2SI, llvm::X86ISD::MCVTTP2UI, llvm::X86ISD::MCVTUI2P, llvm::MachineMemOperand::MOLoad, llvm::ISD::MUL, N, narrowShuffle(), llvm::MVT::Other, llvm::X86ISD::PMULUDQ, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::ISD::SUB, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::ISD::VECTOR_SHUFFLE, llvm::X86ISD::VFPROUND, llvm::X86ISD::VMFPROUND, llvm::X86ISD::VZEXT_LOAD, and llvm::X86ISD::VZEXT_MOVL.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineShuffleOfConcatUndef()

static SDValue combineShuffleOfConcatUndef ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShuffleToAddSubOrFMAddSub()

static SDValue combineShuffleToAddSubOrFMAddSub ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineShuffleToFMAddSub()

static SDValue combineShuffleToFMAddSub ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineSignExtendInReg()

static SDValue combineSignExtendInReg ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSIntToFP()

static SDValue combineSIntToFP ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineStore()

static SDValue combineStore ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 40379 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, assert(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, detectAVGPattern(), detectSSatPattern(), detectUSatPattern(), EmitTruncSStore(), ExperimentalVectorWideningLegalization, llvm::MipsISD::Ext, llvm::ISD::EXTRACT_VECTOR_ELT, F(), llvm::MVT::f64, llvm::CallingConv::Fast, llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getTruncStore(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE4A(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::integer_valuetypes(), llvm::EVT::is128BitVector(), llvm::X86Subtarget::is64Bit(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::isPowerOf2_32(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTruncStoreLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isVolatile(), llvm::MemSDNode::isVolatile(), llvm::MipsISD::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::MinAlign(), N, llvm::SDNode::ops(), llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::TargetLowering::scalarizeVectorStore(), splitVectorStore(), llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useSoftFloat(), llvm::MVT::v16i1, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v1i1, llvm::MVT::v2f64, llvm::MVT::v2i1, llvm::MVT::v2i64, llvm::MVT::v32i1, llvm::MVT::v4i1, llvm::MVT::v4i32, llvm::MVT::v64i1, llvm::MVT::v8i1, and llvm::X86ISD::VEXTRACT_STORE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineSub()

static SDValue combineSub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSubToSubus()

static SDValue combineSubToSubus ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineTargetShuffle()

static SDValue combineTargetShuffle ( SDValue  N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Try to combine x86 target specific shuffles.

Definition at line 33345 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDI, combineRedundantDWordShuffle(), combineX86ShufflesRecursively(), D, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::ISD::FDIV, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86ISD::HADD, llvm::SDValue::hasOneUse(), llvm::MipsISD::Hi, llvm::X86ISD::HSUB, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INSERTPS, isNoopShuffleMask(), isTargetShuffle(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::MipsISD::Lo, llvm::makeArrayRef(), llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::ISD::SCALAR_TO_VECTOR, scaleVectorShuffleBlendMask(), setTargetShuffleZeroElements(), Size, llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::SDNode::uses(), llvm::MVT::v4f32, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VPERMI.

Referenced by combineShuffle().

◆ combineToConsecutiveLoads()

static SDValue combineToConsecutiveLoads ( EVT  VT,
SDNode N,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
bool  isAfterLegalize 
)
static

◆ combineToExtendBoolVectorInReg()

static SDValue combineToExtendBoolVectorInReg ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineToExtendCMOV()

static SDValue combineToExtendCMOV ( SDNode Extend,
SelectionDAG DAG 
)
static

◆ combineToExtendVectorInReg()

static SDValue combineToExtendVectorInReg ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements.

Definition at line 42246 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, ExperimentalVectorWideningLegalization, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), getOpcode_EXTEND_VECTOR_INREG(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, Size, llvm::X86Subtarget::useAVX512Regs(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), and combineZext().

◆ combineToFPTruncExtElt()

static SDValue combineToFPTruncExtElt ( SDNode N,
SelectionDAG DAG 
)
static

If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.

This tries to keep the sequence on XMM registers rather than moving between vector and GPRs.

Definition at line 43006 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), and llvm::ISD::TRUNCATE.

Referenced by combineSIntToFP().

◆ combineTruncate()

static SDValue combineTruncate ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineTruncatedArithmetic()

static SDValue combineTruncatedArithmetic ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc DL 
)
static

◆ combineTruncateWithSat()

static SDValue combineTruncateWithSat ( SDValue  In,
EVT  VT,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineUIntToFP()

static SDValue combineUIntToFP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVectorCompare()

static SDValue combineVectorCompare ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVectorCompareAndMaskUnaryOp()

static SDValue combineVectorCompareAndMaskUnaryOp ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineVectorInsert()

static SDValue combineVectorInsert ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorPack()

static SDValue combineVectorPack ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorShiftImm()

static SDValue combineVectorShiftImm ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorShiftVar()

static SDValue combineVectorShiftVar ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorSignBitsTruncation()

static SDValue combineVectorSignBitsTruncation ( SDNode N,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVectorSizedSetCCEquality()

static SDValue combineVectorSizedSetCCEquality ( SDNode SetCC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVectorTruncation()

static SDValue combineVectorTruncation ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.

We do it here because after type legalization the truncation will be translated into a BUILD_VECTOR with each element that is extracted from a vector and then truncated, and it is difficult to do this optimization based on them.

Definition at line 41056 of file X86ISelLowering.cpp.

References combineVectorTruncationWithPACKSS(), combineVectorTruncationWithPACKUS(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::isPowerOf2_32(), llvm::EVT::isSimple(), and llvm::EVT::isVector().

Referenced by combineTruncate().

◆ combineVectorTruncationWithPACKSS()

static SDValue combineVectorTruncationWithPACKSS ( SDNode N,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineVectorTruncationWithPACKUS()

static SDValue combineVectorTruncationWithPACKUS ( SDNode N,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Truncate using ISD::AND mask and X86ISD::PACKUS.

e.g. trunc <8 x i32> X to <8 x i16> –> MaskX = X & 0xffff (clear high bits to prevent saturation) packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)

Definition at line 41026 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::tgtok::In, llvm::X86ISD::PACKUS, and truncateVectorWithPACK().

Referenced by combineVectorTruncation().

◆ combineVSelectToBLENDV()

static SDValue combineVSelectToBLENDV ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.

This function will also call SimplifyDemandedBits on already created BLENDV to perform additional simplifications.

Definition at line 36530 of file X86ISelLowering.cpp.

◆ combineVSelectWithAllOnesOrZeros()

static SDValue combineVSelectWithAllOnesOrZeros ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.

TODO: Move to DAGCombiner, possibly u