LLVM 19.0.0git
Namespaces | Macros | Enumerations | Functions | Variables
X86ISelLowering.cpp File Reference
#include "X86ISelLowering.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86CallingConv.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>

Go to the source code of this file.

Namespaces

namespace  llvm
 This is an optimization pass for GlobalISel generic memory operations.
 
namespace  llvm::X86
 Define some predicates that are used for node matching.
 

Macros

#define DEBUG_TYPE   "x86-isel"
 
#define NODE_NAME_CASE(NODE)   case X86ISD::NODE: return "X86ISD::" #NODE;
 
#define GET_EGPR_IF_ENABLED(OPC)   (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)
 

Enumerations

enum  BitTestKind : unsigned {
  UndefBit , ConstantBit , NotConstantBit , ShiftBit ,
  NotShiftBit
}
 
enum  { llvm::X86::MaxShuffleCombineDepth = 8 }
 
enum class  ShrinkMode { MULS8 , MULU8 , MULS16 , MULU16 }
 Different mul shrinking modes. More...
 

Functions

static bool isTargetShuffle (unsigned Opcode)
 
static bool isTargetShuffleVariableMask (unsigned Opcode)
 
static bool isX86CCSigned (unsigned X86CC)
 Return true if the condition is an signed comparison operation.
 
static X86::CondCode TranslateIntegerX86CC (ISD::CondCode SetCCOpcode)
 
static X86::CondCode TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
 Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
 
static bool hasFPCMov (unsigned X86CC)
 Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
 
static bool useVPTERNLOG (const X86Subtarget &Subtarget, MVT VT)
 
static bool isUndefOrEqual (int Val, int CmpVal)
 Val is the undef sentinel value or equal to the specified value.
 
static bool isUndefOrEqual (ArrayRef< int > Mask, int CmpVal)
 Return true if every element in Mask is the undef sentinel value or equal to the specified value.
 
static bool isUndefOrEqualInRange (ArrayRef< int > Mask, int CmpVal, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value.
 
static bool isUndefOrZero (int Val)
 Val is either the undef or zero sentinel value.
 
static bool isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.
 
static bool isUndefLowerHalf (ArrayRef< int > Mask)
 Return true if the mask creates a vector whose lower half is undefined.
 
static bool isUndefUpperHalf (ArrayRef< int > Mask)
 Return true if the mask creates a vector whose upper half is undefined.
 
static bool isInRange (int Val, int Low, int Hi)
 Return true if Val falls within the specified range (L, H].
 
static bool isAnyInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if the value of any element in Mask falls within the specified range (L, H].
 
static bool isAnyZero (ArrayRef< int > Mask)
 Return true if the value of any element in Mask is the zero sentinel value.
 
static bool isAnyZeroOrUndef (ArrayRef< int > Mask)
 Return true if the value of any element in Mask is the zero or undef sentinel values.
 
static bool isUndefOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef or if its value falls within the specified range (L, H].
 
static bool isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef or if its value falls within the specified range (L, H].
 
static bool isUndefOrZeroOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef, zero or if its value falls within the specified range (L, H].
 
static bool isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].
 
static bool isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
 Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
 
static bool isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.
 
static bool isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
 Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, bool V2IsZero, SmallVectorImpl< int > &WidenedMask)
 
static bool canWidenShuffleElements (ArrayRef< int > Mask)
 
static bool scaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts, SmallVectorImpl< int > &ScaledMask)
 
static SDValue getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
 
static SDValue getConstVector (ArrayRef< APInt > Bits, const APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getConstVector (ArrayRef< APInt > Bits, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all zero elements.
 
static SDValue getSplitVectorSrc (SDValue LHS, SDValue RHS, bool AllowCommute)
 
static SDValue extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 128-bits from a vector > 128 bits.
 
static SDValue extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 256-bits from a 512-bit vector.
 
static SDValue insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to put 128-bits into a vector > 128 bits.
 
static SDValue widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
 
static SDValue widenSubVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, unsigned WideSizeInBits)
 Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
 
static MVT widenMaskVectorType (MVT VT, const X86Subtarget &Subtarget)
 Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
 
static SDValue widenMaskVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
 
static bool collectConcatOps (SDNode *N, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG)
 
static SDValue isUpperSubvectorUndef (SDValue V, const SDLoc &DL, SelectionDAG &DAG)
 
static bool isFreeToSplitVector (SDNode *N, SelectionDAG &DAG)
 
static std::pair< SDValue, SDValuesplitVector (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue splitVectorOp (SDValue Op, SelectionDAG &DAG)
 Break an operation into 2 half sized ops and then concatenate the results.
 
static SDValue splitVectorIntUnary (SDValue Op, SelectionDAG &DAG)
 Break an unary integer operation into 2 half sized ops and then concatenate the result back.
 
static SDValue splitVectorIntBinary (SDValue Op, SelectionDAG &DAG)
 Break a binary integer operation into 2 half sized ops and then concatenate the result back.
 
template<typename F >
SDValue SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true)
 
static SDValue getAVX512Node (unsigned Opcode, const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert i1-subvector to i1-vector.
 
static SDValue concatSubVectors (SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all bits set.
 
static SDValue getEXTEND_VECTOR_INREG (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG)
 
static SDValue getBitSelect (const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS, SDValue Mask, SelectionDAG &DAG)
 
static SDValue getVectorShuffle (SelectionDAG &DAG, EVT VT, const SDLoc &dl, SDValue V1, SDValue V2, ArrayRef< int > Mask)
 
static SDValue getUnpackl (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackl operation.
 
static SDValue getUnpackh (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackh operation.
 
static SDValue getPack (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, bool PackHiHalf=false)
 Returns a node that packs the LHS + RHS nodes together at half width.
 
static SDValue getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return a vector_shuffle of the specified vector of zero or undef vector.
 
static ConstantPoolSDNodegetTargetConstantPoolFromBasePtr (SDValue Ptr)
 
static const ConstantgetTargetConstantFromBasePtr (SDValue Ptr)
 
static const ConstantgetTargetConstantFromNode (LoadSDNode *Load)
 
static const ConstantgetTargetConstantFromNode (SDValue Op)
 
static bool getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=false)
 
bool llvm::X86::isConstantSplat (SDValue Op, APInt &SplatVal, bool AllowPartialUndefs=true)
 If Op is a constant whose elements are all the same constant or undefined, return true and return the constant value in SplatVal.
 
static bool getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts)
 
static SDValue IsNOT (SDValue V, SelectionDAG &DAG)
 
static void createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary, unsigned NumStages=1)
 Create a shuffle mask that matches the PACKSS/PACKUS truncation.
 
static void getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
 
static void getHorizDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
 
static bool getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
 Calculates the shuffle mask corresponding to the target-specific opcode.
 
static bool getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask)
 
static void computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
 Compute whether each element of a shuffle is zeroable.
 
static bool getTargetShuffleAndZeroables (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, APInt &KnownUndef, APInt &KnownZero)
 Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs.
 
static void resolveTargetShuffleFromZeroables (SmallVectorImpl< int > &Mask, const APInt &KnownUndef, const APInt &KnownZero, bool ResolveKnownZeros=true)
 
static void resolveZeroablesFromTargetShuffle (const SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero)
 
static bool createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond, bool IsBLENDV=false)
 
static bool getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
 
static bool getFauxShuffleMask (SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
 
static void resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask)
 Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
 
static bool getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
 Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.
 
static bool getTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth=0, bool ResolveKnownElts=true)
 
static SDValue getBROADCAST_LOAD (unsigned Opcode, const SDLoc &DL, EVT VT, EVT MemVT, MemSDNode *Mem, unsigned Offset, SelectionDAG &DAG)
 
static SDValue getShuffleScalarElt (SDValue Op, unsigned Index, SelectionDAG &DAG, unsigned Depth)
 Returns the scalar element that will make up the i'th element of the result of the vector shuffle.
 
static SDValue LowerBuildVectorAsInsert (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBuildVectorv16i8 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v16i8.
 
static SDValue LowerBuildVectorv8i16 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v8i16.
 
static SDValue LowerBuildVectorv4x32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v4i32 or v4f32.
 
static SDValue getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
 Return a vector logical shift node.
 
static SDValue LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
 
static bool findEltLoadSrc (SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset)
 
static SDValue EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
 Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
 
static SDValue combineToConsecutiveLoads (EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
 
static ConstantgetConstantVector (MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C)
 
static ConstantgetConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C)
 
static bool isFoldableUseOfShuffle (SDNode *N)
 
static SDValue lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.
 
static int getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
 For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
 
static SDValue buildFromShuffleMostly (SDValue Op, const SDLoc &DL, SelectionDAG &DAG)
 
static SDValue LowerBUILD_VECTORvXbf16 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBUILD_VECTORvXi1 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static LLVM_ATTRIBUTE_UNUSED bool isHorizOp (unsigned Opcode)
 
static bool isHorizontalBinOpPart (const BuildVectorSDNode *N, unsigned Opcode, const SDLoc &DL, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
 This is a helper function of LowerToHorizontalOp().
 
static SDValue ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
 Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
 
static bool isAddSubOrSubAdd (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd)
 Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation.
 
static bool isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses)
 Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1).
 
static SDValue lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node.
 
static bool isHopBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1)
 
static SDValue getHopForBuildVector (const BuildVectorSDNode *BV, const SDLoc &DL, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1)
 
static SDValue LowerToHorizontalOp (const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
 
static SDValue LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerBuildVectorToBitOp (BuildVectorSDNode *Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.
 
static SDValue materializeVectorConstant (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Create a vector constant without a load.
 
static SDValue createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.
 
static SDValue LowerBUILD_VECTORAsVariablePermute (SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isNoopShuffleMask (ArrayRef< int > Mask)
 Tiny helper function to identify a no-op mask.
 
static bool isLaneCrossingShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
 Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.
 
static bool is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
 Test whether there are elements crossing 128-bit lanes in this shuffle mask.
 
static bool isMultiLaneShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
 Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles.
 
static bool isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each sub-lane.
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 128-bit lane.
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask)
 
static bool is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 256-bit lane.
 
static bool isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, unsigned EltSizeInBits, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a target shuffle mask is equivalent within each sub-lane.
 
static bool isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a target shuffle mask is equivalent within each sub-lane.
 
static bool IsElementEquivalent (int MaskSize, SDValue Op, SDValue ExpectedOp, int Idx, int ExpectedIdx)
 Checks whether the vector elements referenced by two shuffle masks are equivalent.
 
static bool isShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue())
 Checks whether a shuffle mask is equivalent to an explicit list of arguments.
 
static bool isTargetShuffleEquivalent (MVT VT, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, const SelectionDAG &DAG, SDValue V1=SDValue(), SDValue V2=SDValue())
 Checks whether a target shuffle mask is equivalent to an explicit pattern.
 
static bool isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT, const SelectionDAG &DAG)
 
static bool is128BitUnpackShuffleMask (ArrayRef< int > Mask, const SelectionDAG &DAG)
 
static bool hasIdenticalHalvesShuffleMask (ArrayRef< int > Mask)
 Return true if a shuffle mask chooses elements identically in its top and bottom halves.
 
static unsigned getV4X86ShuffleImm (ArrayRef< int > Mask)
 Get a 4-lane 8-bit shuffle immediate for a mask.
 
static SDValue getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
 
static bool isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
 
static SDValue lowerShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle with a single PSHUFB of V1 or V2.
 
static SDValue getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.
 
static SDValue lowerShuffleToEXPAND (const SDLoc &DL, MVT VT, const APInt &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerShuffleWithUNPCK256 (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit.
 
static bool matchShuffleAsVTRUNC (MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget)
 
static SDValue getAVX512TruncNode (const SDLoc &DL, MVT DstVT, SDValue Src, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool ZeroUppers)
 
static SDValue lowerShuffleWithVPMOV (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsVTRUNC (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static int canLowerByDroppingElements (ArrayRef< int > Mask, bool MatchEven, bool IsSingleInput)
 Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.
 
static bool matchShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned MaxStages=1)
 
static SDValue lowerShuffleWithPACK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a bitmask instruction for a shuffle.
 
static SDValue lowerShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle using bit math.
 
static SDValue getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.
 
static bool matchShuffleAsBlend (MVT VT, SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const APInt &Zeroable, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask)
 
static uint64_t scaleVectorShuffleBlendMask (uint64_t BlendMask, int Size, int Scale)
 
static SDValue lowerShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle.
 
static SDValue lowerShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false)
 Try to lower as a blend of elements from two inputs followed by a single-input permutation.
 
static SDValue lowerShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower as an unpack of elements from two inputs followed by a single-input permutation.
 
static SDValue lowerShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
 
static SDValue lowerShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place.
 
static bool isBroadcastShuffleMask (ArrayRef< int > Mask)
 
static bool isNoopOrBroadcastShuffleMask (ArrayRef< int > Mask)
 
static bool isSingleElementRepeatedMask (ArrayRef< int > Mask)
 Check if the Mask consists of the same element repeated multiple times.
 
static SDValue lowerShuffleAsDecomposedShuffleMerge (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic routine to decompose a shuffle and blend into independent blends and permutes.
 
static int matchShuffleAsBitRotate (MVT &RotateVT, int EltSizeInBits, const X86Subtarget &Subtarget, ArrayRef< int > Mask)
 
static SDValue lowerShuffleAsBitRotate (const SDLoc &DL, MVT VT, SDValue V1, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower shuffle using X86ISD::VROTLI rotations.
 
static int matchShuffleAsElementRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to match a vector shuffle as an element rotation.
 
static int matchShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to lower a vector shuffle as a byte rotation.
 
static SDValue lowerShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsVALIGN (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a dword/qword rotation.
 
static SDValue lowerShuffleAsByteShiftMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a byte shift sequence.
 
static int matchShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget)
 Try to lower a vector shuffle as a bit shift (shifts in zeros).
 
static SDValue lowerShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool BitwiseOnly)
 
static bool matchShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable)
 
static bool matchShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx)
 
static SDValue lowerShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
 
static SDValue lowerShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle as a zero or any extension.
 
static SDValue lowerShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a zero extension on any microarch.
 
static SDValue getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
 Try to get a scalar value for a specific element of a vector.
 
static bool isShuffleFoldableLoad (SDValue V)
 Helper to test for a load that can be folded with x86 shuffles.
 
template<typename T >
static bool isSoftF16 (T VT, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower insertion of a single element into a zero vector.
 
static SDValue lowerShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements.
 
static bool isSingleSHUFPSMask (ArrayRef< int > Mask)
 Test whether this can be lowered with a single SHUFPS instruction.
 
static bool isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
 Test whether the specified input (0 or 1) is in-place blended by the given mask.
 
static SDValue lowerShuffleOfExtractsAsVperm (const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG)
 If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering.
 
static SDValue lowerShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single element.
 
static bool matchShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 
static SDValue lowerV2F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit floating point shuffles.
 
static SDValue lowerV2I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit integer shuffles.
 
static SDValue lowerShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Lower a vector shuffle using the SHUFPS instruction.
 
static SDValue lowerV4F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane 32-bit floating point shuffles.
 
static SDValue lowerV4I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane i32 vector shuffles.
 
static SDValue lowerV8I16GeneralSingleInputShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
 
static SDValue lowerShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
 Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.
 
static SDValue lowerV8I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of 8-lane i16 shuffles.
 
static SDValue lowerV8F16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 8-lane 16-bit floating point shuffles.
 
static SDValue lowerShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerV16I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of v16i8 shuffles.
 
static SDValue lower128BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Dispatching routine to lower various 128-bit x86 vector shuffles.
 
static SDValue splitAndLowerShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool SimpleOnly)
 Generic routine to split vector shuffle into half-sized shuffles.
 
static SDValue lowerShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Either split a vector in halves or decompose the shuffles and the blend/unpack.
 
static SDValue lowerShuffleAsLanePermuteAndSHUFP (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.
 
static void computeInLaneShuffleMask (const ArrayRef< int > &Mask, int LaneSize, SmallVector< int > &InLaneMask)
 Helper to get compute inlane shuffle mask for a complete shuffle mask.
 
static SDValue lowerShuffleAsLanePermuteAndShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation.
 
static SDValue lowerV2X128Shuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering 2-lane 128-bit shuffles.
 
static SDValue lowerShuffleAsLanePermuteAndRepeatedMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
 
static bool getHalfShuffleMask (ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2)
 If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.
 
static SDValue getShuffleHalfVectors (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG, bool UseConcat=false)
 Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width.
 
static SDValue lowerShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
 
static SDValue lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.
 
static bool matchShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, bool &ForceV1Zero, bool &ForceV2Zero, unsigned &ShuffleImm, ArrayRef< int > Mask, const APInt &Zeroable)
 
static SDValue lowerShuffleWithSHUFPD (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsVTRUNCAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 
static SDValue lowerShufflePairAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerV4F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit floating point shuffles.
 
static SDValue lowerV4I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit integer shuffles.
 
static SDValue lowerV8F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit floating point shuffles.
 
static SDValue lowerV8I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit integer shuffles.
 
static SDValue lowerV16I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 16-bit integer shuffles.
 
static SDValue lowerV32I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 8-bit integer shuffles.
 
static SDValue lower256BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 256-bit x86 vector shuffles.
 
static SDValue lowerV4X128Shuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a 128-bit shuffles.
 
static SDValue lowerV8F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit floating point shuffles.
 
static SDValue lowerV16F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit floating point shuffles.
 
static SDValue lowerV8I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit integer shuffles.
 
static SDValue lowerV16I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit integer shuffles.
 
static SDValue lowerV32I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 16-bit integer shuffles.
 
static SDValue lowerV64I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 64-lane 8-bit integer shuffles.
 
static SDValue lower512BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 512-bit x86 vector shuffles.
 
static SDValue lower1BitShuffleAsKSHIFTR (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static int match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
 
static SDValue lower1BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask)
 Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
 
static bool canCombineAsMaskOperation (SDValue V, const X86Subtarget &Subtarget)
 
static SDValue canonicalizeShuffleMaskWithHorizOp (MutableArrayRef< SDValue > Ops, MutableArrayRef< int > Mask, unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerVECTOR_SHUFFLE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Top-level lowering for x86 vector shuffles.
 
static SDValue lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a VSELECT instruction to a vector shuffle.
 
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
 
static SDValue ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Extract one bit from mask vector, like v16i1 or v8i1.
 
static APInt getExtractedDemandedElts (SDNode *N)
 
static SDValue InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert one bit to mask vector, like v16i1 or v8i1.
 
static SDValue LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InGlue, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)
 
static SDValue LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSGeneralDynamicModelX32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool Is64Bit, bool Is64BitLP64)
 
static SDValue LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
 
static SDValue LowerShiftParts (SDValue Op, SelectionDAG &DAG)
 Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
 
static SDValue LowerI64IntToFP_AVX512DQ (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerI64IntToFP16 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool useVectorCast (unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget)
 
static SDValue vectorizeExtractedCast (SDValue Cast, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction.
 
static SDValue lowerFPToIntToFP (SDValue CastToFP, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops.
 
static SDValue lowerINT_TO_FP_vXi64 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue promoteXINT_TO_FP (SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
 
static bool isLegalConversion (MVT VT, bool IsSigned, const X86Subtarget &Subtarget)
 
static bool shouldUseHorizontalOp (bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Horizontal vector math instructions may be slower than normal math with shuffles.
 
static SDValue LowerUINT_TO_FP_i64 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 64-bit unsigned integer to double expansion.
 
static SDValue LowerUINT_TO_FP_i32 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 32-bit unsigned integer to float expansion.
 
static SDValue lowerUINT_TO_FP_v2i32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerUINT_TO_FP_vXi32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerUINT_TO_FP_vec (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
 
static SDValue truncateVectorWithPACKUS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
 
static SDValue truncateVectorWithPACKSS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Truncate using inreg sign extension and X86ISD::PACKSS.
 
static SDValue matchTruncateWithPACK (unsigned &PackOpcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper to determine if In truncated to DstVT has the necessary signbits / leading zero bits to be truncated with PACKSS / PACKUS, possibly by converting a SRL node to SRA for sign extension.
 
static SDValue LowerTruncateVecPackWithSignBits (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.
 
static SDValue LowerTruncateVecPack (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.
 
static SDValue LowerTruncateVecI1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue expandFP_TO_UINT_SSE (MVT VT, SDValue Src, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerFP16_TO_FP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFP_TO_FP16 (SDValue Op, SelectionDAG &DAG)
 
static SDValue lowerAddSubToHorizontalOp (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.
 
static SDValue LowerFROUND (SDValue Op, SelectionDAG &DAG)
 ISD::FROUND is defined to round to nearest with ties rounding away from 0.
 
static SDValue LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
 The only differences between FABS and FNEG are the mask and the logic op.
 
static SDValue LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue getBT (SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG)
 Helper for attempting to create a X86ISD::BT node.
 
static SDValue getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG)
 Helper for creating a X86ISD::SETCC node.
 
static bool isOrXorXorTree (SDValue X, bool Root=true)
 Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion.
 
template<typename F >
static SDValue emitOrXorXorTree (SDValue X, const SDLoc &DL, SelectionDAG &DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV)
 Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.
 
static SDValue combineVectorSizedSetCCEquality (EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.
 
static bool matchScalarReduction (SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps, SmallVectorImpl< APInt > *SrcMask=nullptr)
 Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns.
 
static SDValue LowerVectorAllEqual (const SDLoc &DL, SDValue LHS, SDValue RHS, ISD::CondCode CC, const APInt &OriginalMask, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
 
static SDValue MatchVectorAllEqualTest (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
 
static bool hasNonFlagsUse (SDValue Op)
 return true if Op has a use that doesn't just read flags.
 
static bool isProfitableToUseFlagOp (SDValue Op)
 
static SDValue EmitTest (SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Emit nodes that will be selected as "test Op0,Op0", or something equivalent.
 
static SDValue EmitCmp (SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.
 
static SDValue LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, X86::CondCode &X86CC)
 Result of 'and' is compared against zero.
 
static bool cheapX86FSETCC_SSE (ISD::CondCode SetCCOpcode)
 
static unsigned translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1, bool &IsAlwaysSignaling)
 Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
 
static SDValue splitIntVSETCC (EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SelectionDAG &DAG, const SDLoc &dl)
 Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back.
 
static SDValue LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG)
 
static SDValue incDecVectorConstant (SDValue V, SelectionDAG &DAG, bool IsInc, bool NSW)
 Given a buildvector constant, return a new vector constant with each element incremented or decremented.
 
static SDValue LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 As another special case, use PSUBUS[BW] when it's profitable.
 
static SDValue LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue EmitAVX512Test (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC)
 
static std::pair< SDValue, SDValuegetX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerXALUO (SDValue Op, SelectionDAG &DAG)
 
static bool isX86LogicalCmp (SDValue Op)
 Return true if opcode is a X86 logical comparison.
 
static bool isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue splitVectorStore (StoreSDNode *Store, SelectionDAG &DAG)
 Change a vector store into a pair of half-size vector stores.
 
static SDValue scalarizeVectorStore (StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
 Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
 
static SDValue LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
 Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.
 
static SDValue LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static unsigned getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable)
 
static SDValue getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount is a constant.
 
static SDValue getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, int ShAmtIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle vector element shifts by a splat shift amount.
 
static SDValue getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Creates an SDNode for a predicated scalar operation.
 
static int getSEHRegistrationNodeSize (const Function *Fn)
 
static SDValue recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
 When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
 
static SDValue getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getGatherNode (SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue expandIntrinsicWChainHelper (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.
 
static void getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).
 
static SDValue LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG)
 
static SDValue MarkEHGuard (SDValue Op, SelectionDAG &DAG)
 
static SDValue EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Truncating Store with signed or unsigned saturation.
 
static SDValue EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Masked Truncating Store with signed or unsigned saturation.
 
static SDValue LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
 
static SDValue createSetFPEnvNodes (SDValue Ptr, SDValue Chain, SDLoc DL, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector CTLZ using native supported vector CTLZ instruction.
 
static SDValue LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerAVG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMINMAX (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerFMINIMUM_FMAXIMUM (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerABD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowervXi8MulWithUNPCK (SDValue A, SDValue B, const SDLoc &dl, MVT VT, bool IsSigned, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue *Low=nullptr)
 
static SDValue LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMULO (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool supportedVectorShiftWithImm (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool supportedVectorShiftWithBaseAmnt (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool supportedVectorVarShift (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static SDValue LowerShiftByScalarImmediate (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerShiftByScalarVariable (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static std::pair< Value *, BitTestKindFindSingleBitChange (Value *V)
 
static bool shouldExpandCmpArithRMWInIR (AtomicRMWInst *AI)
 
static SDValue emitLockedStackOp (SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, const SDLoc &DL)
 Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.
 
static SDValue LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue getPMOVMSKB (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Compute the horizontal sum of bytes in V for the elements of VT.
 
static SDValue LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOP (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTPOP (SDValue N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerPARITY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower atomic_load_ops into LOCK-prefixed operations.
 
static SDValue LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerADDSUBO_CARRY (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
 Widen a vector input to a vector of NVT.
 
static SDValue LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADDRSPACECAST (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCVTPS2PH (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerPREFETCH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static StringRef getInstrStrFromOpNo (const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo)
 
static bool isEFLAGSLiveAfter (MachineBasicBlock::iterator Itr, MachineBasicBlock *BB)
 
static MachineBasicBlockemitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
 Utility function to emit xbegin specifying the start of an RTM region.
 
static bool checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
 
static bool isCMOVPseudo (MachineInstr &MI)
 
static MachineInstrBuilder createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
 
static unsigned getSUBriOpcode (bool IsLP64)
 
static unsigned getOpcodeForIndirectThunk (unsigned RPOpc)
 
static const chargetIndirectThunkSymbol (const X86Subtarget &Subtarget, unsigned Reg)
 
static void computeKnownBitsForPSADBW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
 
static SDValue narrowLoadToVZLoad (LoadSDNode *LN, MVT MemVT, MVT VT, SelectionDAG &DAG)
 
static bool matchUnaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue V1, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
 
static bool matchUnaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static bool matchBinaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary)
 
static bool matchBinaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static SDValue combineX86ShuffleChainWithExtract (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Combine an arbitrary chain of shuffles into a single instruction if possible.
 
static SDValue combineX86ShufflesConstants (ArrayRef< SDValue > Ops, ArrayRef< int > Mask, SDValue Root, bool HasVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode * > SrcNodes, unsigned Depth, unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Fully generic combining of x86 shuffle instructions.
 
static SDValue combineX86ShufflesRecursively (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper entry wrapper to combineX86ShufflesRecursively.
 
static SmallVector< int, 4 > getPSHUFShuffleMask (SDValue N)
 Get the PSHUF-style mask from PSHUF node.
 
static SDValue combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
 Search for a combinable shuffle across a chain ending in pshufd.
 
static SDValue combineCommutableSHUFP (SDValue N, MVT VT, const SDLoc &DL, SelectionDAG &DAG)
 
static bool isUnaryOp (unsigned Opcode)
 
static SDValue canonicalizeShuffleWithOp (SDValue N, SelectionDAG &DAG, const SDLoc &DL)
 
static SDValue canonicalizeLaneShuffleWithRepeatedOps (SDValue V, SelectionDAG &DAG, const SDLoc &DL)
 Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
 
static SDValue combineTargetShuffle (SDValue N, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Try to combine x86 target specific shuffles.
 
static bool isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even)
 Checks if the shuffle mask takes subsequent elements alternately from two vectors.
 
static bool isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd)
 Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation.
 
static SDValue combineShuffleToFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
 
static SDValue combineShuffleToAddSubOrFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
 
static SDValue combineShuffleOfConcatUndef (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue narrowShuffle (ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
 If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size.
 
static SDValue combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool checkBitcastSrcVectorSize (SDValue Src, unsigned Size, bool AllowTruncate)
 
static unsigned getAltBitOpcode (unsigned Opcode)
 
static SDValue adjustBitcastSrcVectorSSE1 (SelectionDAG &DAG, SDValue Src, const SDLoc &DL)
 
static SDValue signExtendBitcastSrcVector (SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
 
static SDValue combineBitcastvxi1 (SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG)
 
static SDValue combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue createMMXBuildVector (BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBitcastToBoolVector (EVT VT, SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool detectExtMul (SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0, SDValue &Op1)
 
static bool detectZextAbsDiff (const SDValue &Abs, SDValue &Op0, SDValue &Op1)
 
static SDValue createVPDPBUSD (SelectionDAG &DAG, SDValue LHS, SDValue RHS, unsigned &LogBias, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue combineMinMaxReduction (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combinePredicateReduction (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVPDPBUSDPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue scalarizeExtEltFP (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op.
 
static SDValue combineArithReduction (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.
 
static SDValue combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
 
static SDValue combineToExtendBoolVectorInReg (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.
 
static SDValue narrowVectorSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
 
static SDValue combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.
 
static SDValue combineLogicBlendIntoConditionalNegate (EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue commuteSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on SELECT and VSELECT nodes.
 
static SDValue combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.
 
static SDValue checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
 
static bool checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
 Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
 
static SDValue combineCarryThroughADD (SDValue EFLAGS, SelectionDAG &DAG)
 
static SDValue combinePTESTCC (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.
 
static SDValue combineSetCCMOVMSK (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.
 
static SDValue combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
 
static bool canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
 
static SDValue reduceVMULWidth (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.
 
static SDValue combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL)
 
static SDValue combineMulToPMADDWD (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMulToPMULDQ (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineShiftToPMULH (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineShiftLeft (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineHorizOpWithShuffle (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorHADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.
 
static SDValue combineAndNotIntoANDNP (SDNode *N, SelectionDAG &DAG)
 Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
 
static SDValue combineAndShuffleNot (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y.
 
static SDValue PromoteMaskArithmetic (SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, unsigned Depth)
 
static SDValue PromoteMaskArithmetic (SDValue N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static unsigned convertIntLogicToFPLogicOpcode (unsigned Opcode)
 
static SDValue convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers.
 
static SDValue combineBitOpWithMOVMSK (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineBitOpWithShift (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineBitOpWithPACK (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineAndMaskToShift (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If this is a zero/all-bits result that is bitwise-anded with a low bits mask.
 
static SDValue getIndexFromUnindexedLoad (LoadSDNode *Ld)
 
static bool hasBZHI (const X86Subtarget &Subtarget, MVT VT)
 
static SDValue combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineScalarAndWithMaskSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue getBMIMatchingOp (unsigned Opc, SelectionDAG &DAG, SDValue OpMustEq, SDValue Op, unsigned Depth)
 
static SDValue combineBMILogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue canonicalizeBitSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask)
 
static SDValue combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerX86CmpEqZeroToCtlzSrl (SDValue Op, SelectionDAG &DAG)
 
static SDValue combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue foldMaskedMergeImpl (SDValue And0_L, SDValue And0_R, SDValue And1_L, SDValue And1_R, const SDLoc &DL, SelectionDAG &DAG)
 
static SDValue foldMaskedMerge (SDNode *Node, SelectionDAG &DAG)
 Fold "masked merge" expressions like (m & x) | (~m & y) into the equivalent ((x ^ y) & m) ^ y) pattern.
 
static SDValue combineAddOrSubToADCOrSBB (bool IsSub, const SDLoc &DL, EVT VT, SDValue X, SDValue Y, SelectionDAG &DAG, bool ZeroSecondOpOnly=false)
 If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
 
static SDValue combineAddOrSubToADCOrSBB (SDNode *N, SelectionDAG &DAG)
 If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
 
static SDValue combineOrXorWithSETCC (SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
 
static SDValue combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG)
 Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)
 
static SDValue foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.
 
static SDValue detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
 Detect patterns of truncation with unsigned saturation:
 
static SDValue detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false)
 Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).
 
static SDValue combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue detectAVGPattern (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient ISD::AVGCEILU (AVG) instruction.
 
static SDValue combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static int getOneTrueElt (SDValue V)
 If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.
 
static bool getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, Align &Alignment, unsigned &Offset)
 Given a masked memory load/store operation, return true if it has one mask bit set.
 
static SDValue reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.
 
static SDValue combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.
 
static SDValue combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVEXTRACT_STORE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isHorizontalBinOp (unsigned HOpcode, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative, SmallVectorImpl< int > &PostShuffleMask)
 Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
 
static SDValue combineToHorizontalAddSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFMulcFCMulc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFaddCFmul (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating-point adds/subs.
 
static SDValue combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
 
static SDValue combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 
static SDValue combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVTRUNC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue isFNEG (SelectionDAG &DAG, SDNode *N, unsigned Depth=0)
 Returns the negated value if the node N flips sign of FP value.
 
static unsigned negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc, bool NegRes)
 
static SDValue combineFneg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating point negations.
 
static SDValue lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue foldXor1SetCC (SDNode *N, SelectionDAG &DAG)
 Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
 
static SDValue combineXorSubCTLZ (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineBITREVERSE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineBEXTR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isNullFPScalarOrVectorConst (SDValue V)
 
static SDValue getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.
 
static SDValue combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FAND nodes.
 
static SDValue combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FANDN nodes.
 
static SDValue combineFOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
 
static SDValue combineFMinFMax (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
 
static SDValue combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineCVTP2I_CVTTP2I (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::ANDNP nodes.
 
static SDValue combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineCVTPH2PS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSextInRegCmov (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.
 
static SDValue combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG)
 
static SDValue combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue getInvertedVectorForFMA (SDValue V, SelectionDAG &DAG)
 
static SDValue combineFMA (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineFMADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue truncateAVX512SetCCNoBWI (EVT VT, EVT OpVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization.
 
static SDValue combineSetCC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineTESTP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineX86GatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue rebuildGatherScatter (MaskedGatherScatterSDNode *GorS, SDValue Index, SDValue Base, SDValue Scale, SelectionDAG &DAG)
 
static SDValue combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize branch condition evaluation.
 
static SDValue combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineToFPTruncExtElt (SDNode *N, SelectionDAG &DAG)
 If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.
 
static SDValue combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSIntToFP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool needCarryOrOverflowFlag (SDValue Flags)
 
static bool onlyZeroFlagUsed (SDValue Flags)
 
static SDValue combineCMP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86AddSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSBB (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue matchPMADDWD (SelectionDAG &DAG, SDValue Op0, SDValue Op1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
 
static SDValue matchPMADDWD_2 (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
 
static SDValue combineAddOfPMADDWD (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT)
 
static SDValue pushAddIntoCmovOfConsts (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 CMOV of constants requires materializing constant operands in registers.
 
static SDValue combineAdd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineSubABS (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineSubSetcc (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineConcatVectorOps (const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
 
static SDValue combineCONCAT_VECTORS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineINSERT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue narrowExtractedVectorSelect (SDNode *Ext, const SDLoc &DL, SelectionDAG &DAG)
 If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width.
 
static SDValue combineEXTRACT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineScalarToVector (SDNode *N, SelectionDAG &DAG)
 
static SDValue combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVPMADD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineEXTEND_VECTOR_INREG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineKSHIFT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineFP16_TO_FP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFP_EXTEND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBROADCAST_LOAD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineFP_ROUND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMOVDQ2Q (SDNode *N, SelectionDAG &DAG)
 
static SDValue combinePDEP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static bool matchAsm (StringRef S, ArrayRef< const char * > Pieces)
 
static bool clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)
 
static X86::CondCode parseConstraintCode (llvm::StringRef Constraint)
 
static bool isGRClass (const TargetRegisterClass &RC)
 Check if RC is a general purpose register class.
 
static bool isFRClass (const TargetRegisterClass &RC)
 Check if RC is a vector register class.
 
static bool isVKClass (const TargetRegisterClass &RC)
 Check if RC is a mask register class.
 

Variables

static cl::opt< int > ExperimentalPrefInnermostLoopAlignment ("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden)
 
static cl::opt< int > BrMergingBaseCostThresh ("x86-br-merging-base-cost", cl::init(2), cl::desc("Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden)
 
static cl::opt< int > BrMergingLikelyBias ("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden)
 
static cl::opt< int > BrMergingUnlikelyBias ("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc("Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden)
 
static cl::opt< boolMulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden)
 
const unsigned X87StateSize = 28
 
const unsigned FPStateSize = 32
 
const unsigned FPStateSizeInBits = FPStateSize * 8
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "x86-isel"

Definition at line 70 of file X86ISelLowering.cpp.

◆ GET_EGPR_IF_ENABLED

#define GET_EGPR_IF_ENABLED (   OPC)    (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)

◆ NODE_NAME_CASE

#define NODE_NAME_CASE (   NODE)    case X86ISD::NODE: return "X86ISD::" #NODE;

Enumeration Type Documentation

◆ BitTestKind

Enumerator
UndefBit 
ConstantBit 
NotConstantBit 
ShiftBit 
NotShiftBit 

Definition at line 30316 of file X86ISelLowering.cpp.

◆ ShrinkMode

enum class ShrinkMode
strong

Different mul shrinking modes.

Enumerator
MULS8 
MULU8 
MULS16 
MULU16 

Definition at line 46718 of file X86ISelLowering.cpp.

Function Documentation

◆ adjustBitcastSrcVectorSSE1()

static SDValue adjustBitcastSrcVectorSSE1 ( SelectionDAG DAG,
SDValue  Src,
const SDLoc DL 
)
static

◆ buildFromShuffleMostly()

static SDValue buildFromShuffleMostly ( SDValue  Op,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ canCombineAsMaskOperation()

static bool canCombineAsMaskOperation ( SDValue  V,
const X86Subtarget Subtarget 
)
static

◆ canLowerByDroppingElements()

static int canLowerByDroppingElements ( ArrayRef< int >  Mask,
bool  MatchEven,
bool  IsSingleInput 
)
static

Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

(even) N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

(odd) N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns
N above, or the number of times even/odd elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 10243 of file X86ISelLowering.cpp.

References assert(), llvm::isPowerOf2_32(), N, and llvm::Offset.

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

◆ canonicalizeBitSelect()

static SDValue canonicalizeBitSelect ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ canonicalizeLaneShuffleWithRepeatedOps()

static SDValue canonicalizeLaneShuffleWithRepeatedOps ( SDValue  V,
SelectionDAG DAG,
const SDLoc DL 
)
static

◆ canonicalizeShuffleMaskWithCommute()

static bool canonicalizeShuffleMaskWithCommute ( ArrayRef< int >  Mask)
static

Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.

Definition at line 17431 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().

◆ canonicalizeShuffleMaskWithHorizOp()

static SDValue canonicalizeShuffleMaskWithHorizOp ( MutableArrayRef< SDValue Ops,
MutableArrayRef< int >  Mask,
unsigned  RootSizeInBits,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ canonicalizeShuffleWithOp()

static SDValue canonicalizeShuffleWithOp ( SDValue  N,
SelectionDAG DAG,
const SDLoc DL 
)
static

◆ canReduceVMulWidth()

static bool canReduceVMulWidth ( SDNode N,
SelectionDAG DAG,
ShrinkMode Mode 
)
static

◆ canWidenShuffleElements() [1/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask)
static

Definition at line 3666 of file X86ISelLowering.cpp.

References canWidenShuffleElements().

◆ canWidenShuffleElements() [2/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
const APInt Zeroable,
bool  V2IsZero,
SmallVectorImpl< int > &  WidenedMask 
)
static

◆ canWidenShuffleElements() [3/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
SmallVectorImpl< int > &  WidenedMask 
)
static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 3600 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::M0(), llvm::M1(), llvm::SmallVectorBase< Size_T >::size(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canWidenShuffleElements(), combineExtractWithShuffle(), combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), LowerShift(), lowerShuffleAsBlendAndPermute(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), lowerVECTOR_SHUFFLE(), and scaleShuffleElements().

◆ cheapX86FSETCC_SSE()

static bool cheapX86FSETCC_SSE ( ISD::CondCode  SetCCOpcode)
static

Definition at line 22951 of file X86ISelLowering.cpp.

References llvm::ISD::SETONE, and llvm::ISD::SETUEQ.

Referenced by convertIntLogicToFPLogic(), and LowerVSETCC().

◆ checkAndUpdateEFLAGSKill()

static bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator  SelectItr,
MachineBasicBlock BB,
const TargetRegisterInfo TRI 
)
static

Definition at line 34544 of file X86ISelLowering.cpp.

References isEFLAGSLiveAfter(), and TRI.

◆ checkBitcastSrcVectorSize()

static bool checkBitcastSrcVectorSize ( SDValue  Src,
unsigned  Size,
bool  AllowTruncate 
)
static

◆ checkBoolTestAndOrSetCCCombine()

static bool checkBoolTestAndOrSetCCCombine ( SDValue  Cond,
X86::CondCode CC0,
X86::CondCode CC1,
SDValue Flags,
bool isAnd 
)
static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 45940 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, Cond, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), llvm::ISD::OR, llvm::X86ISD::OR, and llvm::X86ISD::SETCC.

Referenced by combineCMov().

◆ checkBoolTestSetCCCombine()

static SDValue checkBoolTestSetCCCombine ( SDValue  Cmp,
X86::CondCode CC 
)
static

◆ clobbersFlagRegisters()

static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > &  AsmPieces)
static

◆ collectConcatOps()

static bool collectConcatOps ( SDNode N,
SmallVectorImpl< SDValue > &  Ops,
SelectionDAG DAG 
)
static

◆ combineADC()

static SDValue combineADC ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineAdd()

static SDValue combineAdd ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineAddOfPMADDWD()

static SDValue combineAddOfPMADDWD ( SelectionDAG DAG,
SDValue  N0,
SDValue  N1,
const SDLoc DL,
EVT  VT 
)
static

◆ combineAddOrSubToADCOrSBB() [1/2]

static SDValue combineAddOrSubToADCOrSBB ( bool  IsSub,
const SDLoc DL,
EVT  VT,
SDValue  X,
SDValue  Y,
SelectionDAG DAG,
bool  ZeroSecondOpOnly = false 
)
static

◆ combineAddOrSubToADCOrSBB() [2/2]

static SDValue combineAddOrSubToADCOrSBB ( SDNode N,
SelectionDAG DAG 
)
static

If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.

This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}.

Definition at line 49655 of file X86ISelLowering.cpp.

References combineAddOrSubToADCOrSBB(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), N, llvm::ISD::SUB, X, and Y.

◆ combineAnd()

static SDValue combineAnd ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 48895 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::all_of(), llvm::ISD::AND, assert(), llvm::APInt::clearAllBits(), combineAndLoadToBZHI(), combineAndMaskToShift(), combineAndNotIntoANDNP(), combineAndShuffleNot(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineCompareEqual(), combineScalarAndWithMaskSetcc(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, llvm::APInt::exactLogBase2(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::APInt::getAllOnes(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), getBitWidth(), getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::isConstOrConstSplat(), llvm::isOneConstant(), llvm::SDNode::isOnlyUserOf(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isZero(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), matchScalarReduction(), llvm::X86::MaxShuffleCombineDepth, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, RHS, llvm::APInt::setBit(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::APInt::uge(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineAndLoadToBZHI()

static SDValue combineAndLoadToBZHI ( SDNode Node,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineAndMaskToShift()

static SDValue combineAndMaskToShift ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineAndNotIntoANDNP()

static SDValue combineAndNotIntoANDNP ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineAndnp()

static SDValue combineAndnp ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineAndShuffleNot()

static SDValue combineAndShuffleNot ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineArithReduction()

static SDValue combineArithReduction ( SDNode ExtElt,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.

Definition at line 44291 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::ISD::INSERT_VECTOR_ELT, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::Lo, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::MUL, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, shouldUseHorizontalOp(), SplitOpsAndApply(), splitVector(), llvm::ISD::TRUNCATE, llvm::APInt::ule(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineExtractVectorElt().

◆ combineBasicSADPattern()

static SDValue combineBasicSADPattern ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBEXTR()

static SDValue combineBEXTR ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineBitcast()

static SDValue combineBitcast ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 43212 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::EVT::changeVectorElementTypeToInteger(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countl_zero(), createMMXBuildVector(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_TO_SINT, llvm::X86ISD::FXOR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::APFloatBase::IEEEdouble(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::X86ISD::MOVMSK, N, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::OR, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SETLT, llvm::ISD::TRUNCATE, llvm::X86ISD::VBROADCAST_LOAD, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineBitcastToBoolVector()

static SDValue combineBitcastToBoolVector ( EVT  VT,
SDValue  V,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBitcastvxi1()

static SDValue combineBitcastvxi1 ( SelectionDAG DAG,
EVT  VT,
SDValue  Src,
const SDLoc DL,
const X86Subtarget Subtarget 
)
static

◆ combineBitOpWithMOVMSK()

static SDValue combineBitOpWithMOVMSK ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineBitOpWithPACK()

static SDValue combineBitOpWithPACK ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineBitOpWithShift()

static SDValue combineBitOpWithShift ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineBITREVERSE()

static SDValue combineBITREVERSE ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineBMILogicOp()

static SDValue combineBMILogicOp ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBrCond()

static SDValue combineBrCond ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBROADCAST_LOAD()

static SDValue combineBROADCAST_LOAD ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineBT()

static SDValue combineBT ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineCarryThroughADD()

static SDValue combineCarryThroughADD ( SDValue  EFLAGS,
SelectionDAG DAG 
)
static

◆ combineCastedMaskArithmetic()

static SDValue combineCastedMaskArithmetic ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCMov()

static SDValue combineCMov ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCMP()

static SDValue combineCMP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineCommutableSHUFP()

static SDValue combineCommutableSHUFP ( SDValue  N,
MVT  VT,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ combineCompareEqual()

static SDValue combineCompareEqual ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCONCAT_VECTORS()

static SDValue combineCONCAT_VECTORS ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineConcatVectorOps()

static SDValue combineConcatVectorOps ( const SDLoc DL,
MVT  VT,
ArrayRef< SDValue Ops,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.

ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.

Definition at line 55092 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::all_equal(), llvm::all_of(), llvm::X86TargetLowering::allowsMemoryAccess(), llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::SmallVectorImpl< T >::append(), assert(), llvm::X86ISD::BLENDI, llvm::X86ISD::BLENDV, llvm::CallingConv::C, llvm::EVT::changeVectorElementType(), llvm::ISD::CONCAT_VECTORS, concatSubVectors(), llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_UNDEF, llvm::ISD::CTPOP, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, DL, E, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::CallingConv::Fast, llvm::ISD::FDIV, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), getBROADCAST_LOAD(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstantVector(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86Subtarget::getPreferVectorWidth(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86ISD::HSUB, I, Idx, llvm::APInt::insertBits(), llvm::EVT::is256BitVector(), llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::ISD::isNormalLoad(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::X86::mayFoldLoad(), llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ISD::MUL, llvm::none_of(), llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::X86ISD::PSADBW, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86Subtarget::useAVX512Regs(), llvm::X86Subtarget::useBWIRegs(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::ISD::VSELECT, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, widenSubVector(), and llvm::ISD::XOR.

Referenced by combineCONCAT_VECTORS(), and combineINSERT_SUBVECTOR().

◆ combineCVTP2I_CVTTP2I()

static SDValue combineCVTP2I_CVTTP2I ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineCVTPH2PS()

static SDValue combineCVTPH2PS ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineEXTEND_VECTOR_INREG()

static SDValue combineEXTEND_VECTOR_INREG ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineEXTRACT_SUBVECTOR()

static SDValue combineEXTRACT_SUBVECTOR ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 55942 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, llvm::X86ISD::CMPP, Concat, llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isExtOpcode(), llvm::ISD::isExtVecInRegOpcode(), llvm::SelectionDAG::isSplatValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::MOVDDUP, N, narrowExtractedVectorSelect(), llvm::SDNode::ops(), llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), scaleShuffleElements(), llvm::ISD::SINT_TO_FP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, splitVectorIntBinary(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TRUNCATE, llvm::ISD::UINT_TO_FP, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VFPEXT, llvm::ISD::VSELECT, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineExtractVectorElt()

static SDValue combineExtractVectorElt ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 44484 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, llvm::any_of(), assert(), llvm::ISD::BITCAST, combineArithReduction(), combineBasicSADPattern(), combineBitcastvxi1(), combineExtractWithShuffle(), combineMinMaxReduction(), combinePredicateReduction(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineVPDPBUSDPattern(), llvm::commonAlignment(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::TargetLowering::getVectorElementPointer(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), Idx, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::X86ISD::MMX_MOVD2W, N, llvm::peekThroughBitcasts(), llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), scalarizeExtEltFP(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::TRUNCATE, and llvm::SDNode::uses().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineExtractWithShuffle()

static SDValue combineExtractWithShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 43991 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), canWidenShuffleElements(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalarInteger(), isUndefOrZeroInRange(), llvm::Lo, N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), llvm::APInt::ult(), llvm::APInt::urem(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by combineExtractVectorElt().

◆ combineExtSetcc()

static SDValue combineExtSetcc ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFaddCFmul()

static SDValue combineFaddCFmul ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFaddFsub()

static SDValue combineFaddFsub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on floating-point adds/subs.

Definition at line 51442 of file X86ISelLowering.cpp.

References combineFaddCFmul(), combineToHorizontalAddSub(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFAnd()

static SDValue combineFAnd ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 52358 of file X86ISelLowering.cpp.

References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), lowerX86FPLogicOp(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFAndFNotToFAndn()

static SDValue combineFAndFNotToFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFAndn()

static SDValue combineFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 52375 of file X86ISelLowering.cpp.

References getNullFPConstForNullVal(), isNullFPScalarOrVectorConst(), lowerX86FPLogicOp(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFMA()

static SDValue combineFMA ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineFMADDSUB()

static SDValue combineFMADDSUB ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineFMinFMax()

static SDValue combineFMinFMax ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineFMinNumFMaxNum()

static SDValue combineFMinNumFMaxNum ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFMulcFCMulc()

static SDValue combineFMulcFCMulc ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFneg()

static SDValue combineFneg ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineFOr()

static SDValue combineFOr ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineFP16_TO_FP()

static SDValue combineFP16_TO_FP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFP_EXTEND()

static SDValue combineFP_EXTEND ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFP_ROUND()

static SDValue combineFP_ROUND ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineGatherScatter()

static SDValue combineGatherScatter ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineHorizOpWithShuffle()

static SDValue combineHorizOpWithShuffle ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineINSERT_SUBVECTOR()

static SDValue combineINSERT_SUBVECTOR ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineKSHIFT()

static SDValue combineKSHIFT ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineLoad()

static SDValue combineLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 50328 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, E, extractSubVector(), llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::LoadSDNode::getExtensionType(), llvm::SelectionDAG::getExtLoad(), llvm::TypeSize::getFixed(), llvm::EVT::getFixedSizeInBits(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Value::getType(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), I, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::ISD::NON_EXTLOAD, Ptr, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TokenFactor, llvm::SDNode::uses(), and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineLogicBlendIntoConditionalNegate()

static SDValue combineLogicBlendIntoConditionalNegate ( EVT  VT,
SDValue  Mask,
SDValue  X,
SDValue  Y,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineLogicBlendIntoPBLENDV()

static SDValue combineLogicBlendIntoPBLENDV ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMaskedLoad()

static SDValue combineMaskedLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMaskedLoadConstantMask()

static SDValue combineMaskedLoadConstantMask ( MaskedLoadSDNode ML,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineMaskedStore()

static SDValue combineMaskedStore ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMinMaxReduction()

static SDValue combineMinMaxReduction ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMOVDQ2Q()

static SDValue combineMOVDQ2Q ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineMOVMSK()

static SDValue combineMOVMSK ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMul()

static SDValue combineMul ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMulSpecial()

static SDValue combineMulSpecial ( uint64_t  MulAmt,
SDNode N,
SelectionDAG DAG,
EVT  VT,
const SDLoc DL 
)
static

◆ combineMulToPMADDWD()

static SDValue combineMulToPMADDWD ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMulToPMULDQ()

static SDValue combineMulToPMULDQ ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineOr()

static SDValue combineOr ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 49725 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, assert(), canonicalizeBitSelect(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineCompareEqual(), combineLogicBlendIntoPBLENDV(), combineOrXorWithSETCC(), combineX86ShufflesRecursively(), llvm::ISD::CONCAT_VECTORS, Cond, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, extractSubVector(), foldMaskedMerge(), llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::KSHIFTL, llvm::SelectionDAG::MaskedVectorIsZero(), matchScalarReduction(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::APInt::setBit(), llvm::X86ISD::SETCC, llvm::ISD::SETNE, llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineOrCmpEqZeroToCtlzSrl()

static SDValue combineOrCmpEqZeroToCtlzSrl ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineOrXorWithSETCC()

static SDValue combineOrXorWithSETCC ( SDNode N,
SDValue  N0,
SDValue  N1,
SelectionDAG DAG 
)
static

◆ combinePDEP()

static SDValue combinePDEP ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combinePMULDQ()

static SDValue combinePMULDQ ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combinePMULH()

static SDValue combinePMULH ( SDValue  Src,
EVT  VT,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combinePredicateReduction()

static SDValue combinePredicateReduction ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combinePTESTCC()

static SDValue combinePTESTCC ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineRedundantDWordShuffle()

static SDValue combineRedundantDWordShuffle ( SDValue  N,
MutableArrayRef< int >  Mask,
const SDLoc DL,
SelectionDAG DAG 
)
static

Search for a combinable shuffle across a chain ending in pshufd.

We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.

Definition at line 39751 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm_unreachable, N, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by combineTargetShuffle().

◆ combineSBB()

static SDValue combineSBB ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineScalarAndWithMaskSetcc()

static SDValue combineScalarAndWithMaskSetcc ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineScalarToVector()

static SDValue combineScalarToVector ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineSelect()

static SDValue combineSelect ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 45168 of file X86ISelLowering.cpp.

References llvm::And, llvm::ISD::AND, llvm::ISD::BITCAST, llvm::X86ISD::BLENDV, llvm::CallingConv::C, CC, llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), combineLogicBlendIntoConditionalNegate(), combineSelectOfTwoConstants(), combineToExtendBoolVectorInReg(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combinevXi1ConstantToInteger(), commuteSelect(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createShuffleMaskFromVSELECT(), DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isConstOrConstSplat(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), IsNOT(), llvm::isNullConstant(), llvm::isNullOrNullSplat(), llvm::isOneConstant(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZero(), llvm::EVT::isVector(), LHS, llvm::ISD::matchUnaryPredicate(), N, narrowVectorSelect(), llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, llvm::ISD::OR, llvm::X86ISD::PCMPGT, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SELECT, Select, llvm::ISD::SETCC, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::simplifySelect(), std::swap(), llvm::ISD::TRUNCATE, and llvm::ISD::VSELECT.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineSelectOfTwoConstants()

static SDValue combineSelectOfTwoConstants ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineSetCC()

static SDValue combineSetCC ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 53309 of file X86ISelLowering.cpp.

References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, CC, combineVectorSizedSetCCEquality(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::ISD::FNEG, llvm::SelectionDAG::getAllOnesConstant(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), incDecVectorConstant(), llvm::KnownBits::intersectWith(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isInteger(), llvm::APInt::isMinSignedValue(), isNullFPScalarOrVectorConst(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), llvm::EVT::isScalarInteger(), llvm::APInt::isSignBitSet(), llvm::ISD::isSignedIntSetCC(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), LHS, LowerVSETCC(), llvm::SelectionDAG::MaskedValueIsZero(), MatchVectorAllEqualTest(), N, llvm::KnownBits::One, llvm::ISD::OR, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, truncateAVX512SetCCNoBWI(), llvm::ISD::UMIN, and llvm::KnownBits::Zero.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineSetCCAtomicArith()

static SDValue combineSetCCAtomicArith ( SDValue  Cmp,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSetCCEFLAGS()

static SDValue combineSetCCEFLAGS ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.

Definition at line 46448 of file X86ISelLowering.cpp.

References CC, checkBoolTestSetCCCombine(), combineCarryThroughADD(), combinePTESTCC(), combineSetCCAtomicArith(), combineSetCCMOVMSK(), and llvm::X86::COND_B.

Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().

◆ combineSetCCMOVMSK()

static SDValue combineSetCCMOVMSK ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Definition at line 46204 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, CC, llvm::MVT::changeVectorElementTypeToInteger(), llvm::X86ISD::CMP, collectConcatOps(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::APInt::isAllOnes(), isAnyZeroOrUndef(), llvm::APInt::isMask(), llvm::SDValue::isUndef(), llvm::APInt::isZero(), LHS, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PCMPEQ, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, RHS, scaleShuffleElements(), llvm::APInt::setBit(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUB, llvm::X86ISD::TESTP, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by combineSetCCEFLAGS().

◆ combineSext()

static SDValue combineSext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineSextInRegCmov()

static SDValue combineSextInRegCmov ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftLeft()

static SDValue combineShiftLeft ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftRightArithmetic()

static SDValue combineShiftRightArithmetic ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShiftRightLogical()

static SDValue combineShiftRightLogical ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineShiftToPMULH()

static SDValue combineShiftToPMULH ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShuffle()

static SDValue combineShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineShuffleOfConcatUndef()

static SDValue combineShuffleOfConcatUndef ( SDNode N,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShuffleToAddSubOrFMAddSub()

static SDValue combineShuffleToAddSubOrFMAddSub ( SDNode N,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineShuffleToFMAddSub()

static SDValue combineShuffleToFMAddSub ( SDNode N,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineSignExtendInReg()

static SDValue combineSignExtendInReg ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSIntToFP()

static SDValue combineSIntToFP ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineStore()

static SDValue combineStore ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 50755 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::ANY_EXTEND, combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, detectAVGPattern(), detectSSatPattern(), detectUSatPattern(), EmitTruncSStore(), llvm::ISD::EXTRACT_VECTOR_ELT, F, llvm::CallingConv::Fast, llvm::MemSDNode::getAAInfo(), llvm::MemSDNode::getAddressSpace(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::TypeSize::getFixed(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getTruncStore(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::MemSDNode::isSimple(), llvm::EVT::isSimple(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDNode::ops(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRW, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::ISD::SCALAR_TO_VECTOR, scalarizeVectorStore(), splitVectorStore(), llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::Align::value(), llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineSub()

static SDValue combineSub ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineSubABS()

static SDValue combineSubABS ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineSubSetcc()

static SDValue combineSubSetcc ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineTargetShuffle()

static SDValue combineTargetShuffle ( SDValue  N,
const SDLoc DL,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Try to combine x86 target specific shuffles.

Definition at line 40154 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDI, llvm::CallingConv::C, canonicalizeLaneShuffleWithRepeatedOps(), collectConcatOps(), combineCommutableSHUFP(), combineRedundantDWordShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShufflesRecursively(), llvm::ISD::CONCAT_VECTORS, D, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::ISD::FDIV, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::ConstantVector::get(), llvm::MemSDNode::getBasePtr(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::TypeSize::getFixed(), llvm::MVT::getFixedSizeInBits(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), getPSHUFShuffleMask(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleAndZeroables(), getTargetShuffleInputs(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::MVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isEXTLoad(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::MemSDNode::isSimple(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::ISD::isUNINDEXEDLoad(), llvm::VectorType::isValidElementType(), llvm::EVT::isVector(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86::MaxShuffleCombineDepth, llvm::MachineMemOperand::MOLoad, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, N, narrowLoadToVZLoad(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, scaleShuffleElements(), scaleVectorShuffleBlendMask(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::SmallVectorBase< Size_T >::size(), Size, llvm::SM_SentinelUndef, llvm::ISD::SRL, std::swap(), llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, llvm::X86ISD::VZEXT_LOAD, llvm::X86ISD::VZEXT_MOVL, and widenSubVector().

Referenced by combineShuffle().

◆ combineTESTP()

static SDValue combineTESTP ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineToConsecutiveLoads()

static SDValue combineToConsecutiveLoads ( EVT  VT,
SDValue  Op,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
bool  IsAfterLegalize 
)
static

◆ combineToExtendBoolVectorInReg()

static SDValue combineToExtendBoolVectorInReg ( unsigned  Opcode,
const SDLoc DL,
EVT  VT,
SDValue  N0,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineToExtendCMOV()

static SDValue combineToExtendCMOV ( SDNode Extend,
SelectionDAG DAG 
)
static

◆ combineToFPTruncExtElt()

static SDValue combineToFPTruncExtElt ( SDNode N,
SelectionDAG DAG 
)
static

If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.

This tries to keep the sequence on XMM registers rather than moving between vector and GPRs.

Definition at line 53974 of file X86ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, and llvm::ISD::TRUNCATE.

Referenced by combineSIntToFP().

◆ combineToHorizontalAddSub()

static SDValue combineToHorizontalAddSub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineTruncate()

static SDValue combineTruncate ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineTruncatedArithmetic()

static SDValue combineTruncatedArithmetic ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc DL 
)
static

Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.

e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) ) TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove anything that is guaranteed to be transformed by DAGCombiner.

Definition at line 51458 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorOfConstantSDNodes(), IsFreeTruncation, llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineTruncate().

◆ combineTruncateWithSat()

static SDValue combineTruncateWithSat ( SDValue  In,
EVT  VT,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineUIntToFP()

static SDValue combineUIntToFP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVectorCompare()

static SDValue combineVectorCompare ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVectorCompareAndMaskUnaryOp()

static SDValue combineVectorCompareAndMaskUnaryOp ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineVectorHADDSUB()

static SDValue combineVectorHADDSUB ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorInsert()

static SDValue combineVectorInsert ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorPack()

static SDValue combineVectorPack ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorShiftImm()

static SDValue combineVectorShiftImm ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorShiftVar()

static SDValue combineVectorShiftVar ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVectorSizedSetCCEquality()

static SDValue combineVectorSizedSetCCEquality ( EVT  VT,
SDValue  X,
SDValue  Y,
ISD::CondCode  CC,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVEXTRACT_STORE()

static SDValue combineVEXTRACT_STORE ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVPDPBUSDPattern()

static SDValue combineVPDPBUSDPattern ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineVPMADD()

static SDValue combineVPMADD ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineVSelectToBLENDV()

static SDValue combineVSelectToBLENDV ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVSelectWithAllOnesOrZeros()

static SDValue combineVSelectWithAllOnesOrZeros ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineVTRUNC()

static SDValue combineVTRUNC ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combinevXi1ConstantToInteger()

static SDValue combinevXi1ConstantToInteger ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ combineX86AddSub()

static SDValue combineX86AddSub ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineX86GatherScatter()

static SDValue combineX86GatherScatter ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineX86INT_TO_FP()

static SDValue combineX86INT_TO_FP ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineX86SetCC()

static SDValue combineX86SetCC ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Definition at line 53883 of file X86ISelLowering.cpp.

References CC, combineSetCCEFLAGS(), DL, getSETCC(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineX86ShuffleChain()

static SDValue combineX86ShuffleChain ( ArrayRef< SDValue Inputs,
SDValue  Root,
ArrayRef< int >  BaseMask,
int  Depth,
bool  HasVariableMask,
bool  AllowVariableCrossLaneMask,
bool  AllowVariablePerLaneMask,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Combine an arbitrary chain of shuffles into a single instruction if possible.

This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.

Definition at line 38055 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::AllOnes, llvm::ISD::AND, assert(), combineX86ShuffleChainWithExtract(), llvm::ISD::CONCAT_VECTORS, llvm::Depth, DL, llvm::ArrayRef< T >::empty(), extract128BitVector(), extractSubVector(), llvm::X86ISD::EXTRQI, llvm::X86ISD::FAND, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, insertSubVector(), is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isInRange(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::SelectionDAG::isSplatValue(), isTargetShuffleEquivalent(), llvm::TargetLoweringBase::isTypeLegal(), isUndef(), isUndefOrEqual(), isUndefOrInRange(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::MVT::isVector(), LHS, llvm::Lo, lowerShuffleWithPERMV(), matchBinaryPermuteShuffle(), matchBinaryShuffle(), matchShuffleAsEXTRQ(), matchShuffleAsInsertPS(), matchShuffleAsINSERTQ(), matchShuffleAsVTRUNC(), matchUnaryPermuteShuffle(), matchUnaryShuffle(), llvm::X86::mayFoldLoad(), llvm::narrowShuffleMaskElts(), OpIndex, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveZeroablesFromTargetShuffle(), RHS, llvm::ISD::SCALAR_TO_VECTOR, scaleShuffleElements(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::X86ISD::SHUF128, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::X86Subtarget::useAVX512Regs(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPPERM, llvm::ISD::VSELECT, llvm::X86ISD::VTRUNC, and widenSubVector().

Referenced by combineX86ShuffleChainWithExtract(), and combineX86ShufflesRecursively().

◆ combineX86ShuffleChainWithExtract()

static SDValue combineX86ShuffleChainWithExtract ( ArrayRef< SDValue Inputs,
SDValue  Root,
ArrayRef< int >  BaseMask,
int  Depth,
bool  HasVariableMask,
bool  AllowVariableCrossLaneMask,
bool  AllowVariablePerLaneMask,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineX86ShufflesConstants()

static SDValue combineX86ShufflesConstants ( ArrayRef< SDValue Ops,
ArrayRef< int >  Mask,
SDValue  Root,
bool  HasVariableMask,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineX86ShufflesRecursively() [1/2]

static SDValue combineX86ShufflesRecursively ( ArrayRef< SDValue SrcOps,
int  SrcOpIndex,
SDValue  Root,
ArrayRef< int >  RootMask,
ArrayRef< const SDNode * >  SrcNodes,
unsigned  Depth,
unsigned  MaxDepth,
bool  HasVariableMask,
bool  AllowVariableCrossLaneMask,
bool  AllowVariablePerLaneMask,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Fully generic combining of x86 shuffle instructions.

This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:

1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.

Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.

FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.

Definition at line 39258 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), llvm::SDNode::areOnlyUsersOf(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::countr_zero(), llvm::Depth, E, llvm::ArrayRef< T >::empty(), llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::ISD::EXTRACT_SUBVECTOR, extractBits(), llvm::APInt::extractBits(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), getOnesVector(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::Hi, I, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllOnes(), isInRange(), llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::EVT::isSimple(), isTargetShuffleVariableMask(), isUndefOrZero(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::APInt::isZero(), llvm::KnownBits::isZero(), LHS, llvm::Lo, MaxDepth, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleFromZeroables(), resolveTargetShuffleInputsAndMask(), RHS, llvm::APIntOps::ScaleBitMask(), llvm::APInt::setBit(), llvm::TargetLowering::SimplifyMultipleUseDemandedVectorElts(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::APInt::trunc(), widenSubVector(), and llvm::APInt::zext().

Referenced by combineAnd(), combineAndnp(), combineEXTEND_VECTOR_INREG(), combineINSERT_SUBVECTOR(), combineOr(), combineShuffle(), combineTargetShuffle(), combineVectorInsert(), combineVectorPack(), combineVectorShiftImm(), combineX86ShufflesRecursively(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

◆ combineX86ShufflesRecursively() [2/2]

static SDValue combineX86ShufflesRecursively ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Helper entry wrapper to combineX86ShufflesRecursively.

Definition at line 39695 of file X86ISelLowering.cpp.

References combineX86ShufflesRecursively(), and llvm::X86::MaxShuffleCombineDepth.

◆ combineXor()

static SDValue combineXor ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineXorSubCTLZ()

static SDValue combineXorSubCTLZ ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineZext()

static SDValue combineZext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ commuteSelect()

static SDValue commuteSelect ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ computeInLaneShuffleMask()

static void computeInLaneShuffleMask ( const ArrayRef< int > &  Mask,
int  LaneSize,
SmallVector< int > &  InLaneMask 
)
static

Helper to get compute inlane shuffle mask for a complete shuffle mask.

Definition at line 14803 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::assign(), and Size.

Referenced by lowerShuffleAsLanePermuteAndShuffle(), and lowerV8F32Shuffle().

◆ computeKnownBitsForPSADBW()

static void computeKnownBitsForPSADBW ( SDValue  LHS,
SDValue  RHS,
KnownBits Known,
const APInt DemandedElts,
const SelectionDAG DAG,
unsigned  Depth 
)
static

◆ computeZeroableShuffleElements()

static void computeZeroableShuffleElements ( ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
APInt KnownUndef,
APInt KnownZero 
)
static

Compute whether each element of a shuffle is zeroable.

A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.

Definition at line 5477 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::extractBits(), llvm::SDValue::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZero(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86::isZeroNode(), llvm::peekThroughBitcasts(), llvm::APInt::setBit(), and Size.

Referenced by lowerVECTOR_SHUFFLE().

◆ concatSubVectors()

static SDValue concatSubVectors ( SDValue  V1,
SDValue  V2,
SelectionDAG DAG,
const SDLoc dl 
)
static

◆ convertIntLogicToFPLogic()

static SDValue convertIntLogicToFPLogic ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ convertIntLogicToFPLogicOpcode()

static unsigned convertIntLogicToFPLogicOpcode ( unsigned  Opcode)
static

◆ convertShiftLeftToScale()

static SDValue convertShiftLeftToScale ( SDValue  Amt,
const SDLoc dl,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ createMMXBuildVector()

static SDValue createMMXBuildVector ( BuildVectorSDNode BV,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ createPackShuffleMask()

static void createPackShuffleMask ( MVT  VT,
SmallVectorImpl< int > &  Mask,
bool  Unary,
unsigned  NumStages = 1 
)
static

Create a shuffle mask that matches the PACKSS/PACKUS truncation.

A multi-stage pack shuffle mask is created by specifying NumStages > 1. Note: This ignores saturation, so inputs must be checked first.

Definition at line 5079 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), and llvm::Offset.

Referenced by getFauxShuffleMask(), and matchShuffleWithPACK().

◆ createPHIsForCMOVsInSinkBB()

static MachineInstrBuilder createPHIsForCMOVsInSinkBB ( MachineBasicBlock::iterator  MIItBegin,
MachineBasicBlock::iterator  MIItEnd,
MachineBasicBlock TrueMBB,
MachineBasicBlock FalseMBB,
MachineBasicBlock SinkMBB 
)
static

◆ createPSADBW()

static SDValue createPSADBW ( SelectionDAG DAG,
const SDValue Zext0,
const SDValue Zext1,
const SDLoc DL,
const X86Subtarget Subtarget 
)
static

◆ createSetFPEnvNodes()

static SDValue createSetFPEnvNodes ( SDValue  Ptr,
SDValue  Chain,
SDLoc  DL,
EVT  MemVT,
MachineMemOperand MMO,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ createShuffleMaskFromVSELECT()

static bool createShuffleMaskFromVSELECT ( SmallVectorImpl< int > &  Mask,
SDValue  Cond,
bool  IsBLENDV = false 
)
static

◆ createVariablePermute()

static SDValue createVariablePermute ( MVT  VT,
SDValue  SrcVec,
SDValue  IndicesVec,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ createVPDPBUSD()

static SDValue createVPDPBUSD ( SelectionDAG DAG,
SDValue  LHS,
SDValue  RHS,
unsigned LogBias,
const SDLoc DL,
const X86Subtarget Subtarget 
)
static

◆ detectAVGPattern()

static SDValue detectAVGPattern ( SDValue  In,
EVT  VT,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc DL 
)
static

◆ detectExtMul()

static bool detectExtMul ( SelectionDAG DAG,
const SDValue Mul,
SDValue Op0,
SDValue Op1 
)
static

◆ detectPMADDUBSW()

static SDValue detectPMADDUBSW ( SDValue  In,
EVT  VT,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc DL 
)
static

◆ detectSSatPattern()

static SDValue detectSSatPattern ( SDValue  In,
EVT  VT,
bool  MatchPackUS = false 
)
static

Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).

With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type]. Return the source value to be truncated or SDValue() if the pattern was not matched.

Definition at line 50043 of file X86ISelLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::APInt::getAllOnes(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::ISD::isConstantSplatVector(), llvm::APInt::sext(), llvm::SMax, llvm::ISD::SMAX, llvm::SMin, llvm::ISD::SMIN, and llvm::APInt::zext().

Referenced by combineStore(), combineTruncateWithSat(), combineVTRUNC(), and detectPMADDUBSW().

◆ detectUSatPattern()

static SDValue detectUSatPattern ( SDValue  In,
EVT  VT,
SelectionDAG DAG,
const SDLoc DL 
)
static

Detect patterns of truncation with unsigned saturation:

  1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type). Return the source value x to be truncated or SDValue() if the pattern was not matched.
  2. (truncate (smin (smax (x, C1), C2)) to dest_type), where C1 >= 0 and C2 is unsigned max of destination type.

    (truncate (smax (smin (x, C2), C1)) to dest_type) where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.

    These two patterns are equivalent to: (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type) So return the smax(x, C1) value to be truncated or SDValue() if the pattern was not matched.

Definition at line 49996 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), llvm::APInt::isNonNegative(), llvm::SMax, llvm::ISD::SMAX, llvm::SMin, llvm::ISD::SMIN, llvm::APInt::uge(), llvm::UMin, and llvm::ISD::UMIN.

Referenced by combineStore(), combineTruncateWithSat(), and combineVTRUNC().

◆ detectZextAbsDiff()

static bool detectZextAbsDiff ( const SDValue Abs,
SDValue Op0,
SDValue Op1 
)
static

◆ EltsFromConsecutiveLoads()

static SDValue EltsFromConsecutiveLoads ( EVT  VT,
ArrayRef< SDValue Elts,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
bool  IsAfterLegalize 
)
static

Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.

Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a

Definition at line 6826 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::sampleprof::Base, concatSubVectors(), llvm::APInt::countr_zero(), DL, llvm::ArrayRef< T >::drop_back(), EltsFromConsecutiveLoads(), llvm::APInt::extractBits(), findEltLoadSrc(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getFloatingPointVT(), llvm::MVT::getFloatingPointVT(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MachinePointerInfo::isDereferenceable(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isOperationLegal(), llvm::isPowerOf2_32(), llvm::MemSDNode::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndef(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Match, llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::MachineMemOperand::MOLoad, llvm::Offset, llvm::peekThroughBitcasts(), llvm::APInt::popcount(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VZEXT_LOAD.

Referenced by combineConcatVectorOps(), combineToConsecutiveLoads(), and EltsFromConsecutiveLoads().

◆ EmitAVX512Test()

static SDValue EmitAVX512Test ( SDValue  Op0,
SDValue  Op1,
ISD::CondCode  CC,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SDValue X86CC 
)
static

◆ EmitCmp()

static SDValue EmitCmp ( SDValue  Op0,
SDValue  Op1,
unsigned  X86CC,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ emitLockedStackOp()

static SDValue emitLockedStackOp ( SelectionDAG DAG,
const X86Subtarget Subtarget,
SDValue  Chain,
const SDLoc DL 
)
static

Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.

Location is chosen to be a) very likely accessed only by a single thread to minimize cache traffic, and b) definitely dereferenceable. Returns the new Chain result.

Definition at line 30786 of file X86ISelLowering.cpp.

References DL, llvm::X86Subtarget::getFrameLowering(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), and llvm::X86FrameLowering::has128ByteRedZone().

Referenced by LowerATOMIC_FENCE(), LowerATOMIC_STORE(), and lowerAtomicArith().

◆ EmitMaskedTruncSStore()

static SDValue EmitMaskedTruncSStore ( bool  SignedSat,
SDValue  Chain,
const SDLoc DL,
SDValue  Val,
SDValue  Ptr,
SDValue  Mask,
EVT  MemVT,
MachineMemOperand MMO,
SelectionDAG DAG 
)
static

Emit Masked Truncating Store with signed or unsigned saturation.

Definition at line 26662 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), Ptr, llvm::X86ISD::VMTRUNCSTORES, and llvm::X86ISD::VMTRUNCSTOREUS.

Referenced by LowerINTRINSIC_W_CHAIN().

◆ emitOrXorXorTree()

template<typename F >
static SDValue emitOrXorXorTree ( SDValue  X,
const SDLoc DL,
SelectionDAG DAG,
EVT  VecVT,
EVT  CmpVT,
bool  HasPT,
F  SToV 
)
static

◆ EmitTest()

static SDValue EmitTest ( SDValue  Op,
unsigned  X86CC,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ EmitTruncSStore()

static SDValue EmitTruncSStore ( bool  SignedSat,
SDValue  Chain,
const SDLoc DL,
SDValue  Val,
SDValue  Ptr,
EVT  MemVT,
MachineMemOperand MMO,
SelectionDAG DAG 
)
static

◆ emitXBegin()

static MachineBasicBlock * emitXBegin ( MachineInstr MI,
MachineBasicBlock MBB,
const TargetInstrInfo TII 
)
static

◆ expandFP_TO_UINT_SSE()

static SDValue expandFP_TO_UINT_SSE ( MVT  VT,
SDValue  Src,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ ExpandHorizontalBinOp()

static SDValue ExpandHorizontalBinOp ( const SDValue V0,
const SDValue V1,
const SDLoc DL,
SelectionDAG DAG,
unsigned  X86Opcode,
bool  Mode,
bool  isUndefLO,
bool  isUndefHI 
)
static

Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.

This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.

The kind of horizontal binary operation is defined by X86Opcode.

Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI

Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI

If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.

Definition at line 7801 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::is256BitVector(), and llvm::SDNode::isUndef().

Referenced by LowerToHorizontalOp().

◆ expandIntrinsicWChainHelper()

static SDValue expandIntrinsicWChainHelper ( SDNode N,
const SDLoc DL,
SelectionDAG DAG,
unsigned  TargetOpcode,
unsigned  SrcReg,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static

Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.

If operand ScrReg is a valid register identifier, then operand 2 of N is copied to SrcReg. The assumption is that SrcReg is an implicit input to TargetOpcode. Returns a Glue value which can be used to add extra copy-from-reg if the expanded intrinsics implicitly defines extra registers (i.e. not just EDX:EAX).

Definition at line 26528 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_PAIR, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), N, llvm::ISD::OR, Results, and llvm::ISD::SHL.

Referenced by getReadTimeStampCounter(), LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().

◆ ExtendToType()

static SDValue ExtendToType ( SDValue  InOp,
MVT  NVT,
SelectionDAG DAG,
bool  FillWithZeroes = false 
)
static

◆ extract128BitVector()

static SDValue extract128BitVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static

Generate a DAG to grab 128-bits from a vector > 128 bits.

This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.

Definition at line 3874 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineExtractWithShuffle(), combineTargetShuffle(), combineX86ShuffleChain(), ExpandHorizontalBinOp(), getTargetVShiftNode(), lowerAddSubToHorizontalOp(), lowerShuffleAsBroadcast(), LowerTruncateVecI1(), lowerV8I16Shuffle(), narrowExtractedVectorSelect(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), and vectorizeExtractedCast().

◆ extract256BitVector()

static SDValue extract256BitVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static

Generate a DAG to grab 256-bits from a 512-bit vector.

Definition at line 3882 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().

◆ ExtractBitFromMaskVector()

static SDValue ExtractBitFromMaskVector ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ extractSubVector()

static SDValue extractSubVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl,
unsigned  vectorWidth 
)
static

◆ findEltLoadSrc()

static bool findEltLoadSrc ( SDValue  Elt,
LoadSDNode *&  Ld,
int64_t &  ByteOffset 
)
static

◆ FindSingleBitChange()

static std::pair< Value *, BitTestKind > FindSingleBitChange ( Value V)
static

◆ foldMaskedMerge()

static SDValue foldMaskedMerge ( SDNode Node,
SelectionDAG DAG 
)
static

Fold "masked merge" expressions like (m & x) | (~m & y) into the equivalent ((x ^ y) & m) ^ y) pattern.

This is typically a better representation for targets without a fused "and-not" operation. This function is intended to be called from a TargetLowering::PerformDAGCombine callback on ISD::OR nodes.

Definition at line 49440 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, foldMaskedMergeImpl(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), and llvm::ISD::OR.

Referenced by combineOr().

◆ foldMaskedMergeImpl()

static SDValue foldMaskedMergeImpl ( SDValue  And0_L,
SDValue  And0_R,
SDValue  And1_L,
SDValue  And1_R,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ foldVectorXorShiftIntoCmp()

static SDValue foldVectorXorShiftIntoCmp ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ foldXor1SetCC()

static SDValue foldXor1SetCC ( SDNode N,
SelectionDAG DAG 
)
static

Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)

Definition at line 52100 of file X86ISelLowering.cpp.

References DL, llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::isOneConstant(), LHS, N, llvm::X86ISD::SETCC, and llvm::ISD::XOR.

Referenced by combineXor().

◆ foldXorTruncShiftIntoCmp()

static SDValue foldXorTruncShiftIntoCmp ( SDNode N,
SelectionDAG DAG 
)
static

◆ getAltBitOpcode()

static unsigned getAltBitOpcode ( unsigned  Opcode)
static

◆ getAVX2GatherNode()

static SDValue getAVX2GatherNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static

◆ getAVX512Node()

static SDValue getAVX512Node ( unsigned  Opcode,
const SDLoc DL,
MVT  VT,
ArrayRef< SDValue Ops,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ getAVX512TruncNode()

static SDValue getAVX512TruncNode ( const SDLoc DL,
MVT  DstVT,
SDValue  Src,
const X86Subtarget Subtarget,
SelectionDAG DAG,
bool  ZeroUppers 
)
static

◆ getBitSelect()

static SDValue getBitSelect ( const SDLoc DL,
MVT  VT,
SDValue  LHS,
SDValue  RHS,
SDValue  Mask,
SelectionDAG DAG 
)
static

◆ getBMIMatchingOp()

static SDValue getBMIMatchingOp ( unsigned  Opc,
SelectionDAG DAG,
SDValue  OpMustEq,
SDValue  Op,
unsigned  Depth 
)
static

◆ getBROADCAST_LOAD()

static SDValue getBROADCAST_LOAD ( unsigned  Opcode,
const SDLoc DL,
EVT  VT,
EVT  MemVT,
MemSDNode Mem,
unsigned  Offset,
SelectionDAG DAG 
)
static

◆ getBT()

static SDValue getBT ( SDValue  Src,
SDValue  BitNo,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ getConstantVector() [1/2]

static Constant * getConstantVector ( MVT  VT,
ArrayRef< APInt Bits,
const APInt Undefs,
LLVMContext C 
)
static

◆ getConstantVector() [2/2]

static Constant * getConstantVector ( MVT  VT,
const APInt SplatValue,
unsigned  SplatBitSize,
LLVMContext C 
)
static

◆ getConstVector() [1/3]

static SDValue getConstVector ( ArrayRef< APInt Bits,
const APInt Undefs,
MVT  VT,
SelectionDAG DAG,
const SDLoc dl 
)
static

◆ getConstVector() [2/3]

static SDValue getConstVector ( ArrayRef< APInt Bits,
MVT  VT,
SelectionDAG DAG,
const SDLoc dl 
)
static

Definition at line 3780 of file X86ISelLowering.cpp.

References getConstVector(), and llvm::APInt::getZero().

◆ getConstVector() [3/3]

static SDValue getConstVector ( ArrayRef< int >  Values,
MVT  VT,
SelectionDAG DAG,
const SDLoc dl,
bool  IsMask = false 
)
static

◆ getEXTEND_VECTOR_INREG()

static SDValue getEXTEND_VECTOR_INREG ( unsigned  Opcode,
const SDLoc DL,
EVT  VT,
SDValue  In,
SelectionDAG DAG 
)
static

◆ getExtractedDemandedElts()

static APInt getExtractedDemandedElts ( SDNode N)
static

◆ getFauxShuffleMask()

static bool getFauxShuffleMask ( SDValue  N,
const APInt DemandedElts,
SmallVectorImpl< int > &  Mask,
SmallVectorImpl< SDValue > &  Ops,
const SelectionDAG DAG,
unsigned  Depth,
bool  ResolveKnownElts 
)
static

Definition at line 5734 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDV, llvm::SmallVectorImpl< T >::clear(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createPackShuffleMask(), createShuffleMaskFromVSELECT(), llvm::DecodeZeroExtendMask(), llvm::Depth, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::EVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPackDemandedElts(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isByteSized(), isInRange(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::X86ISD::VBROADCAST, llvm::ISD::VECTOR_SHUFFLE, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::ISD::VSELECT, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::X86ISD::VTRUNC, llvm::ISD::ZERO_EXTEND, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::APInt::zextOrTrunc().

Referenced by getTargetShuffleInputs().

◆ getGatherNode()

static SDValue getGatherNode ( SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static

◆ getHalfShuffleMask()

static bool getHalfShuffleMask ( ArrayRef< int >  Mask,
MutableArrayRef< int >  HalfMask,
int &  HalfIdx1,
int &  HalfIdx2 
)
static

If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.

A mask of half the width with mask indexes adjusted to access the extracted halves of the original shuffle operands is returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or lower half of each input operand is accessed.

Definition at line 15175 of file X86ISelLowering.cpp.

References assert(), isUndefLowerHalf(), isUndefUpperHalf(), and llvm::ArrayRef< T >::size().

Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().

◆ getHopForBuildVector()

static SDValue getHopForBuildVector ( const BuildVectorSDNode BV,
const SDLoc DL,
SelectionDAG DAG,
unsigned  HOpcode,
SDValue  V0,
SDValue  V1 
)
static

◆ getHorizDemandedElts()

static void getHorizDemandedElts ( EVT  VT,
const APInt DemandedElts,
APInt DemandedLHS,
APInt DemandedRHS 
)
static

◆ getIndexFromUnindexedLoad()

static SDValue getIndexFromUnindexedLoad ( LoadSDNode Ld)
static

◆ getIndirectThunkSymbol()

static const char * getIndirectThunkSymbol ( const X86Subtarget Subtarget,
unsigned  Reg 
)
static

Definition at line 35324 of file X86ISelLowering.cpp.

References assert(), and llvm_unreachable.

◆ getInstrStrFromOpNo()

static StringRef getInstrStrFromOpNo ( const SmallVectorImpl< StringRef > &  AsmStrs,
unsigned  OpNo 
)
static

◆ getInvertedVectorForFMA()

static SDValue getInvertedVectorForFMA ( SDValue  V,
SelectionDAG DAG 
)
static

◆ getMaskNode()

static SDValue getMaskNode ( SDValue  Mask,
MVT  MaskVT,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static

◆ getNullFPConstForNullVal()

static SDValue getNullFPConstForNullVal ( SDValue  V,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.

In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.

Definition at line 52315 of file X86ISelLowering.cpp.

References getZeroVector(), and isNullFPScalarOrVectorConst().

Referenced by combineFAnd(), and combineFAndn().

◆ getOnesVector()

static SDValue getOnesVector ( EVT  VT,
SelectionDAG DAG,
const SDLoc dl 
)
static

Returns a vector of specified type with all bits set.

Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>. Then bitcast to their original type, ensuring they get CSE'd.

Definition at line 4419 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by combineEXTRACT_SUBVECTOR(), combineX86ShufflesRecursively(), and materializeVectorConstant().

◆ getOneTrueElt()

static int getOneTrueElt ( SDValue  V)
static

If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.

Otherwise, return -1.

Definition at line 50474 of file X86ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), and llvm::EVT::getVectorNumElements().

Referenced by getParamsForOneTrueMaskedElt().

◆ getOpcodeForIndirectThunk()

static unsigned getOpcodeForIndirectThunk ( unsigned  RPOpc)
static

Definition at line 35310 of file X86ISelLowering.cpp.

References llvm_unreachable.

◆ getPack()

static SDValue getPack ( SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc dl,
MVT  VT,
SDValue  LHS,
SDValue  RHS,
bool  PackHiHalf = false 
)
static

◆ getPackDemandedElts()

static void getPackDemandedElts ( EVT  VT,
const APInt DemandedElts,
APInt DemandedLHS,
APInt DemandedRHS 
)
static

◆ getParamsForOneTrueMaskedElt()

static bool getParamsForOneTrueMaskedElt ( MaskedLoadStoreSDNode MaskedOp,
SelectionDAG DAG,
SDValue Addr,
SDValue Index,
Align Alignment,
unsigned Offset 
)
static

Given a masked memory load/store operation, return true if it has one mask bit set.

If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.

Definition at line 50509 of file X86ISelLowering.cpp.

References Addr, llvm::commonAlignment(), llvm::MemSDNode::getBasePtr(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::MemSDNode::getOriginalAlign(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::Offset.

Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().

◆ getPMOVMSKB()

static SDValue getPMOVMSKB ( const SDLoc DL,
SDValue  V,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ getPrefetchNode()

static SDValue getPrefetchNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static

◆ getPSHUFShuffleMask()

static SmallVector< int, 4 > getPSHUFShuffleMask ( SDValue  N)
static

Get the PSHUF-style mask from PSHUF node.

This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.

Definition at line 39708 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, N, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.

Referenced by combineRedundantDWordShuffle(), and combineTargetShuffle().

◆ getReadTimeStampCounter()

static void getReadTimeStampCounter ( SDNode N,
const SDLoc DL,
unsigned  Opcode,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static

Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).

This function is also used to custom lower READCYCLECOUNTER nodes.

Definition at line 26583 of file X86ISelLowering.cpp.

References DL, expandIntrinsicWChainHelper(), llvm::SelectionDAG::getCopyFromReg(), llvm::SDValue::getValue(), N, and Results.

Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().

◆ getScalarMaskingNode()

static SDValue getScalarMaskingNode ( SDValue  Op,
SDValue  Mask,
SDValue  PreservedSrc,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Creates an SDNode for a predicated scalar operation.

Returns
(X86vselect Mask, Op, PreservedSrc). The mask is coming as MVT::i8 and it should be transformed to MVT::v1i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction.

Definition at line 25291 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_SAE, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getZeroVector(), llvm::SDValue::isUndef(), llvm::X86ISD::SELECTS, and llvm::X86ISD::VFPCLASSS.

◆ getScalarValueForVectorElement()

static SDValue getScalarValueForVectorElement ( SDValue  V,
int  Idx,
SelectionDAG DAG 
)
static

◆ getScatterNode()

static SDValue getScatterNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static

◆ getSEHRegistrationNodeSize()

static int getSEHRegistrationNodeSize ( const Function Fn)
static

◆ getSETCC()

static SDValue getSETCC ( X86::CondCode  Cond,
SDValue  EFLAGS,
const SDLoc dl,
SelectionDAG DAG 
)
static

◆ getShuffleHalfVectors()

static SDValue getShuffleHalfVectors ( const SDLoc DL,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  HalfMask,
int  HalfIdx1,
int  HalfIdx2,
bool  UndefLower,
SelectionDAG DAG,
bool  UseConcat = false 
)
static

◆ getShuffleScalarElt()

static SDValue getShuffleScalarElt ( SDValue  Op,
unsigned  Index,
SelectionDAG DAG,
unsigned  Depth 
)
static

◆ getShuffleVectorZeroOrUndef()

static SDValue getShuffleVectorZeroOrUndef ( SDValue  V2,
int  Idx,
bool  IsZero,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Return a vector_shuffle of the specified vector of zero or undef vector.

This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).

Definition at line 4600 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and Idx.

Referenced by LowerUINT_TO_FP_i32().

◆ getSplitVectorSrc()

static SDValue getSplitVectorSrc ( SDValue  LHS,
SDValue  RHS,
bool  AllowCommute 
)
static

◆ getSUBriOpcode()

static unsigned getSUBriOpcode ( bool  IsLP64)
static

Definition at line 34948 of file X86ISelLowering.cpp.

◆ getTargetConstantBitsFromNode()

static bool getTargetConstantBitsFromNode ( SDValue  Op,
unsigned  EltSizeInBits,
APInt UndefElts,
SmallVectorImpl< APInt > &  EltBits,
bool  AllowWholeUndefs = true,
bool  AllowPartialUndefs = false 
)
static

Definition at line 4647 of file X86ISelLowering.cpp.

References llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, E, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), llvm::Constant::getAggregateElement(), llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), getBitWidth(), llvm::APInt::getBoolValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), getTargetConstantFromNode(), llvm::Value::getType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::APInt::insertBits(), llvm::APInt::isAllOnes(), isAnyInRange(), llvm::Type::isDoubleTy(), llvm::Type::isFloatTy(), llvm::Type::isHalfTy(), llvm::Type::isIntegerTy(), llvm::Type::isVectorTy(), llvm::peekThroughBitcasts(), Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), llvm::APInt::setBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST_LOAD, and llvm::X86ISD::VZEXT_MOVL.

Referenced by canonicalizeBitSelect(), combineAnd(), combineAndnp(), combineBitcast(), combineConcatVectorOps(), combineExtractVectorElt(), combineLoad(), combineMOVMSK(), combineOr(), combineOrXorWithSETCC(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftVar(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), convertShiftLeftToScale(), createShuffleMaskFromVSELECT(), getFauxShuffleMask(), getTargetConstantBitsFromNode(), getTargetShuffleAndZeroables(), getTargetShuffleMaskIndices(), llvm::X86::isConstantSplat(), isFNEG(), IsNOT(), LowerVSETCC(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

◆ getTargetConstantFromBasePtr()

static const Constant * getTargetConstantFromBasePtr ( SDValue  Ptr)
static

◆ getTargetConstantFromNode() [1/2]

static const Constant * getTargetConstantFromNode ( LoadSDNode Load)
static

◆ getTargetConstantFromNode() [2/2]

static const Constant * getTargetConstantFromNode ( SDValue  Op)
static

◆ getTargetConstantPoolFromBasePtr()

static ConstantPoolSDNode * getTargetConstantPoolFromBasePtr ( SDValue  Ptr)
static

Definition at line 4615 of file X86ISelLowering.cpp.

References Ptr, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.

Referenced by getTargetConstantFromBasePtr().

◆ getTargetShuffleAndZeroables()

static bool getTargetShuffleAndZeroables ( SDValue  N,
SmallVectorImpl< int > &  Mask,
SmallVectorImpl< SDValue > &  Ops,
APInt KnownUndef,
APInt KnownZero 
)
static

◆ getTargetShuffleInputs() [1/3]

static bool getTargetShuffleInputs ( SDValue  Op,
const APInt DemandedElts,
SmallVectorImpl< SDValue > &  Inputs,
SmallVectorImpl< int > &  Mask,
APInt KnownUndef,
APInt KnownZero,
const SelectionDAG DAG,
unsigned  Depth,
bool  ResolveKnownElts 
)
static

Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.

Returns true if the target shuffle mask was decoded.

Definition at line 6250 of file X86ISelLowering.cpp.

References llvm::Depth, getFauxShuffleMask(), getTargetShuffleAndZeroables(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, resolveTargetShuffleFromZeroables(), and resolveZeroablesFromTargetShuffle().

◆ getTargetShuffleInputs() [2/3]

static bool getTargetShuffleInputs ( SDValue  Op,
const APInt DemandedElts,
SmallVectorImpl< SDValue > &  Inputs,
SmallVectorImpl< int > &  Mask,
const SelectionDAG DAG,
unsigned  Depth,
bool  ResolveKnownElts 
)
static

◆ getTargetShuffleInputs() [3/3]

static bool getTargetShuffleInputs ( SDValue  Op,
SmallVectorImpl< SDValue > &  Inputs,
SmallVectorImpl< int > &  Mask,
const SelectionDAG DAG,
unsigned  Depth = 0,
bool  ResolveKnownElts = true 
)
static

◆ getTargetShuffleMask() [1/2]

static bool getTargetShuffleMask ( SDNode N,
MVT  VT,
bool  AllowSentinelZero,
SmallVectorImpl< SDValue > &  Ops,
SmallVectorImpl< int > &  Mask 
)
static

Definition at line 5462 of file X86ISelLowering.cpp.

References getTargetShuffleMask(), and N.

◆ getTargetShuffleMask() [2/2]

static bool getTargetShuffleMask ( SDNode N,
MVT  VT,
bool  AllowSentinelZero,
SmallVectorImpl< SDValue > &  Ops,
SmallVectorImpl< int > &  Mask,
bool IsUnary 
)
static

Calculates the shuffle mask corresponding to the target-specific opcode.

If the mask could be calculated, returns it in Mask, returns the shuffle operands in Ops, and returns true. Sets IsUnary to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.

Definition at line 5160 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BLENDI, llvm::DecodeBLENDMask(), llvm::DecodeEXTRQIMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeINSERTQIMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVALIGNMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::decodeVSHUF64x2FamilyMask(), llvm::DecodeZeroMoveLowMask(), llvm::SmallVectorBase< Size_T >::empty(), llvm::X86ISD::EXTRQI, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetShuffleMaskIndices(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, isAnyZero(), llvm_unreachable, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, N, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VALIGN, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.

Referenced by canonicalizeShuffleWithOp(), combineConcatVectorOps(), combineSelect(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getPSHUFShuffleMask(), getShuffleScalarElt(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), and llvm::X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode().

◆ getTargetShuffleMaskIndices()

static bool getTargetShuffleMaskIndices ( SDValue  MaskNode,
unsigned  MaskEltSizeInBits,
SmallVectorImpl< uint64_t > &  RawMask,
APInt UndefElts 
)
static

◆ getTargetVShiftByConstNode()

static SDValue getTargetVShiftByConstNode ( unsigned  Opc,
const SDLoc dl,
MVT  VT,
SDValue  SrcOp,
uint64_t  ShiftAmt,
SelectionDAG DAG 
)
static

◆ getTargetVShiftNode()

static SDValue getTargetVShiftNode ( unsigned  Opc,
const SDLoc dl,
MVT  VT,
SDValue  SrcOp,
SDValue  ShAmt,
int  ShAmtIdx,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ getTargetVShiftUniformOpcode()

static unsigned getTargetVShiftUniformOpcode ( unsigned  Opc,
bool  IsVariable 
)
static

◆ GetTLSADDR()

static SDValue GetTLSADDR ( SelectionDAG DAG,
SDValue  Chain,
GlobalAddressSDNode GA,
SDValue InGlue,
const EVT  PtrVT,
unsigned  ReturnReg,
unsigned char  OperandFlags,
bool  LocalDynamic = false 
)
static

◆ getUnderlyingExtractedFromVec()

static int getUnderlyingExtractedFromVec ( SDValue ExtractedFromVec,
SDValue  ExtIdx 
)
static

For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.

Modifies ExtractedFromVec to the real vector and returns the real index.

Definition at line 7447 of file X86ISelLowering.cpp.

References assert(), llvm::SDNode::getAsZExtVal(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), Idx, and isUndefOrInRange().

Referenced by buildFromShuffleMostly().

◆ getUnpackh()

static SDValue getUnpackh ( SelectionDAG DAG,
const SDLoc dl,
EVT  VT,
SDValue  V1,
SDValue  V2 
)
static

◆ getUnpackl()

static SDValue getUnpackl ( SelectionDAG DAG,
const SDLoc dl,
EVT  VT,
SDValue  V1,
SDValue  V2 
)
static

◆ getV4X86ShuffleImm()

static unsigned getV4X86ShuffleImm ( ArrayRef< int >  Mask)
static

Get a 4-lane 8-bit shuffle immediate for a mask.

This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.

NB: We rely heavily on "undef" masks preserving the input lane.

Definition at line 9705 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), and llvm::find_if().

Referenced by combineVectorShiftImm(), getV4X86ShuffleImm8ForMask(), matchBinaryPermuteShuffle(), and matchUnaryPermuteShuffle().

◆ getV4X86ShuffleImm8ForMask()

static SDValue getV4X86ShuffleImm8ForMask ( ArrayRef< int >  Mask,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ getVectorMaskingNode()

static SDValue getVectorMaskingNode ( SDValue  Op,
SDValue  Mask,
SDValue  PreservedSrc,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.

Definition at line 25265 of file X86ISelLowering.cpp.

References getMaskNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), and llvm::ISD::VSELECT.

Referenced by lowerShuffleAsBlend().

◆ getVectorShuffle()

static SDValue getVectorShuffle ( SelectionDAG DAG,
EVT  VT,
const SDLoc dl,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask 
)
static

◆ getVShift()

static SDValue getVShift ( bool  isLeft,
EVT  VT,
SDValue  SrcOp,
unsigned  NumBits,
SelectionDAG DAG,
const TargetLowering TLI,
const SDLoc dl 
)
static

◆ getX86XALUOOp()

static std::pair< SDValue, SDValue > getX86XALUOOp ( X86::CondCode Cond,
SDValue  Op,
SelectionDAG DAG 
)
static

◆ getZeroVector()

static SDValue getZeroVector ( MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static

◆ hasBZHI()

static bool hasBZHI ( const X86Subtarget Subtarget,
MVT  VT 
)
static

Definition at line 48654 of file X86ISelLowering.cpp.

Referenced by combineAndLoadToBZHI().

◆ hasFPCMov()

static bool hasFPCMov ( unsigned  X86CC)
static

Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.

Definition at line 2865 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.

Referenced by combineCMov().

◆ hasIdenticalHalvesShuffleMask()

static bool hasIdenticalHalvesShuffleMask ( ArrayRef< int >  Mask)
static

Return true if a shuffle mask chooses elements identically in its top and bottom halves.

For example, any splat mask has the same top and bottom halves. If an element is undefined in only one half of the mask, the halves are not considered identical.

Definition at line 9687 of file X86ISelLowering.cpp.

References assert().

Referenced by LowerAVXExtend().

◆ hasNonFlagsUse()

static bool hasNonFlagsUse ( SDValue  Op)
static

return true if Op has a use that doesn't just read flags.

Definition at line 22477 of file X86ISelLowering.cpp.

References llvm::ISD::BRCOND, llvm::Value::hasOneUse(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::TRUNCATE, and llvm::Value::use_begin().

◆ incDecVectorConstant()

static SDValue incDecVectorConstant ( SDValue  V,
SelectionDAG DAG,
bool  IsInc,
bool  NSW 
)
static

Given a buildvector constant, return a new vector constant with each element incremented or decremented.

If incrementing or decrementing would result in unsigned overflow or underflow or this is not a simple vector constant, return an empty value.

Definition at line 23070 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMaxValue(), llvm::APInt::isMinSignedValue(), llvm::APInt::isZero(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by combineSetCC(), LowerVSETCC(), and LowerVSETCCWithSUBUS().

◆ insert128BitVector()

static SDValue insert128BitVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static

Generate a DAG to put 128-bits into a vector > 128 bits.

This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering INSERT_VECTOR_ELT operations easier.

Definition at line 3918 of file X86ISelLowering.cpp.

References assert(), llvm::SDValue::getValueType(), insertSubVector(), and llvm::EVT::is128BitVector().

Referenced by LowerSCALAR_TO_VECTOR(), and lowerV4X128Shuffle().

◆ insert1BitVector()

static SDValue insert1BitVector ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ InsertBitToMaskVector()

static SDValue InsertBitToMaskVector ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ insertSubVector()

static SDValue insertSubVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl,
unsigned  vectorWidth 
)
static

◆ is128BitLaneCrossingShuffleMask()

static bool is128BitLaneCrossingShuffleMask ( MVT  VT,
ArrayRef< int >  Mask 
)
static

◆ is128BitLaneRepeatedShuffleMask() [1/2]

static bool is128BitLaneRepeatedShuffleMask ( MVT  VT,
ArrayRef< int >  Mask 
)
static

Definition at line 9433 of file X86ISelLowering.cpp.

References isRepeatedShuffleMask().

◆ is128BitLaneRepeatedShuffleMask() [2/2]

static bool is128BitLaneRepeatedShuffleMask ( MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

◆ is128BitUnpackShuffleMask()

static bool is128BitUnpackShuffleMask ( ArrayRef< int >  Mask,
const SelectionDAG DAG 
)
static

◆ is256BitLaneRepeatedShuffleMask()

static bool is256BitLaneRepeatedShuffleMask ( MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a shuffle mask is equivalent within each 256-bit lane.

Definition at line 9440 of file X86ISelLowering.cpp.

References isRepeatedShuffleMask().

Referenced by lowerV8F64Shuffle(), lowerV8I64Shuffle(), and matchUnaryPermuteShuffle().

◆ isAddSubOrSubAdd() [1/2]

static bool isAddSubOrSubAdd ( const BuildVectorSDNode BV,
const X86Subtarget Subtarget,
SelectionDAG DAG,
SDValue Opnd0,
SDValue Opnd1,
unsigned NumExtracts,
bool IsSubAdd 
)
static

Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation.

If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1 (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters Opnd0 and Opnd1.

Definition at line 7842 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), std::swap(), and llvm::ISD::UNDEF.

Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().

◆ isAddSubOrSubAdd() [2/2]

static bool isAddSubOrSubAdd ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG,
SDValue Opnd0,
SDValue Opnd1,
bool IsSubAdd 
)
static

Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation.

If true is returned then the operands of ADDSUB(SUBADD) operation are written to the parameters Opnd0 and Opnd1.

We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.

Definition at line 40934 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE3(), isAddSubOrSubAddMask(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, and llvm::ISD::VECTOR_SHUFFLE.

◆ isAddSubOrSubAddMask()

static bool isAddSubOrSubAddMask ( ArrayRef< int >  Mask,
bool Op0Even 
)
static

Checks if the shuffle mask takes subsequent elements alternately from two vectors.

For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.

Definition at line 40897 of file X86ISelLowering.cpp.

References Size.

Referenced by combineShuffleToFMAddSub(), and isAddSubOrSubAdd().

◆ isAndOrOfSetCCs()

static bool isAndOrOfSetCCs ( SDValue  Op,
unsigned Opc 
)
static

Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.

Definition at line 24693 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::OR, and llvm::X86ISD::SETCC.

◆ isAnyInRange()

static bool isAnyInRange ( ArrayRef< int >  Mask,
int  Low,
int  Hi 
)
static

Return true if the value of any element in Mask falls within the specified range (L, H].

Definition at line 3518 of file X86ISelLowering.cpp.

References llvm::any_of(), llvm::Hi, isInRange(), and llvm::Low.

Referenced by getTargetConstantBitsFromNode().

◆ isAnyZero()

static bool isAnyZero ( ArrayRef< int >  Mask)
static

Return true if the value of any element in Mask is the zero sentinel value.

Definition at line 3523 of file X86ISelLowering.cpp.

References llvm::any_of(), and llvm::SM_SentinelZero.

Referenced by combineHorizOpWithShuffle(), combineX86ShuffleChain(), getTargetShuffleMask(), isHorizontalBinOp(), matchBinaryPermuteShuffle(), matchShuffleAsByteRotate(), and matchUnaryPermuteShuffle().

◆ isAnyZeroOrUndef()

static bool isAnyZeroOrUndef ( ArrayRef< int >  Mask)
static

Return true if the value of any element in Mask is the zero or undef sentinel values.

Definition at line 3529 of file X86ISelLowering.cpp.

References llvm::any_of(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineSetCCMOVMSK().

◆ isBroadcastShuffleMask()

static bool isBroadcastShuffleMask ( ArrayRef< int >  Mask)
static

◆ isCMOVPseudo()

static bool isCMOVPseudo ( MachineInstr MI)
static

Definition at line 34559 of file X86ISelLowering.cpp.

References MI.

◆ isEFLAGSLiveAfter()

static bool isEFLAGSLiveAfter ( MachineBasicBlock::iterator  Itr,
MachineBasicBlock BB 
)
static

◆ IsElementEquivalent()

static bool IsElementEquivalent ( int  MaskSize,
SDValue  Op,
SDValue  ExpectedOp,
int  Idx,
int  ExpectedIdx 
)
static

◆ isFMAddSubOrFMSubAdd()

static bool isFMAddSubOrFMSubAdd ( const X86Subtarget Subtarget,
SelectionDAG DAG,
SDValue Opnd0,
SDValue Opnd1,
SDValue Opnd2,
unsigned  ExpectedUses 
)
static

Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1).

If (and only if) true is returned, the operands of FMADDSUB/FMSUBADD are written to parameters Opnd0, Opnd1, Opnd2.

Prior to calling this function it should be known that there is some SDNode that potentially can be replaced with an X86ISD::ADDSUB operation using Opnd0 and Opnd1 as operands. Also, this method is called before replacement of such SDNode with ADDSUB operation. Thus the number of Opnd0 uses is expected to be equal to 2. For example, this function may be called for the following IR: AB = fmul fast <2 x double> A, B Sub = fsub fast <2 x double> AB, C Add = fadd fast <2 x double> AB, C Addsub = shufflevector <2 x double> Sub, <2 x double> Add, <2 x i32> <i32 0, i32 3> There is a def for Addsub here, which potentially can be replaced by X86ISD::ADDSUB operation: Addsub = X86ISD::ADDSUB AB, C and such ADDSUB can further be replaced with FMADDSUB: Addsub = FMADDSUB A, B, C.

The main reason why this method is called before the replacement of the recognized ADDSUB idiom with ADDSUB operation is that such replacement is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit FMADDSUB is.

Definition at line 7969 of file X86ISelLowering.cpp.

References llvm::FPOpFusion::Fast, llvm::ISD::FMUL, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNode::hasNUsesOfValue(), llvm::TargetMachine::Options, and Options.

Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().

◆ isFNEG()

static SDValue isFNEG ( SelectionDAG DAG,
SDNode N,
unsigned  Depth = 0 
)
static

Returns the negated value if the node N flips sign of FP value.

FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000) or FSUB(0, x) AVX512F does not have FXOR, so FNEG is lowered as (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). In this case we go though all bitcasts. This also recognizes splat of a negated value and returns the splat of that value.

Definition at line 51819 of file X86ISelLowering.cpp.

References llvm::Depth, E, llvm::ISD::FNEG, llvm::ISD::FSUB, llvm::X86ISD::FXOR, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::INSERT_VECTOR_ELT, isFNEG(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaxRecursionDepth, N, llvm::peekThroughBitcasts(), llvm::SmallVectorBase< Size_T >::size(), std::swap(), llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::XOR.

Referenced by combineFneg(), llvm::X86TargetLowering::getNegatedExpression(), and isFNEG().

◆ isFoldableUseOfShuffle()

static bool isFoldableUseOfShuffle ( SDNode N)
static

◆ isFRClass()

static bool isFRClass ( const TargetRegisterClass RC)
static

Check if RC is a vector register class.

I.e., FR* / VR* or one of their variant.

Definition at line 57577 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

◆ isFreeToSplitVector()

static bool isFreeToSplitVector ( SDNode N,
SelectionDAG DAG 
)
static

◆ isGRClass()

static bool isGRClass ( const TargetRegisterClass RC)
static

Check if RC is a general purpose register class.

I.e., GR* or one of their variant.

Definition at line 57567 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

◆ isHopBuildVector()

static bool isHopBuildVector ( const BuildVectorSDNode BV,
SelectionDAG DAG,
unsigned HOpcode,
SDValue V0,
SDValue V1 
)
static

◆ isHorizontalBinOp()

static bool isHorizontalBinOp ( unsigned  HOpcode,
SDValue LHS,
SDValue RHS,
SelectionDAG DAG,
const X86Subtarget Subtarget,
bool  IsCommutative,
SmallVectorImpl< int > &  PostShuffleMask 
)
static

Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.

A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'.

Definition at line 51073 of file X86ISelLowering.cpp.

References A, llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), B, llvm::sampleprof::Base, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, llvm::SmallVectorImpl< T >::clear(), llvm::ShuffleVectorSDNode::commuteMask(), D, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isMultiLaneShuffleMask(), llvm::isNullConstant(), isSequentialOrUndefInRange(), isUndefOrInRange(), LHS, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleInputsAndMask(), RHS, scaleShuffleElements(), shouldUseHorizontalOp(), llvm::SmallVectorBase< Size_T >::size(), llvm::ArrayRef< T >::slice(), llvm::SM_SentinelUndef, llvm::SelectionDAG::SplitVector(), std::swap(), and llvm::SDNode::uses().

Referenced by combineToHorizontalAddSub().

◆ isHorizontalBinOpPart()

static bool isHorizontalBinOpPart ( const BuildVectorSDNode N,
unsigned  Opcode,
const SDLoc DL,
SelectionDAG DAG,
unsigned  BaseIdx,
unsigned  LastIdx,
SDValue V0,
SDValue V1 
)
static

This is a helper function of LowerToHorizontalOp().

This function checks that the build_vector N in input implements a 128-bit partial horizontal operation on a 256-bit vector, but that operation may not match the layout of an x86 256-bit horizontal instruction. In other words, if this returns true, then some extraction/insertion will be required to produce a valid horizontal instruction.

Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.

This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).

TODO: This function was originally used to match both real and fake partial horizontal operations, but the index-matching logic is incorrect for that. See the corrected implementation in isHopBuildVector(). Can we reduce this code because it is only used for partial h-op matching now?

Definition at line 7687 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), and N.

Referenced by LowerToHorizontalOp().

◆ isHorizOp()

static LLVM_ATTRIBUTE_UNUSED bool isHorizOp ( unsigned  Opcode)
static

◆ isInRange()

static bool isInRange ( int  Val,
int  Low,
int  Hi 
)
static

◆ isLaneCrossingShuffleMask()

static bool isLaneCrossingShuffleMask ( unsigned  LaneSizeInBits,
unsigned  ScalarSizeInBits,
ArrayRef< int >  Mask 
)
static

Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.

X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.

Definition at line 9337 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by is128BitLaneCrossingShuffleMask().

◆ isLegalConversion()

static bool isLegalConversion ( MVT  VT,
bool  IsSigned,
const X86Subtarget Subtarget 
)
static

◆ isMultiLaneShuffleMask()

static bool isMultiLaneShuffleMask ( unsigned  LaneSizeInBits,
unsigned  ScalarSizeInBits,
ArrayRef< int >  Mask 
)
static

Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles.

Definition at line 9360 of file X86ISelLowering.cpp.

References assert().

Referenced by isHorizontalBinOp().

◆ isNonZeroElementsInOrder()

static bool isNonZeroElementsInOrder ( const APInt Zeroable,
ArrayRef< int >  Mask,
const EVT VectorType,
bool IsZeroSideLeft 
)
static

Definition at line 9741 of file X86ISelLowering.cpp.

References assert().

Referenced by lowerShuffleToEXPAND().

◆ isNoopOrBroadcastShuffleMask()

static bool isNoopOrBroadcastShuffleMask ( ArrayRef< int >  Mask)
static

◆ isNoopShuffleMask()

static bool isNoopShuffleMask ( ArrayRef< int >  Mask)
static

Tiny helper function to identify a no-op mask.

This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.

Definition at line 9323 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by combineTargetShuffle(), isNoopOrBroadcastShuffleMask(), lowerShuffleAsByteRotate(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerV8I16GeneralSingleInputShuffle(), and matchShuffleAsBitRotate().

◆ IsNOT()

static SDValue IsNOT ( SDValue  V,
SelectionDAG DAG 
)
static

◆ isNullFPScalarOrVectorConst()

static bool isNullFPScalarOrVectorConst ( SDValue  V)
static

◆ isOrXorXorTree()

static bool isOrXorXorTree ( SDValue  X,
bool  Root = true 
)
static

Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion.

Definition at line 21943 of file X86ISelLowering.cpp.

References isOrXorXorTree(), llvm::ISD::OR, X, and llvm::ISD::XOR.

Referenced by combineVectorSizedSetCCEquality(), and isOrXorXorTree().

◆ isProfitableToUseFlagOp()

static bool isProfitableToUseFlagOp ( SDValue  Op)
static

Definition at line 22498 of file X86ISelLowering.cpp.

References llvm::ISD::CopyToReg, llvm::ISD::SETCC, and llvm::ISD::STORE.

Referenced by EmitTest().

◆ isRepeatedShuffleMask()

static bool isRepeatedShuffleMask ( unsigned  LaneSizeInBits,
MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a shuffle mask is equivalent within each sub-lane.

This checks a shuffle mask to see if it is performing the same lane-relative shuffle in each sub-lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.

The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is suitable for use with existing 128-bit shuffles as entries from the second vector have been remapped to [LaneSize, 2*LaneSize).

Definition at line 9397 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), Size, and llvm::SM_SentinelUndef.

Referenced by is128BitLaneRepeatedShuffleMask(), and is256BitLaneRepeatedShuffleMask().

◆ isRepeatedTargetShuffleMask() [1/2]

static bool isRepeatedTargetShuffleMask ( unsigned  LaneSizeInBits,
MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a target shuffle mask is equivalent within each sub-lane.

Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.

Definition at line 9484 of file X86ISelLowering.cpp.

References llvm::MVT::getScalarSizeInBits(), and isRepeatedTargetShuffleMask().

◆ isRepeatedTargetShuffleMask() [2/2]

static bool isRepeatedTargetShuffleMask ( unsigned  LaneSizeInBits,
unsigned  EltSizeInBits,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a target shuffle mask is equivalent within each sub-lane.

Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.

Definition at line 9447 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), isUndefOrZero(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canonicalizeShuffleMaskWithHorizOp(), isRepeatedTargetShuffleMask(), and matchBinaryPermuteShuffle().

◆ isSequentialOrUndefInRange()

static bool isSequentialOrUndefInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size,
int  Low,
int  Step = 1 
)
static

◆ isSequentialOrUndefOrZeroInRange()

static bool isSequentialOrUndefOrZeroInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size,
int  Low,
int  Step = 1 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.

Definition at line 3575 of file X86ISelLowering.cpp.

References isUndefOrZero(), llvm::Low, and Size.

Referenced by combineX86ShuffleChain(), and matchShuffleWithUNPCK().

◆ isShuffleEquivalent()

static bool isShuffleEquivalent ( ArrayRef< int >  Mask,
ArrayRef< int >  ExpectedMask,
SDValue  V1 = SDValue(),
SDValue  V2 = SDValue() 
)
static

Checks whether a shuffle mask is equivalent to an explicit list of arguments.

This is a fast way to test a shuffle mask against a fixed pattern:

if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }

It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.

Definition at line 9553 of file X86ISelLowering.cpp.

References assert(), IsElementEquivalent(), llvm::ArrayRef< T >::size(), and Size.

Referenced by lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), lowerV16F32Shuffle(), lowerV2F64Shuffle(), lowerV2X128Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4X128Shuffle(), lowerV8F32Shuffle(), and lowerV8F64Shuffle().

◆ isShuffleFoldableLoad()

static bool isShuffleFoldableLoad ( SDValue  V)
static

Helper to test for a load that can be folded with x86 shuffles.

This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.

Definition at line 12141 of file X86ISelLowering.cpp.

References llvm::ISD::isNON_EXTLoad(), and llvm::peekThroughOneUseBitcasts().

Referenced by canonicalizeShuffleWithOp(), and lowerShuffleAsBroadcast().

◆ isShuffleMaskInputInPlace()

static bool isShuffleMaskInputInPlace ( int  Input,
ArrayRef< int >  Mask 
)
static

Test whether the specified input (0 or 1) is in-place blended by the given mask.

This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.

Definition at line 12358 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by lowerV4F64Shuffle(), and lowerV4I64Shuffle().

◆ isSingleElementRepeatedMask()

static bool isSingleElementRepeatedMask ( ArrayRef< int >  Mask)
static

Check if the Mask consists of the same element repeated multiple times.

Definition at line 11072 of file X86ISelLowering.cpp.

References llvm::SM_SentinelUndef.

Referenced by lowerShuffleAsDecomposedShuffleMerge().

◆ isSingleSHUFPSMask()

static bool isSingleSHUFPSMask ( ArrayRef< int >  Mask)
static

Test whether this can be lowered with a single SHUFPS instruction.

This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.

Definition at line 12335 of file X86ISelLowering.cpp.

References assert().

Referenced by lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUndefHalf(), lowerV16I32Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), and lowerV8I32Shuffle().

◆ isSoftF16()

template<typename T >
static bool isSoftF16 ( T  VT,
const X86Subtarget Subtarget 
)
static

◆ isTargetShuffle()

static bool isTargetShuffle ( unsigned  Opcode)
static

◆ isTargetShuffleEquivalent()

static bool isTargetShuffleEquivalent ( MVT  VT,
ArrayRef< int >  Mask,
ArrayRef< int >  ExpectedMask,
const SelectionDAG DAG,
SDValue  V1 = SDValue(),
SDValue  V2 = SDValue() 
)
static

Checks whether a target shuffle mask is equivalent to an explicit pattern.

The masks must be exactly the same width.

If an element in Mask matches SM_SentinelUndef (-1) then the corresponding value in ExpectedMask is always accepted. Otherwise the indices must match.

SM_SentinelZero is accepted as a valid negative index but must match in both, or via a known bits test.

Definition at line 9585 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), IsElementEquivalent(), isInRange(), isUndefOrZeroOrInRange(), llvm::APInt::isZero(), llvm::SelectionDAG::MaskedVectorIsZero(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineX86ShuffleChain(), is128BitUnpackShuffleMask(), isUnpackWdShuffleMask(), matchBinaryShuffle(), matchShuffleWithPACK(), matchShuffleWithUNPCK(), and matchUnaryShuffle().

◆ isTargetShuffleVariableMask()

static bool isTargetShuffleVariableMask ( unsigned  Opcode)
static

◆ isTruncWithZeroHighBitsInput()

static bool isTruncWithZeroHighBitsInput ( SDValue  V,
SelectionDAG DAG 
)
static

◆ isUnaryOp()

static bool isUnaryOp ( unsigned  Opcode)
static

Definition at line 39934 of file X86ISelLowering.cpp.

References llvm::ISD::CTLZ, llvm::ISD::CTPOP, and llvm::ISD::CTTZ.

Referenced by canonicalizeShuffleWithOp().

◆ isUndefInRange()

static bool isUndefInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.

Definition at line 3494 of file X86ISelLowering.cpp.

References llvm::all_of(), and Size.

Referenced by canonicalizeShuffleMaskWithHorizOp(), isUndefLowerHalf(), isUndefUpperHalf(), lowerShuffleAsVTRUNC(), lowerShuffleWithVPMOV(), matchBinaryPermuteShuffle(), matchShuffleAsINSERTQ(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

◆ isUndefLowerHalf()

static bool isUndefLowerHalf ( ArrayRef< int >  Mask)
static

Return true if the mask creates a vector whose lower half is undefined.

Definition at line 3500 of file X86ISelLowering.cpp.

References isUndefInRange().

Referenced by getHalfShuffleMask(), and lowerShuffleWithUndefHalf().

◆ isUndefOrEqual() [1/2]

static bool isUndefOrEqual ( ArrayRef< int >  Mask,
int  CmpVal 
)
static

Return true if every element in Mask is the undef sentinel value or equal to the specified value.

Definition at line 3473 of file X86ISelLowering.cpp.

References llvm::all_of(), and llvm::SM_SentinelUndef.

◆ isUndefOrEqual() [2/2]

static bool isUndefOrEqual ( int  Val,
int  CmpVal 
)
static

◆ isUndefOrEqualInRange()

static bool isUndefOrEqualInRange ( ArrayRef< int >  Mask,
int  CmpVal,
unsigned  Pos,
unsigned  Size 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value.

Definition at line 3481 of file X86ISelLowering.cpp.

References llvm::all_of(), and Size.

Referenced by matchUnaryShuffle().

◆ isUndefOrInRange() [1/2]

static bool isUndefOrInRange ( ArrayRef< int >  Mask,
int  Low,
int  Hi 
)
static

Return true if every element in Mask is undef or if its value falls within the specified range (L, H].

Definition at line 3543 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::Hi, isUndefOrInRange(), and llvm::Low.

◆ isUndefOrInRange() [2/2]

static bool isUndefOrInRange ( int  Val,
int  Low,
int  Hi 
)
static

◆ isUndefOrZero()

static bool isUndefOrZero ( int  Val)
static

◆ isUndefOrZeroInRange()

static bool isUndefOrZeroInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.

Definition at line 3586 of file X86ISelLowering.cpp.

References llvm::all_of(), isUndefOrZero(), and Size.

Referenced by combineExtractWithShuffle(), combineX86ShuffleChain(), matchBinaryPermuteShuffle(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

◆ isUndefOrZeroOrInRange() [1/2]

static bool isUndefOrZeroOrInRange ( ArrayRef< int >  Mask,
int  Low,
int  Hi 
)
static

Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].

Definition at line 3556 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::Hi, isUndefOrZeroOrInRange(), and llvm::Low.

◆ isUndefOrZeroOrInRange() [2/2]

static bool isUndefOrZeroOrInRange ( int  Val,
int  Low,
int  Hi 
)
static

Return true if Val is undef, zero or if its value falls within the specified range (L, H].

Definition at line 3550 of file X86ISelLowering.cpp.

References llvm::Hi, isInRange(), isUndefOrZero(), and llvm::Low.

Referenced by canonicalizeShuffleMaskWithHorizOp(), isTargetShuffleEquivalent(), isUndefOrZeroOrInRange(), and matchShuffleWithSHUFPD().

◆ isUndefUpperHalf()

static bool isUndefUpperHalf ( ArrayRef< int >  Mask)
static

Return true if the mask creates a vector whose upper half is undefined.

Definition at line 3506 of file X86ISelLowering.cpp.

References isUndefInRange().

Referenced by getHalfShuffleMask(), lowerShuffleAsSpecificZeroOrAnyExtend(), lowerShuffleWithUndefHalf(), matchShuffleAsEXTRQ(), matchShuffleAsINSERTQ(), and narrowShuffle().

◆ isUnpackWdShuffleMask()

static bool isUnpackWdShuffleMask ( ArrayRef< int >  Mask,
MVT  VT,
const SelectionDAG DAG 
)
static

◆ isUpperSubvectorUndef()

static SDValue isUpperSubvectorUndef ( SDValue  V,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ isVKClass()

static bool isVKClass ( const TargetRegisterClass RC)
static

Check if RC is a mask register class.

I.e., VK* or one of their variant.

Definition at line 57588 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

◆ isX86CCSigned()

static bool isX86CCSigned ( unsigned  X86CC)
static

◆ isX86LogicalCmp()

static bool isX86LogicalCmp ( SDValue  Op)
static

◆ lower128BitShuffle()

static SDValue lower128BitShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Dispatching routine to lower various 128-bit x86 vector shuffles.

This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.

Definition at line 14443 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getVectorShuffle(), llvm_unreachable, lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV8F16Shuffle(), lowerV8I16Shuffle(), and llvm::MVT::SimpleTy.

Referenced by lowerVECTOR_SHUFFLE().

◆ lower1BitShuffle()

static SDValue lower1BitShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lower1BitShuffleAsKSHIFTR()

static SDValue lower1BitShuffleAsKSHIFTR ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lower256BitShuffle()

static SDValue lower256BitShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lower512BitShuffle()

static SDValue lower512BitShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

High-level routine to lower various 512-bit x86 vector shuffles.

This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.

Definition at line 17142 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX512(), llvm_unreachable, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleWithUndefHalf(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV32I16Shuffle(), lowerV64I8Shuffle(), lowerV8F64Shuffle(), lowerV8I64Shuffle(), llvm::MVT::SimpleTy, and splitAndLowerShuffle().

Referenced by lowerVECTOR_SHUFFLE().

◆ LowerABD()

static SDValue LowerABD ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerABS()

static SDValue LowerABS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerADDRSPACECAST()

static SDValue LowerADDRSPACECAST ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerADDSAT_SUBSAT()

static SDValue LowerADDSAT_SUBSAT ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerAddSub()

static SDValue lowerAddSub ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerADDSUBO_CARRY()

static SDValue LowerADDSUBO_CARRY ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ lowerAddSubToHorizontalOp()

static SDValue lowerAddSubToHorizontalOp ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerADJUST_TRAMPOLINE()

static SDValue LowerADJUST_TRAMPOLINE ( SDValue  Op,
SelectionDAG DAG 
)
static

Definition at line 27373 of file X86ISelLowering.cpp.

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ LowerAndToBT()

static SDValue LowerAndToBT ( SDValue  And,
ISD::CondCode  CC,
const SDLoc dl,
SelectionDAG DAG,
X86::CondCode X86CC 
)
static

◆ LowerANY_EXTEND()

static SDValue LowerANY_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerAsSplatVectorLoad()

static SDValue LowerAsSplatVectorLoad ( SDValue  SrcOp,
MVT  VT,
const SDLoc dl,
SelectionDAG DAG 
)
static

◆ LowerATOMIC_FENCE()

static SDValue LowerATOMIC_FENCE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerATOMIC_STORE()

static SDValue LowerATOMIC_STORE ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerAtomicArith()

static SDValue lowerAtomicArith ( SDValue  N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerAtomicArithWithLOCK()

static SDValue lowerAtomicArithWithLOCK ( SDValue  N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerAVG()

static SDValue LowerAVG ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerAVXCONCAT_VECTORS()

static SDValue LowerAVXCONCAT_VECTORS ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerAVXExtend()

static SDValue LowerAVXExtend ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerBITCAST()

static SDValue LowerBITCAST ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerBITREVERSE()

static SDValue LowerBITREVERSE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerBITREVERSE_XOP()

static SDValue LowerBITREVERSE_XOP ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerBUILD_VECTORAsVariablePermute()

static SDValue LowerBUILD_VECTORAsVariablePermute ( SDValue  V,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerBUILD_VECTORvXbf16()

static SDValue LowerBUILD_VECTORvXbf16 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerBUILD_VECTORvXi1()

static SDValue LowerBUILD_VECTORvXi1 ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerBuildVectorAsBroadcast()

static SDValue lowerBuildVectorAsBroadcast ( BuildVectorSDNode BVOp,
const SDLoc dl,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.

A single scalar load, or a constant. b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).

The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.

Definition at line 7208 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::ISD::BITCAST, llvm::CallingConv::C, llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::BitVector::count(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::TargetLoweringBase::getPointerTy(), llvm::BuildVectorSDNode::getRepeatedSequence(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasNUsesOfValue(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::BuildVectorSDNode::isConstantSplat(), isFoldableUseOfShuffle(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isVector(), llvm::MachineMemOperand::MOLoad, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SelectionDAG::shouldOptForSize(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VBROADCASTM, and llvm::ISD::ZERO_EXTEND.

◆ LowerBuildVectorAsInsert()

static SDValue LowerBuildVectorAsInsert ( SDValue  Op,
const SDLoc DL,
const APInt NonZeroMask,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerBuildVectorToBitOp()

static SDValue lowerBuildVectorToBitOp ( BuildVectorSDNode Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.

NOTE: Its not in our interest to start make a general purpose vectorizer from this, but enough scalar bit operations are created from the later legalization + scalarization stages to need basic support.

Definition at line 8288 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::any_of(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), LHS, LowerShift(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::XOR.

◆ LowerBuildVectorv16i8()

static SDValue LowerBuildVectorv16i8 ( SDValue  Op,
const SDLoc DL,
const APInt NonZeroMask,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerBuildVectorv4x32()

static SDValue LowerBuildVectorv4x32 ( SDValue  Op,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerBuildVectorv8i16()

static SDValue LowerBuildVectorv8i16 ( SDValue  Op,
const SDLoc DL,
const APInt NonZeroMask,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Custom lower build_vector of v8i16.

Definition at line 6552 of file X86ISelLowering.cpp.

References DL, llvm::X86Subtarget::hasSSE41(), and LowerBuildVectorAsInsert().

◆ LowerCMP_SWAP()

static SDValue LowerCMP_SWAP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerCONCAT_VECTORS()

static SDValue LowerCONCAT_VECTORS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerCONCAT_VECTORSvXi1()

static SDValue LowerCONCAT_VECTORSvXi1 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerCTLZ()

static SDValue LowerCTLZ ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerCTPOP()

static SDValue LowerCTPOP ( SDValue  N,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerCTTZ()

static SDValue LowerCTTZ ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerCVTPS2PH()

static SDValue LowerCVTPS2PH ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerEXTEND_VECTOR_INREG()

static SDValue LowerEXTEND_VECTOR_INREG ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerEXTRACT_SUBVECTOR()

static SDValue LowerEXTRACT_SUBVECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerEXTRACT_VECTOR_ELT_SSE4()

static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerFABSorFNEG()

static SDValue LowerFABSorFNEG ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerFCOPYSIGN()

static SDValue LowerFCOPYSIGN ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerFGETSIGN()

static SDValue LowerFGETSIGN ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerFMINIMUM_FMAXIMUM()

static SDValue LowerFMINIMUM_FMAXIMUM ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerFP16_TO_FP()

static SDValue LowerFP16_TO_FP ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerFP_TO_FP16()

static SDValue LowerFP_TO_FP16 ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ lowerFPToIntToFP()

static SDValue lowerFPToIntToFP ( SDValue  CastToFP,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerFROUND()

static SDValue LowerFROUND ( SDValue  Op,
SelectionDAG DAG 
)
static

ISD::FROUND is defined to round to nearest with ties rounding away from 0.

This mode isn't supported in hardware on X86. But as long as we aren't compiling with trapping math, we can emulate this with trunc(X + copysign(nextafter(0.5, 0.0), X)).

Definition at line 21723 of file X86ISelLowering.cpp.

References llvm::APFloat::convert(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::FADD, llvm::ISD::FCOPYSIGN, llvm::ISD::FTRUNC, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::APFloat::next(), and llvm::APFloatBase::rmNearestTiesToEven.

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ LowerFSINCOS()

static SDValue LowerFSINCOS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerFunnelShift()

static SDValue LowerFunnelShift ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerHorizontalByteSum()

static SDValue LowerHorizontalByteSum ( SDValue  V,
MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Compute the horizontal sum of bytes in V for the elements of VT.

Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.

Definition at line 31005 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), High, llvm::Low, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by LowerVectorCTPOP().

◆ LowerI64IntToFP16()

static SDValue LowerI64IntToFP16 ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerI64IntToFP_AVX512DQ()

static SDValue LowerI64IntToFP_AVX512DQ ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerINSERT_SUBVECTOR()

static SDValue LowerINSERT_SUBVECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 18378 of file X86ISelLowering.cpp.

References assert(), and insert1BitVector().

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ lowerINT_TO_FP_vXi64()

static SDValue lowerINT_TO_FP_vXi64 ( SDValue  Op,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerINTRINSIC_W_CHAIN()

static SDValue LowerINTRINSIC_W_CHAIN ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 26681 of file X86ISelLowering.cpp.

References llvm::X86ISD::AADD, llvm::X86ISD::AAND, Addr, llvm::X86ISD::AESDEC128KL, llvm::X86ISD::AESDEC256KL, llvm::X86ISD::AESDECWIDE128KL, llvm::X86ISD::AESDECWIDE256KL, llvm::X86ISD::AESENC128KL, llvm::X86ISD::AESENC256KL, llvm::X86ISD::AESENCWIDE128KL, llvm::X86ISD::AESENCWIDE256KL, llvm::X86ISD::AOR, assert(), llvm::X86ISD::AXOR, llvm::sampleprof::Base, CC, llvm::X86ISD::CMOV, llvm::X86ISD::CMPCCXADD, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateStackObject(), DL, EmitMaskedTruncSStore(), EmitTruncSStore(), llvm::X86ISD::ENQCMD, llvm::X86ISD::ENQCMDS, expandIntrinsicWChainHelper(), llvm::GATHER, llvm::GATHER_AVX2, llvm::SDNode::getAsZExtVal(), getAVX2GatherNode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMaskedStore(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPrefetchNode(), getReadTimeStampCounter(), llvm::MVT::getScalarSizeInBits(), getScatterNode(), getSETCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTruncStore(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::isAllOnesConstant(), llvm::X86::isExtendedSwiftAsyncFrameSupported(), isValid(), llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm::X86ISD::LBTC, llvm::X86ISD::LBTC_RM, llvm::X86ISD::LBTR, llvm::X86ISD::LBTR_RM, llvm::X86ISD::LBTS, llvm::X86ISD::LBTS_RM, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LWPINS, llvm::X86ISD::LXOR, MarkEHGuard(), MarkEHRegistrationNode(), llvm::ISD::MERGE_VALUES, llvm::Offset, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, Operation, llvm::PREFETCH, llvm::X86ISD::RDPKRU, llvm::RDPMC, llvm::RDPRU, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, Results, llvm::SCATTER, llvm::MachineFrameInfo::setAdjustsStack(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment(), llvm::ISD::SHL, Size, llvm::X86ISD::TESTUI, llvm::X86ISD::TPAUSE, llvm::TRUNCATE_TO_MEM_VI16, llvm::TRUNCATE_TO_MEM_VI32, llvm::TRUNCATE_TO_MEM_VI8, llvm::IntrinsicData::Type, llvm::X86ISD::UMWAIT, llvm::ISD::UNINDEXED, llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, llvm::X86ISD::WRPKRU, llvm::XGETBV, llvm::XTEST, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ LowerIntVSETCC_AVX512()

static SDValue LowerIntVSETCC_AVX512 ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerLoad()

static SDValue LowerLoad ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMGATHER()

static SDValue LowerMGATHER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMINMAX()

static SDValue LowerMINMAX ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMLOAD()

static SDValue LowerMLOAD ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMSCATTER()

static SDValue LowerMSCATTER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMSTORE()

static SDValue LowerMSTORE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMUL()

static SDValue LowerMUL ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMULH()

static SDValue LowerMULH ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerMULO()

static SDValue LowerMULO ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerPARITY()

static SDValue LowerPARITY ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerPREFETCH()

static SDValue LowerPREFETCH ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 32012 of file X86ISelLowering.cpp.

◆ LowerREADCYCLECOUNTER()

static SDValue LowerREADCYCLECOUNTER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerRotate()

static SDValue LowerRotate ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 29966 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::X86ISD::BLENDV, llvm::CallingConv::C, convertShiftLeftToScale(), DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getSplatSourceVector(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::X86::isConstantSplat(), llvm::SelectionDAG::isSplatValue(), llvm::MVT::isVector(), llvm::Lo, llvm::ISD::MUL, llvm::ISD::MULHU, llvm::ISD::OR, llvm::X86ISD::PCMPGT, llvm::X86ISD::PMULUDQ, llvm::ISD::ROTL, llvm::ISD::SHL, splitVectorIntBinary(), llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithImm(), supportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::APInt::urem(), llvm::X86Subtarget::useBWIRegs(), useVPTERNLOG(), llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ LowerSCALAR_TO_VECTOR()

static SDValue LowerSCALAR_TO_VECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerShift()

static SDValue LowerShift ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 29302 of file X86ISelLowering.cpp.

References A, llvm::ISD::ADD, assert(), llvm::X86ISD::BLENDV, llvm::CallingConv::C, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), canWidenShuffleElements(), convertShiftLeftToScale(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtOrTrunc(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isKnownNeverZero(), llvm::MVT::isVector(), llvm::Lo, LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::ISD::OR, llvm::X86ISD::PACKUS, llvm::X86ISD::PCMPGT, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), R2, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, splitVectorIntBinary(), llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::X86ISD::VPSHA, llvm::X86ISD::VPSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

◆ LowerShiftByScalarImmediate()

static SDValue LowerShiftByScalarImmediate ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerShiftByScalarVariable()

static SDValue LowerShiftByScalarVariable ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerShiftParts()

static SDValue LowerShiftParts ( SDValue  Op,
SelectionDAG DAG 
)
static

Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.

TODO: Can this be moved to general expansion code?

Definition at line 18913 of file X86ISelLowering.cpp.

References llvm::TargetLowering::expandShiftParts(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Hi, and llvm::Lo.

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ lowerShuffleAsBitBlend()

static SDValue lowerShuffleAsBitBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Try to emit a blend instruction for a shuffle using bit math.

This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.

Definition at line 10469 of file X86ISelLowering.cpp.

References llvm::AllOnes, assert(), DL, llvm::SelectionDAG::getAllOnesConstant(), getBitSelect(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getVectorElementType(), llvm::MVT::isInteger(), and Size.

Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBlend(), lowerV16I8Shuffle(), and lowerV8I16Shuffle().

◆ lowerShuffleAsBitMask()

static SDValue lowerShuffleAsBitMask ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsBitRotate()

static SDValue lowerShuffleAsBitRotate ( const SDLoc DL,
MVT  VT,
SDValue  V1,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsBlend()

static SDValue lowerShuffleAsBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Original,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to emit a blend instruction for a shuffle.

This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is a blend, or convertible into a blend with zero.

Definition at line 10585 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BLENDI, llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, is128BitLaneRepeatedShuffleMask(), llvm::ISD::isNormalLoad(), llvm_unreachable, llvm::Lo, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), matchShuffleAsBlend(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::shouldOptForSize(), llvm::MVT::SimpleTy, llvm::SmallVectorBase< Size_T >::size(), Size, and std::swap().

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV2X128Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

◆ lowerShuffleAsBlendAndPermute()

static SDValue lowerShuffleAsBlendAndPermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG,
bool  ImmBlends = false 
)
static

Try to lower as a blend of elements from two inputs followed by a single-input permutation.

This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.

Definition at line 10746 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), DL, llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), and Size.

Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV4F32Shuffle(), and lowerV4I64Shuffle().

◆ lowerShuffleAsBlendOfPSHUFBs()

static SDValue lowerShuffleAsBlendOfPSHUFBs ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
SelectionDAG DAG,
bool V1InUse,
bool V2InUse 
)
static

◆ lowerShuffleAsBroadcast()

static SDValue lowerShuffleAsBroadcast ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower broadcast of a single element.

For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out.

Definition at line 12425 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::getSplatIndex(), llvm::MVT::getStoreSize(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), llvm::ISD::isNormalLoad(), isShuffleFoldableLoad(), lowerShuffleAsTruncBroadcast(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::X86ISD::MOVDDUP, llvm::Offset, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by lower512BitShuffle(), lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV8F16Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerVECTOR_SHUFFLE().

◆ lowerShuffleAsByteRotate()

static SDValue lowerShuffleAsByteRotate ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsByteRotateAndPermute()

static SDValue lowerShuffleAsByteRotateAndPermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsByteShiftMask()

static SDValue lowerShuffleAsByteShiftMask ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsDecomposedShuffleMerge()

static SDValue lowerShuffleAsDecomposedShuffleMerge ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsElementInsertion()

static SDValue lowerShuffleAsElementInsertion ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsInsertPS()

static SDValue lowerShuffleAsInsertPS ( const SDLoc DL,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsLanePermuteAndPermute()

static SDValue lowerShuffleAsLanePermuteAndPermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.

This is mainly for cases where we can have non-repeating permutes in each lane.

TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask, we should investigate merging them.

Attempts to find a sublane permute with the given size that gets all elements into their target lanes.

If successful, fills CrossLaneMask and InLaneMask and returns true. If unsuccessful, returns false and may overwrite InLaneMask.

Definition at line 14700 of file X86ISelLowering.cpp.

References DL, llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), isSequentialOrUndefInRange(), isUndefOrEqual(), llvm::narrowShuffleMaskElts(), and llvm::SM_SentinelUndef.

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), and lowerV64I8Shuffle().

◆ lowerShuffleAsLanePermuteAndRepeatedMask()

static SDValue lowerShuffleAsLanePermuteAndRepeatedMask ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.

This attempts to create a repeated lane shuffle where each lane uses one or two of the lanes of the inputs. The lanes of the input vectors are shuffled in one or two independent shuffles to get the lanes into the position needed by the final shuffle.

Definition at line 15002 of file X86ISelLowering.cpp.

References assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getVectorShuffle(), is128BitLaneRepeatedShuffleMask(), llvm::M1(), and std::swap().

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

◆ lowerShuffleAsLanePermuteAndShuffle()

static SDValue lowerShuffleAsLanePermuteAndShuffle ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation.

This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.

Definition at line 14823 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), computeInLaneShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::isFloatingPoint(), lowerShuffleAsLanePermuteAndSHUFP(), Size, and splitAndLowerShuffle().

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), and lowerV8F32Shuffle().

◆ lowerShuffleAsLanePermuteAndSHUFP()

static SDValue lowerShuffleAsLanePermuteAndSHUFP ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsPermuteAndUnpack()

static SDValue lowerShuffleAsPermuteAndUnpack ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.

This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.

Definition at line 10868 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), Size, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV16I8Shuffle(), lowerV4I32Shuffle(), and lowerV8I16Shuffle().

◆ lowerShuffleAsRepeatedMaskAndLanePermute()

static SDValue lowerShuffleAsRepeatedMaskAndLanePermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.

Definition at line 15377 of file X86ISelLowering.cpp.

References assert(), DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), Idx, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), isUndefOrInRange(), llvm::M1(), and llvm::SM_SentinelUndef.

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

◆ lowerShuffleAsShift()

static SDValue lowerShuffleAsShift ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG,
bool  BitwiseOnly 
)
static

◆ lowerShuffleAsSpecificZeroOrAnyExtend()

static SDValue lowerShuffleAsSpecificZeroOrAnyExtend ( const SDLoc DL,
MVT  VT,
int  Scale,
int  Offset,
bool  AnyExt,
SDValue  InputV,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lower a vector shuffle as a zero or any extension.

Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget. The extended elements are consecutive and begin and can start from an offsetted element index in the input; to avoid excess shuffling the offset must either being in the bottom lane or at the start of a higher lane. All extended elements must be from the same lane.

Definition at line 11830 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getEXTEND_VECTOR_INREG(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::MVT::is128BitVector(), isUndefUpperHalf(), llvm::Lo, llvm::Offset, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::ISD::ZERO_EXTEND.

Referenced by lowerShuffleAsZeroOrAnyExtend().

◆ lowerShuffleAsSplitOrBlend()

static SDValue lowerShuffleAsSplitOrBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Either split a vector in halves or decompose the shuffles and the blend/unpack.

This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.

Definition at line 14610 of file X86ISelLowering.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::count(), DL, llvm::MVT::getSizeInBits(), lowerShuffleAsDecomposedShuffleMerge(), llvm::SmallBitVector::resize(), Size, and splitAndLowerShuffle().

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

◆ lowerShuffleAsTruncBroadcast()

static SDValue lowerShuffleAsTruncBroadcast ( const SDLoc DL,
MVT  VT,
SDValue  V0,
int  BroadcastIdx,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsUNPCKAndPermute()

static SDValue lowerShuffleAsUNPCKAndPermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Try to lower as an unpack of elements from two inputs followed by a single-input permutation.

This matches the pattern where we can unpack elements from two inputs and then reduce the shuffle to a single-input (wider) permutation.

Definition at line 10785 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::Hi, isUndefOrInRange(), llvm::Lo, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerShuffleAsDecomposedShuffleMerge().

◆ lowerShuffleAsVALIGN()

static SDValue lowerShuffleAsVALIGN ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a dword/qword rotation.

AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary rotation of the concatenation of two vectors; This routine will try to generically lower a vector shuffle through such an pattern.

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 11442 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::countl_one(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isSequentialOrUndefInRange(), llvm::Lo, llvm::Low, matchShuffleAsElementRotate(), and llvm::X86ISD::VALIGN.

Referenced by lowerV16I32Shuffle(), lowerV2I64Shuffle(), lowerV4I64Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

◆ lowerShuffleAsVTRUNC()

static SDValue lowerShuffleAsVTRUNC ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsVTRUNCAndUnpack()

static SDValue lowerShuffleAsVTRUNCAndUnpack ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
SelectionDAG DAG 
)
static

◆ lowerShuffleAsZeroOrAnyExtend()

static SDValue lowerShuffleAsZeroOrAnyExtend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a zero extension on any microarch.

This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).

The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.

Definition at line 11994 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), isSequentialOrUndefInRange(), llvm::Lower, lowerShuffleAsSpecificZeroOrAnyExtend(), llvm::Offset, and llvm::X86ISD::VZEXT_MOVL.

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().

◆ lowerShuffleOfExtractsAsVperm()

static SDValue lowerShuffleOfExtractsAsVperm ( const SDLoc DL,
SDValue  N0,
SDValue  N1,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

◆ lowerShufflePairAsUNPCKAndPermute()

static SDValue lowerShufflePairAsUNPCKAndPermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

◆ lowerShuffleToEXPAND()

static SDValue lowerShuffleToEXPAND ( const SDLoc DL,
MVT  VT,
const APInt Zeroable,
ArrayRef< int >  Mask,
SDValue V1,
SDValue V2,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerShuffleWithPACK()

static SDValue lowerShuffleWithPACK ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerShuffleWithPERMV()

static SDValue lowerShuffleWithPERMV ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithPSHUFB()

static SDValue lowerShuffleWithPSHUFB ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithSHUFPD()

static SDValue lowerShuffleWithSHUFPD ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithSHUFPS()

static SDValue lowerShuffleWithSHUFPS ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static

Lower a vector shuffle using the SHUFPS instruction.

This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.

Definition at line 12914 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::commuteMask(), llvm::count_if(), DL, llvm::find_if(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), lowerShuffleWithSHUFPS(), llvm::X86ISD::SHUFP, and std::swap().

Referenced by lowerShuffleWithSHUFPS(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV4F32Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

◆ lowerShuffleWithSSE4A()

static SDValue lowerShuffleWithSSE4A ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithUndefHalf()

static SDValue lowerShuffleWithUndefHalf ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithUNPCK()

static SDValue lowerShuffleWithUNPCK ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithUNPCK256()

static SDValue lowerShuffleWithUNPCK256 ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static

◆ lowerShuffleWithVPMOV()

static SDValue lowerShuffleWithVPMOV ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerSIGN_EXTEND()

static SDValue LowerSIGN_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerSIGN_EXTEND_Mask()

static SDValue LowerSIGN_EXTEND_Mask ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerStore()

static SDValue LowerStore ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 24570 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeAction(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is32BitVector(), llvm::MVT::is64BitVector(), isFreeToSplitVector(), llvm::MVT::isInteger(), llvm::StoreSDNode::isTruncatingStore(), llvm::EVT::isVector(), splitVectorStore(), llvm::ISD::TRUNCATE, llvm::TargetLoweringBase::TypeWidenVector, and llvm::X86ISD::VEXTRACT_STORE.

Referenced by llvm::X86TargetLowering::LowerOperation().

◆ lowerToAddSubOrFMAddSub()

static SDValue lowerToAddSubOrFMAddSub ( const BuildVectorSDNode BV,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerToHorizontalOp()

static SDValue LowerToHorizontalOp ( const BuildVectorSDNode BV,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerToTLSExecModel()

static SDValue LowerToTLSExecModel ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT,
TLSModel::Model  model,
bool  is64Bit,
bool  isPIC 
)
static

◆ LowerToTLSGeneralDynamicModel32()

static SDValue LowerToTLSGeneralDynamicModel32 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static

◆ LowerToTLSGeneralDynamicModel64()

static SDValue LowerToTLSGeneralDynamicModel64 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static

◆ LowerToTLSGeneralDynamicModelX32()

static SDValue LowerToTLSGeneralDynamicModelX32 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static

◆ LowerToTLSLocalDynamicModel()

static SDValue LowerToTLSLocalDynamicModel ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT,
bool  Is64Bit,
bool  Is64BitLP64 
)
static

◆ LowerTruncateVecI1()

static SDValue LowerTruncateVecI1 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerTruncateVecPack()

static SDValue LowerTruncateVecPack ( MVT  DstVT,
SDValue  In,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerTruncateVecPackWithSignBits()

static SDValue LowerTruncateVecPackWithSignBits ( MVT  DstVT,
SDValue  In,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.

vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.

Definition at line 20435 of file X86ISelLowering.cpp.

References DL, llvm::MVT::getHalfNumVectorElementsVT(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), isUpperSubvectorUndef(), llvm::Lo, LowerTruncateVecPackWithSignBits(), matchTruncateWithPACK(), truncateVectorWithPACK(), and widenSubVector().

Referenced by LowerTruncateVecPackWithSignBits().

◆ LowerUINT_TO_FP_i32()

static SDValue LowerUINT_TO_FP_i32 ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerUINT_TO_FP_i64()

static SDValue LowerUINT_TO_FP_i64 ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerUINT_TO_FP_v2i32()

static SDValue lowerUINT_TO_FP_v2i32 ( SDValue  Op,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerUINT_TO_FP_vec()

static SDValue lowerUINT_TO_FP_vec ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerUINT_TO_FP_vXi32()

static SDValue lowerUINT_TO_FP_vXi32 ( SDValue  Op,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ lowerV16F32Shuffle()

static SDValue lowerV16F32Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV16I16Shuffle()

static SDValue lowerV16I16Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV16I32Shuffle()

static SDValue lowerV16I32Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV16I8Shuffle()

static SDValue lowerV16I8Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Generic lowering of v16i8 shuffles.

This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.

Definition at line 14087 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), canLowerByDroppingElements(), llvm::copy_if(), llvm::count_if(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), I, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithSSE4A(), lowerShuffleWithUNPCK(), lowerShuffleWithVPMOV(), llvm::none_of(), llvm::X86ISD::PACKUS, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VPPERM, and llvm::X86ISD::VSRLI.

Referenced by lower128BitShuffle().

◆ lowerV2F64Shuffle()

static SDValue lowerV2F64Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle lowering of 2-lane 64-bit floating point shuffles.

This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.

Definition at line 12730 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::SM_SentinelUndef, and llvm::X86ISD::VPERMILPI.

Referenced by lower128BitShuffle().

◆ lowerV2I64Shuffle()

static SDValue lowerV2I64Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle lowering of 2-lane 64-bit integer shuffles.

Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.

Definition at line 12814 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), and llvm::X86ISD::PSHUFD.

Referenced by lower128BitShuffle().

◆ lowerV2X128Shuffle()

static SDValue lowerV2X128Shuffle ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV32I16Shuffle()

static SDValue lowerV32I16Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV32I8Shuffle()

static SDValue lowerV32I8Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV4F32Shuffle()

static SDValue lowerV4F32Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV4F64Shuffle()

static SDValue lowerV4F64Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV4I32Shuffle()

static SDValue lowerV4I32Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV4I64Shuffle()

static SDValue lowerV4I64Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV4X128Shuffle()

static SDValue lowerV4X128Shuffle ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV64I8Shuffle()

static SDValue lowerV64I8Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV8F16Shuffle()

static SDValue lowerV8F16Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV8F32Shuffle()

static SDValue lowerV8F32Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV8F64Shuffle()

static SDValue lowerV8F64Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV8I16GeneralSingleInputShuffle()

static SDValue lowerV8I16GeneralSingleInputShuffle ( const SDLoc DL,
MVT  VT,
SDValue  V,
MutableArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.

The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.

The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.

This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.

Definition at line 13251 of file X86ISelLowering.cpp.

References llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::copy_if(), llvm::count(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::is_contained(), isNoopShuffleMask(), isSequentialOrUndefInRange(), isUndefOrEqual(), isUndefOrInRange(), llvm_unreachable, llvm::lower_bound(), lowerV8I16GeneralSingleInputShuffle(), llvm::M0(), llvm::M1(), llvm::Match, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), and std::swap().

Referenced by lowerV16I16Shuffle(), lowerV32I16Shuffle(), lowerV8I16GeneralSingleInputShuffle(), and lowerV8I16Shuffle().

◆ lowerV8I16Shuffle()

static SDValue lowerV8I16Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Generic lowering of 8-lane i16 shuffles.

This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.

The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.

Definition at line 13808 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::any_of(), assert(), llvm::X86ISD::BLENDI, canLowerByDroppingElements(), concatSubVectors(), llvm::count_if(), DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithSSE4A(), lowerShuffleWithUNPCK(), lowerShuffleWithVPMOV(), lowerV8I16GeneralSingleInputShuffle(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.

Referenced by lower128BitShuffle().

◆ lowerV8I32Shuffle()

static SDValue lowerV8I32Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerV8I64Shuffle()

static SDValue lowerV8I64Shuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const APInt Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerVACOPY()

static SDValue LowerVACOPY ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerVECTOR_SHUFFLE()

static SDValue lowerVECTOR_SHUFFLE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Top-level lowering for x86 vector shuffles.

This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.

Definition at line 17558 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::any_of(), assert(), canCombineAsMaskOperation(), canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), llvm::ShuffleVectorSDNode::commuteMask(), computeZeroableShuffleElements(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm_unreachable, lower128BitShuffle(), lower1BitShuffle(), lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBroadcast(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelZero, and std::swap().

◆ LowerVectorAllEqual()

static SDValue LowerVectorAllEqual ( const SDLoc DL,
SDValue  LHS,
SDValue  RHS,
ISD::CondCode  CC,
const APInt OriginalMask,
const X86Subtarget Subtarget,
SelectionDAG DAG,
X86::CondCode X86CC 
)
static

◆ LowerVectorCTLZ()

static SDValue LowerVectorCTLZ ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerVectorCTLZ_AVX512CDI()

static SDValue LowerVectorCTLZ_AVX512CDI ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerVectorCTLZInRegLUT()

static SDValue LowerVectorCTLZInRegLUT ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerVectorCTPOP()

static SDValue LowerVectorCTPOP ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerVectorCTPOPInRegLUT()

static SDValue LowerVectorCTPOPInRegLUT ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ lowerVSELECTtoVectorShuffle()

static SDValue lowerVSELECTtoVectorShuffle ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a VSELECT instruction to a vector shuffle.

Definition at line 17700 of file X86ISelLowering.cpp.

References Cond, createShuffleMaskFromVSELECT(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::isBuildVectorOfConstantSDNodes(), LHS, and RHS.

◆ LowerVSETCC()

static SDValue LowerVSETCC ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 23159 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::EVT::changeVectorElementTypeToInteger(), cheapX86FSETCC_SSE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, llvm::SelectionDAG::ComputeNumSignBits(), Cond, EQ, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), incDecVectorConstant(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isConstantSplatVector(), llvm::isConstOrConstSplat(), llvm::MVT::isFloatingPoint(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMinSignedValue(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), isSoftF16(), llvm::ISD::isTrueWhenEqual(), llvm::ISD::isUnsignedIntSetCC(), llvm::APInt::isZero(), llvm_unreachable, llvm::APInt::logBase2(), LowerIntVSETCC_AVX512(), LowerVSETCCWithSUBUS(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::ISD::SETEQ, llvm::SDNode::setFlags(), llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::SelectionDAG::SignBitIsZero(), splitIntVSETCC(), llvm::ISD::SRA, llvm::X86ISD::STRICT_CMPM, llvm::X86ISD::STRICT_CMPP, llvm::ISD::STRICT_FSETCC, llvm::ISD::STRICT_FSETCCS, std::swap(), llvm::ISD::TokenFactor, translateX86FSETCC(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::X86ISD::VPCOM, llvm::X86ISD::VPCOMU, and llvm::ISD::XOR.

◆ LowerVSETCCWithSUBUS()

static SDValue LowerVSETCCWithSUBUS ( SDValue  Op0,
SDValue  Op1,
MVT  VT,
ISD::CondCode  Cond,
const SDLoc dl,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowervXi8MulWithUNPCK()

static SDValue LowervXi8MulWithUNPCK ( SDValue  A,
SDValue  B,
const SDLoc dl,
MVT  VT,
bool  IsSigned,
const X86Subtarget Subtarget,
SelectionDAG DAG,
SDValue Low = nullptr 
)
static

◆ lowerX86CmpEqZeroToCtlzSrl()

static SDValue lowerX86CmpEqZeroToCtlzSrl ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ lowerX86FPLogicOp()

static SDValue lowerX86FPLogicOp ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ LowerXALUO()

static SDValue LowerXALUO ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ LowerZERO_EXTEND()

static SDValue LowerZERO_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ LowerZERO_EXTEND_Mask()

static SDValue LowerZERO_EXTEND_Mask ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ MarkEHGuard()

static SDValue MarkEHGuard ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ MarkEHRegistrationNode()

static SDValue MarkEHRegistrationNode ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ match1BitShuffleAsKSHIFT()

static int match1BitShuffleAsKSHIFT ( unsigned Opcode,
ArrayRef< int >  Mask,
int  MaskOffset,
const APInt Zeroable 
)
static

◆ matchAsm()

static bool matchAsm ( StringRef  S,
ArrayRef< const char * >  Pieces 
)
static

◆ matchBinaryPermuteShuffle()

static bool matchBinaryPermuteShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
const APInt Zeroable,
bool  AllowFloatDomain,
bool  AllowIntDomain,
SDValue V1,
SDValue V2,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT ShuffleVT,
unsigned PermuteImm 
)
static

◆ matchBinaryShuffle()

static bool matchBinaryShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
bool  AllowFloatDomain,
bool  AllowIntDomain,
SDValue V1,
SDValue V2,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT SrcVT,
MVT DstVT,
bool  IsUnary 
)
static

◆ matchLogicBlend()

static bool matchLogicBlend ( SDNode N,
SDValue X,
SDValue Y,
SDValue Mask 
)
static

◆ matchPMADDWD()

static SDValue matchPMADDWD ( SelectionDAG DAG,
SDValue  Op0,
SDValue  Op1,
const SDLoc DL,
EVT  VT,
const X86Subtarget Subtarget 
)
static

◆ matchPMADDWD_2()

static SDValue matchPMADDWD_2 ( SelectionDAG DAG,
SDValue  N0,
SDValue  N1,
const SDLoc DL,
EVT  VT,
const X86Subtarget Subtarget 
)
static

◆ matchScalarReduction()

static bool matchScalarReduction ( SDValue  Op,
ISD::NodeType  BinOp,
SmallVectorImpl< SDValue > &  SrcOps,
SmallVectorImpl< APInt > *  SrcMask = nullptr 
)
static

◆ matchShuffleAsBitRotate()

static int matchShuffleAsBitRotate ( MVT RotateVT,
int  EltSizeInBits,
const X86Subtarget Subtarget,
ArrayRef< int >  Mask 
)
static

◆ matchShuffleAsBlend()

static bool matchShuffleAsBlend ( MVT  VT,
SDValue  V1,
SDValue  V2,
MutableArrayRef< int >  Mask,
const APInt Zeroable,
bool ForceV1Zero,
bool ForceV2Zero,
uint64_t BlendMask 
)
static

◆ matchShuffleAsByteRotate()

static int matchShuffleAsByteRotate ( MVT  VT,
SDValue V1,
SDValue V2,
ArrayRef< int >  Mask 
)
static

Try to lower a vector shuffle as a byte rotation.

SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:

v8i16 [11, 12, 13, 14, 15, 0, 1, 2]

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 11363 of file X86ISelLowering.cpp.

References is128BitLaneRepeatedShuffleMask(), isAnyZero(), matchShuffleAsElementRotate(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by lowerShuffleAsByteRotate(), and matchBinaryPermuteShuffle().

◆ matchShuffleAsElementRotate()

static int matchShuffleAsElementRotate ( SDValue V1,
SDValue V2,
ArrayRef< int >  Mask 
)
static

Try to match a vector shuffle as an element rotation.

This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.

Definition at line 11278 of file X86ISelLowering.cpp.

References assert(), llvm::Hi, llvm::Lo, and llvm::SM_SentinelUndef.

Referenced by lowerShuffleAsVALIGN(), matchBinaryPermuteShuffle(), and matchShuffleAsByteRotate().

◆ matchShuffleAsEXTRQ()

static bool matchShuffleAsEXTRQ ( MVT  VT,
SDValue V1,
SDValue V2,
ArrayRef< int >  Mask,
uint64_t BitLen,
uint64_t BitIdx,
const APInt Zeroable 
)
static

◆ matchShuffleAsInsertPS()

static bool matchShuffleAsInsertPS ( SDValue V1,
SDValue V2,
unsigned InsertPSMask,
const APInt Zeroable,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

◆ matchShuffleAsINSERTQ()

static bool matchShuffleAsINSERTQ ( MVT  VT,
SDValue V1,
SDValue V2,
ArrayRef< int >  Mask,
uint64_t BitLen,
uint64_t BitIdx 
)
static

◆ matchShuffleAsShift()

static int matchShuffleAsShift ( MVT ShiftVT,
unsigned Opcode,
unsigned  ScalarSizeInBits,
ArrayRef< int >  Mask,
int  MaskOffset,
const APInt Zeroable,
const X86Subtarget Subtarget 
)
static

Try to lower a vector shuffle as a bit shift (shifts in zeros).

Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.

PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]

Definition at line 11580 of file X86ISelLowering.cpp.

References llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), isSequentialOrUndefInRange(), llvm::Left, llvm::Low, Size, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.

Referenced by lowerShuffleAsShift(), and matchUnaryPermuteShuffle().

◆ matchShuffleAsVTRUNC()

static bool matchShuffleAsVTRUNC ( MVT SrcVT,
MVT DstVT,
MVT  VT,
ArrayRef< int >  Mask,
const APInt Zeroable,
const X86Subtarget Subtarget 
)
static

◆ matchShuffleWithPACK()

static bool matchShuffleWithPACK ( MVT  VT,
MVT SrcVT,
SDValue V1,
SDValue V2,
unsigned PackOpcode,
ArrayRef< int >  TargetMask,
const SelectionDAG DAG,
const X86Subtarget Subtarget,
unsigned  MaxStages = 1 
)
static

◆ matchShuffleWithSHUFPD()

static bool matchShuffleWithSHUFPD ( MVT  VT,
SDValue V1,
SDValue V2,
bool ForceV1Zero,
bool ForceV2Zero,
unsigned ShuffleImm,
ArrayRef< int >  Mask,
const APInt Zeroable 
)
static

◆ matchShuffleWithUNPCK()

static bool matchShuffleWithUNPCK ( MVT  VT,
SDValue V1,
SDValue V2,
unsigned UnpackOpcode,
bool  IsUnary,
ArrayRef< int >  TargetMask,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ matchTruncateWithPACK()

static SDValue matchTruncateWithPACK ( unsigned PackOpcode,
EVT  DstVT,
SDValue  In,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ matchUnaryPermuteShuffle()

static bool matchUnaryPermuteShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
const APInt Zeroable,
bool  AllowFloatDomain,
bool  AllowIntDomain,
const SelectionDAG DAG,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT ShuffleVT,
unsigned PermuteImm 
)
static

◆ matchUnaryShuffle()

static bool matchUnaryShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
bool  AllowFloatDomain,
bool  AllowIntDomain,
SDValue  V1,
const SelectionDAG DAG,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT SrcVT,
MVT DstVT 
)
static

◆ MatchVectorAllEqualTest()

static SDValue MatchVectorAllEqualTest ( SDValue  LHS,
SDValue  RHS,
ISD::CondCode  CC,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG,
X86::CondCode X86CC 
)
static

◆ materializeVectorConstant()

static SDValue materializeVectorConstant ( SDValue  Op,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Create a vector constant without a load.

SSE/AVX provide the bare minimum functionality to do this, so it's all zeros, all ones, or some derivation that is cheap to calculate.

Definition at line 8365 of file X86ISelLowering.cpp.

References DL, getOnesVector(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), and llvm::ISD::isBuildVectorAllZeros().

◆ narrowExtractedVectorSelect()

static SDValue narrowExtractedVectorSelect ( SDNode Ext,
const SDLoc DL,
SelectionDAG DAG 
)
static

If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width.

This is a common pattern for AVX1 integer code because 256-bit selects may be legal, but there is almost no integer math/logic available for 256-bit. This function should only be called with legal types (otherwise, the calls to get simple value types will assert).

Definition at line 55891 of file X86ISelLowering.cpp.

References assert(), DL, extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isFreeToSplitVector(), llvm_unreachable, and llvm::ISD::VSELECT.

Referenced by combineEXTRACT_SUBVECTOR().

◆ narrowLoadToVZLoad()

static SDValue narrowLoadToVZLoad ( LoadSDNode LN,
MVT  MemVT,
MVT  VT,
SelectionDAG DAG 
)
static

◆ narrowShuffle()

static SDValue narrowShuffle ( ShuffleVectorSDNode Shuf,
SelectionDAG DAG 
)
static

If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size.

Definition at line 41122 of file X86ISelLowering.cpp.

References getHalfShuffleMask(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDNode::getOperand(), getShuffleHalfVectors(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), and isUndefUpperHalf().

Referenced by combineShuffle().

◆ narrowVectorSelect()

static SDValue narrowVectorSelect ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)

Definition at line 44878 of file X86ISelLowering.cpp.

References llvm::X86ISD::BLENDV, Cond, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::hasOneUse(), llvm::EVT::is256BitVector(), isFreeToSplitVector(), N, SplitOpsAndApply(), and llvm::ISD::VSELECT.

Referenced by combineSelect(), and llvm::InstCombinerImpl::visitShuffleVectorInst().

◆ needCarryOrOverflowFlag()

static bool needCarryOrOverflowFlag ( SDValue  Flags)
static

◆ negateFMAOpcode()

static unsigned negateFMAOpcode ( unsigned  Opcode,
bool  NegMul,
bool  NegAcc,
bool  NegRes 
)
static

◆ onlyZeroFlagUsed()

static bool onlyZeroFlagUsed ( SDValue  Flags)
static

◆ parseConstraintCode()

static X86::CondCode parseConstraintCode ( llvm::StringRef  Constraint)
static

◆ promoteExtBeforeAdd()

static SDValue promoteExtBeforeAdd ( SDNode Ext,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.

This can eliminate extend, add, and shift instructions.

Definition at line 52849 of file X86ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::willNotOverflowAdd(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), and combineZext().

◆ PromoteMaskArithmetic() [1/2]

static SDValue PromoteMaskArithmetic ( SDValue  N,
const SDLoc DL,
EVT  VT,
SelectionDAG DAG,
unsigned  Depth 
)
static

◆ PromoteMaskArithmetic() [2/2]

static SDValue PromoteMaskArithmetic ( SDValue  N,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ promoteXINT_TO_FP()

static SDValue promoteXINT_TO_FP ( SDValue  Op,
const SDLoc dl,
SelectionDAG DAG 
)
static

◆ pushAddIntoCmovOfConsts()

static SDValue pushAddIntoCmovOfConsts ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

CMOV of constants requires materializing constant operands in registers.

Try to fold those constants into an 'add' instruction to reduce instruction count. We do this with CMOV rather the generic 'select' because there are earlier folds that may be used to turn select-of-constants into logic hacks.

Definition at line 54804 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::all_of(), llvm::X86ISD::CMOV, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm::X86::mayFoldLoad(), N, std::swap(), X, and Y.

Referenced by combineAdd().

◆ rebuildGatherScatter()

static SDValue rebuildGatherScatter ( MaskedGatherScatterSDNode GorS,
SDValue  Index,
SDValue  Base,
SDValue  Scale,
SelectionDAG DAG 
)
static

◆ recoverFramePointer()

static SDValue recoverFramePointer ( SelectionDAG DAG,
const Function Fn,
SDValue  EntryEBP 
)
static

When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.

Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - ParentFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.

Definition at line 25340 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::GlobalValue::dropLLVMManglingEscape(), llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.

◆ reduceMaskedLoadToScalarLoad()

static SDValue reduceMaskedLoadToScalarLoad ( MaskedLoadSDNode ML,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.

Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.

Definition at line 50539 of file X86ISelLowering.cpp.

References Addr, assert(), llvm::EVT::changeVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_VECTOR_ELT, ML, and llvm::Offset.

Referenced by combineMaskedLoad().

◆ reduceMaskedStoreToScalarStore()

static SDValue reduceMaskedStoreToScalarStore ( MaskedStoreSDNode MS,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.

Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.

Definition at line 50675 of file X86ISelLowering.cpp.

References Addr, llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::MachinePointerInfo::getWithOffset(), and llvm::Offset.

Referenced by combineMaskedStore().

◆ reduceVMULWidth()

static SDValue reduceVMULWidth ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.

Two typical patterns are handled: Pattern1: %2 = sext/zext <N x i8> %1 to <N x i32> %4 = sext/zext <N x i8> %3 to <N x i32> %5 = mul <N x i32> %2, %4

Pattern2: %2 = zext/sext <N x i16> %1 to <N x i32> %4 = zext/sext <N x i16> %3 to <N x i32> or %4 = build_vector <N x i32> C1, ..., CN (C1..CN are constants) %5 = mul <N x i32> %2, %4

There are four mul shrinking modes: If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is -128 to 128, and the scalar value range of %4 is also -128 to 128, generate pmullw+sext32 for it (MULS8 mode). If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is 0 to 255, and the scalar value range of %4 is also 0 to 255, generate pmullw+zext32 for it (MULU8 mode). If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767, generate pmullw+pmulhw for it (MULS16 mode). If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is 0 to 65535, and the scalar value range of %4 is also 0 to 65535, generate pmullw+pmulhuw for it (MULU16 mode).

Definition at line 46782 of file X86ISelLowering.cpp.

References canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::Function::hasMinSize(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineMul().

◆ resolveTargetShuffleFromZeroables()

static void resolveTargetShuffleFromZeroables ( SmallVectorImpl< int > &  Mask,
const APInt KnownUndef,
const APInt KnownZero,
bool  ResolveKnownZeros = true 
)
static

◆ resolveTargetShuffleInputsAndMask()

static void resolveTargetShuffleInputsAndMask ( SmallVectorImpl< SDValue > &  Inputs,
SmallVectorImpl< int > &  Mask 
)
static

◆ resolveZeroablesFromTargetShuffle()

static void resolveZeroablesFromTargetShuffle ( const SmallVectorImpl< int > &  Mask,
APInt KnownUndef,
APInt KnownZero 
)
static

◆ scalarizeExtEltFP()

static SDValue scalarizeExtEltFP ( SDNode ExtElt,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ scalarizeVectorStore()

static SDValue scalarizeVectorStore ( StoreSDNode Store,
MVT  StoreVT,
SelectionDAG DAG 
)
static

◆ scaleShuffleElements()

static bool scaleShuffleElements ( ArrayRef< int >  Mask,
unsigned  NumDstElts,
SmallVectorImpl< int > &  ScaledMask 
)
static

◆ scaleVectorShuffleBlendMask()

static uint64_t scaleVectorShuffleBlendMask ( uint64_t  BlendMask,
int  Size,
int  Scale 
)
static

Definition at line 10570 of file X86ISelLowering.cpp.

References Size.

Referenced by combineTargetShuffle().

◆ shouldExpandCmpArithRMWInIR()

static bool shouldExpandCmpArithRMWInIR ( AtomicRMWInst AI)
static

◆ shouldUseHorizontalOp()

static bool shouldUseHorizontalOp ( bool  IsSingleSource,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Horizontal vector math instructions may be slower than normal math with shuffles.

Limit horizontal op codegen based on size/speed trade-offs, uarch implementation, and likely shuffle complexity of the alternate sequence.

Definition at line 19382 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::shouldOptForSize().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineVectorHADDSUB(), isHorizontalBinOp(), lowerAddSubToHorizontalOp(), and LowerUINT_TO_FP_i64().

◆ signExtendBitcastSrcVector()

static SDValue signExtendBitcastSrcVector ( SelectionDAG DAG,
EVT  SExtVT,
SDValue  Src,
const SDLoc DL 
)
static

◆ SplitAndExtendv16i1()

static SDValue SplitAndExtendv16i1 ( unsigned  ExtOpc,
MVT  VT,
SDValue  In,
const SDLoc dl,
SelectionDAG DAG 
)
static

◆ splitAndLowerShuffle()

static SDValue splitAndLowerShuffle ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG,
bool  SimpleOnly 
)
static

◆ splitIntVSETCC()

static SDValue splitIntVSETCC ( EVT  VT,
SDValue  LHS,
SDValue  RHS,
ISD::CondCode  Cond,
SelectionDAG DAG,
const SDLoc dl 
)
static

Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back.

Definition at line 23020 of file X86ISelLowering.cpp.

References assert(), CC, llvm::ISD::CONCAT_VECTORS, Cond, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::EVT::isInteger(), LHS, RHS, llvm::ISD::SETCC, and splitVector().

Referenced by LowerVSETCC().

◆ SplitOpsAndApply()

template<typename F >
SDValue SplitOpsAndApply ( SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc DL,
EVT  VT,
ArrayRef< SDValue Ops,
F  Builder,
bool  CheckBWI = true 
)

◆ splitVector()

static std::pair< SDValue, SDValue > splitVector ( SDValue  Op,
SelectionDAG DAG,
const SDLoc dl 
)
static

◆ splitVectorIntBinary()

static SDValue splitVectorIntBinary ( SDValue  Op,
SelectionDAG DAG 
)
static

Break a binary integer operation into 2 half sized ops and then concatenate the result back.

Definition at line 4115 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), and splitVectorOp().

Referenced by combineEXTRACT_SUBVECTOR(), LowerABD(), LowerADDSAT_SUBSAT(), lowerAddSub(), LowerAVG(), LowerMINMAX(), LowerMUL(), LowerMULH(), LowerRotate(), and LowerShift().

◆ splitVectorIntUnary()

static SDValue splitVectorIntUnary ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ splitVectorOp()

static SDValue splitVectorOp ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ splitVectorStore()

static SDValue splitVectorStore ( StoreSDNode Store,
SelectionDAG DAG 
)
static

◆ supportedVectorShiftWithBaseAmnt()

static bool supportedVectorShiftWithBaseAmnt ( EVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static

Definition at line 28994 of file X86ISelLowering.cpp.

References supportedVectorShiftWithImm().

Referenced by LowerFunnelShift(), and LowerShiftByScalarVariable().

◆ supportedVectorShiftWithImm()

static bool supportedVectorShiftWithImm ( EVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static

◆ supportedVectorVarShift()

static bool supportedVectorVarShift ( EVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static

◆ TranslateIntegerX86CC()

static X86::CondCode TranslateIntegerX86CC ( ISD::CondCode  SetCCOpcode)
static

◆ TranslateX86CC()

static X86::CondCode TranslateX86CC ( ISD::CondCode  SetCCOpcode,
const SDLoc DL,
bool  isFP,
SDValue LHS,
SDValue RHS,
SelectionDAG DAG 
)
static

◆ translateX86FSETCC()

static unsigned translateX86FSETCC ( ISD::CondCode  SetCCOpcode,
SDValue Op0,
SDValue Op1,
bool IsAlwaysSignaling 
)
static

◆ truncateAVX512SetCCNoBWI()

static SDValue truncateAVX512SetCCNoBWI ( EVT  VT,
EVT  OpVT,
SDValue  LHS,
SDValue  RHS,
ISD::CondCode  CC,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization.

Definition at line 53295 of file X86ISelLowering.cpp.

References CC, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::isVector(), LHS, RHS, and llvm::ISD::TRUNCATE.

Referenced by combineSetCC().

◆ truncateVectorWithPACK()

static SDValue truncateVectorWithPACK ( unsigned  Opcode,
EVT  DstVT,
SDValue  In,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ truncateVectorWithPACKSS()

static SDValue truncateVectorWithPACKSS ( EVT  DstVT,
SDValue  In,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ truncateVectorWithPACKUS()

static SDValue truncateVectorWithPACKUS ( EVT  DstVT,
SDValue  In,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.

e.g. trunc <8 x i32> X to <8 x i16> --> MaskX = X & 0xffff (clear high bits to prevent saturation) packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)

Definition at line 20325 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86ISD::PACKUS, and truncateVectorWithPACK().

Referenced by LowerTruncateVecPack().

◆ useVectorCast()

static bool useVectorCast ( unsigned  Opcode,
MVT  FromVT,
MVT  ToVT,
const X86Subtarget Subtarget 
)
static

◆ useVPTERNLOG()

static bool useVPTERNLOG ( const X86Subtarget Subtarget,
MVT  VT 
)
static

◆ vectorizeExtractedCast()

static SDValue vectorizeExtractedCast ( SDValue  Cast,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ widenMaskVector()

static SDValue widenMaskVector ( SDValue  Vec,
bool  ZeroNewElements,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static

Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.

Definition at line 3964 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), widenMaskVectorType(), and widenSubVector().

Referenced by combineCMP(), ExtractBitFromMaskVector(), lower1BitShuffle(), lower1BitShuffleAsKSHIFTR(), and LowerEXTRACT_SUBVECTOR().

◆ widenMaskVectorType()

static MVT widenMaskVectorType ( MVT  VT,
const X86Subtarget Subtarget 
)
static

Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.

Definition at line 3954 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getVectorElementType(), and llvm::MVT::getVectorNumElements().

Referenced by insert1BitVector(), LowerCONCAT_VECTORSvXi1(), and widenMaskVector().

◆ widenSubVector() [1/2]

static SDValue widenSubVector ( MVT  VT,
SDValue  Vec,
bool  ZeroNewElements,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static

◆ widenSubVector() [2/2]

static SDValue widenSubVector ( SDValue  Vec,
bool  ZeroNewElements,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl,
unsigned  WideSizeInBits 
)
static

Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.

Definition at line 3940 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorVT(), and widenSubVector().

Variable Documentation

◆ BrMergingBaseCostThresh

cl::opt< int > BrMergingBaseCostThresh("x86-br-merging-base-cost", cl::init(2), cl::desc( "Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden) ( "x86-br-merging-base-cost"  ,
cl::init(2)  ,
cl::desc( "Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches.")  ,
cl::Hidden   
)
static

◆ BrMergingLikelyBias

cl::opt< int > BrMergingLikelyBias("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden) ( "x86-br-merging-likely-bias"  ,
cl::init(0)  ,
cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches.")  ,
cl::Hidden   
)
static

◆ BrMergingUnlikelyBias

cl::opt< int > BrMergingUnlikelyBias("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc( "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden) ( "x86-br-merging-unlikely-bias"  ,
cl::init(-1)  ,
cl::desc( "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches.")  ,
cl::Hidden   
)
static

◆ ExperimentalPrefInnermostLoopAlignment

cl::opt< int > ExperimentalPrefInnermostLoopAlignment("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc( "Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden) ( "x86-experimental-pref-innermost-loop-alignment"  ,
cl::init(4)  ,
cl::desc( "Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment.")  ,
cl::Hidden   
)
static

◆ FPStateSize

const unsigned FPStateSize = 32

Definition at line 27701 of file X86ISelLowering.cpp.

◆ FPStateSizeInBits

const unsigned FPStateSizeInBits = FPStateSize * 8

Definition at line 27702 of file X86ISelLowering.cpp.

◆ MulConstantOptimization

cl::opt< bool > MulConstantOptimization("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden) ( "mul-constant-optimization"  ,
cl::init(true ,
cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc.")  ,
cl::Hidden   
)
static

Referenced by combineMul().

◆ X87StateSize

const unsigned X87StateSize = 28

Definition at line 27700 of file X86ISelLowering.cpp.

Referenced by createSetFPEnvNodes().